diff --git a/.clang-format b/.clang-format index 1bd3afb..8960d1c 100644 --- a/.clang-format +++ b/.clang-format @@ -10,7 +10,7 @@ IndentWidth: 2 # 连续的空行保留几行 MaxEmptyLinesToKeep: 1 # 圆括号的换行方式 -BreakBeforeBraces: Custom +BreakBeforeBraces: Custom # 是否允许短方法单行 AllowShortFunctionsOnASingleLine: false # 支持一行的if @@ -81,4 +81,4 @@ BraceWrapping: # 分离空语句 SplitEmptyRecord: false # 分离空命名空间 - SplitEmptyNamespace: false \ No newline at end of file + SplitEmptyNamespace: false diff --git a/.github/workflows/Compilation.yml b/.github/workflows/Compilation.yml index 00d17ad..ce4854b 100644 --- a/.github/workflows/Compilation.yml +++ b/.github/workflows/Compilation.yml @@ -12,7 +12,7 @@ jobs: name: 'NvidiaGpuTensorRT' runs-on: ubuntu-latest container: - image: ghcr.io/teddywesside1/easy_deploy_base_dev:nvidia_gpu_tensorrt_u2204 + image: ghcr.io/zz990099/easy_deploy_tool:nvidia_gpu_trt10_u2204 options: --privileged --ipc host @@ -20,11 +20,14 @@ jobs: -w /workspace steps: - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Compile Codes run: - cd /workspace && - mkdir build && cd build && - cmake .. -DBUILD_TESTING=ON -DENABLE_TENSORRT=ON -DENABLE_ORT=ON && + cd /workspace && + mkdir build && cd build && + cmake .. -DBUILD_TESTING=ON -DENABLE_TENSORRT=ON -DENABLE_ORT=ON && make -j rk3588: @@ -33,10 +36,12 @@ jobs: permissions: contents: read packages: read # 访问容器镜像仓库的权限 - + steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive # 启用 QEMU 虚拟化支持 - name: Set up QEMU @@ -62,7 +67,7 @@ jobs: --name easy_deploy_container \ --platform linux/arm64 \ -v "$PWD:/workspace" \ - ghcr.io/teddywesside1/easy_deploy_base_dev:rknn_u2204 \ + ghcr.io/zz990099/easy_deploy_tool:rknn_230_u2204 \ tail -f /dev/null # ========== 编译执行阶段 ========== diff --git a/.github/workflows/Lint.yml b/.github/workflows/Lint.yml new file mode 100644 index 0000000..8ae99c2 --- /dev/null +++ b/.github/workflows/Lint.yml @@ -0,0 +1,26 @@ +name: pre-commit Checks + +on: + pull_request: # 在 PR 时触发 + push: # 在推送代码到 main/master 分支时触发 + branches: [main, master] + +jobs: + pre-commit: + name: Run pre-commit checks + runs-on: ubuntu-latest # 使用 Ubuntu 环境 + + steps: + - name: Checkout code + uses: actions/checkout@v4 # 检出代码 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" # 指定 Python 版本 + + - name: Install pre-commit + run: pip install pre-commit # 安装 pre-commit + + - name: Run pre-commit checks + run: pre-commit run --all-files # 运行所有检查 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8a66214 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "easy_deploy_tool"] + path = easy_deploy_tool + url = git@github.com:zz990099/EasyDeployTool.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9f43cc2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +repos: + # 官方仓库中的基础钩子 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-xml + - id: check-yaml + args: ["--allow-multiple-documents"] + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + exclude_types: [rst] + - id: fix-byte-order-marker + + # 运行 Uncrustify 格式化 C/C++ 代码 + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v20.1.0" # 指定 clang-format 版本 + hooks: + - id: clang-format + name: clang-format (check) + args: [--style=file, --dry-run, --Werror] # 检查模式 + types: [c, c++] diff --git a/CMakeLists.txt b/CMakeLists.txt index d8a9759..28df3a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,18 +2,11 @@ cmake_minimum_required(VERSION 3.8) project(easy_deployment) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - -add_subdirectory(deploy_core) -add_subdirectory(deploy_utils) +if (BUILD_TESTING) + enable_testing() +endif() -add_subdirectory(inference_core) +add_subdirectory(easy_deploy_tool) add_subdirectory(detection_2d) - add_subdirectory(sam) - -if (BUILD_TESTING) - add_subdirectory(simple_tests) -endif() \ No newline at end of file diff --git a/README.md b/README.md index 0b181c3..1fc4372 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,20 @@ # EasyDeploy - -
-
- - Logo - - -

EasyDeploy

- -

- Provides a easy way to deploy algorithms based on deep learning! -
- Report Bug or Request Feature -

-
## About The Project -The engineering deployment of deep learning algorithms relies on various inference frameworks, which often differ significantly from one another. These differences lead to low deployment and migration efficiency, especially when there is a need to support multiple hardware platforms. +This project implements several common detection and segmentation algorithms using the [EasyDeployTool](https://github.com/zz990099/EasyDeployTool) library, supporting `TensorRT`, `OnnxRuntime`, and `RKNN` inference frameworks. -The `EasyDeploy` project aims to address these challenges in two key ways: +### Models and Inference Frameworks Supported -1. **Abstracting inference framework functionalities**: By decoupling the pre-processing and post-processing procedures of algorithms from the inference process of deep learning models, `EasyDeploy` enables rapid deployment and migration of algorithms across multiple inference frameworks and hardware platforms. - -2. **Asynchronous inference pipeline**: The project implements an asynchronous inference workflow, which significantly improves model inference throughput on platforms that support multi-core parallel inference. - -### Features - -1. Abstracting inference framework (hardware platform) characteristics to enable efficient algorithm deployment and migration. - -2. Asynchronous inference pipeline to improve workflow throughput. - -3. Supporting segmented and distributed model inference, enabling asynchronous inference across devices such as CPU, GPU and NPU. - -### Models and Inference Frameworks Supported - -- **Deployed Inference Frameworks**: - 1. TensorRT - 2. ONNX-Runtime +- **Deployed Inference Frameworks**: + 1. TensorRT + 2. ONNX-Runtime 3. RKNN -- **Deployed Algorithms**: - 1. YOLOv8 - 2. RT-DETR - 3. MobileSAM +- **Deployed Algorithms**: + 1. YOLOv8 + 2. RT-DETR + 3. MobileSAM 4. NanoSAM ## Demo test Results @@ -105,46 +76,63 @@ The following test results showcase the visualized reasoning outcomes of the alg ## Getting Started -### Dependency - -- The `EasyDeploy` project is entirely written in C++ and built using the CMake tool. It relies on the following dependencies: - - **OpenCV** - - **CMake** - - **glog** - - **GoogleTest** - - Specific dependencies for each **inference framework** - -### Environment Build -- Follow [EnvironmentSetup](doc/EnviromentSetup.md) to setup enviroment with scripts quickly. - -## What You Could Do With This Project - -EasyDeploy aims to minimize the impact of inference framework-specific characteristics on the deployment of deep learning algorithms. To achieve this, we have developed an abstract base class named BaseInferCore and created specialized base classes for certain types of algorithms, such as 2D detection and instance segmentation. - -Additionally, EasyDeploy provides an asynchronous inference pipeline to further enhance deployment efficiency. - -With these features, EasyDeploy offers the following capabilities: - -- **Direct use of pre-implemented algorithms**: - - If you need to directly use algorithms such as YOLOv8, RT-DETR, MobileSAM or NanoSAM, EasyDeploy has already implemented and optimized their deployment. - - [QuickStart](doc/QuickStart.md) may help. - -- **Deploying a new algorithm efficiently**: - - If you need to deploy a new algorithm without worrying about the specific implementation details of inference frameworks, or if you want to easily migrate your algorithm to other inference frameworks, the BaseInferCore abstract base class can help you quickly implement and migrate the algorithm. - - [HowToDeployModels](doc/HowToDeployModels.md) may help. - -- **Migrating algorithms to a new inference framework**: - - If you want to migrate algorithms based on BaseInferCore to a new inference framework, implementing a subclass of BaseInferCore will allow you to migrate all algorithms to the new framework with ease. - - [HowToDeployModels](doc/HowToDeployModels.md) may help. - -- **Improving inference throughput**: - - If you need to increase the throughput of algorithm inference, EasyDeploy provides an asynchronous inference pipeline. For certain algorithm types (e.g., 2D detection, SAM), asynchronous base classes are already available, enabling you to boost the throughput of your models with minimal effort. - -- **Segmented distributed asynchronous inference**: - - If you need to implement simple segmented, distributed, asynchronous inference for algorithms, the abstract base classes and asynchronous pipeline features provided in EasyDeploy make it easy to achieve this functionality. - -## Todo - -- [ ] Use `rga` library on rk3588 to do image pre-process. -- [x] Update ONNX-Runtime inference core code and test it out. -- [x] Test TensorRT inference framework on jetson platform. +### Download Project + +Clone the git repository: +```bash +git clone git@github.com:zz990099/EasyDeploy.git +cd EasyDeploy +git submodule init && git submodule update +``` + +### Build Enviroment + +Set up the working environment using Docker: +```bash +cd EasyDeploy +bash easy_deploy_tool/docker/easy_deploy_startup.sh +# Select `jetson` -> `trt10_u2204`/`trt8_u2204` +bash easy_deploy_tool/docker/into_docker.sh +``` + +### Compile Codes + +Inside the Docker container, compile the project. Use the `-DENABLE_*` flags to enable specific inference frameworks. Available options: `-DENABLE_TENSORRT=ON`, `-DENABLE_ORT=ON`, `-DENABLE_RKNN=ON` (compatible with each other). +```bash +cd /workspace +mdkir build && cd build +cmake .. -DBUILD_TESTING=ON -DENABLE_TENSORRT=ON # -DENABLE_RKNN=ON +make -j +``` + +### Convert Model + +1. Download models from [Google Drive](https://drive.google.com/drive/folders/1yVEOzo59aob_1uXwv343oeh0dTKuHT58?usp=drive_link) and place them in `/workspace/models/`. + +2. Inside the Docker container, run the model conversion script: +```bash +cd /workspace +bash tools/cvt_onnx2trt.sh +# bash tools/cvt_onnx2rknn_all.sh +``` + +### Run Test Cases + +1. Execute test cases (refer to the code for details): +```bash +cd /workspace/build +# List available test cases +ctest -N +# List available test cases +ctest +``` + +## References + +- [ultralytics](https://github.com/ultralytics/ultralytics) +- [ultralytics-rknn](https://github.com/airockchip/ultralytics_yolov8/blob/main/RKOPT_README.md) +- [rt-detr](https://github.com/lyuwenyu/RT-DETR) +- [sam](https://github.com/facebookresearch/segment-anything) +- [mobilesam](https://github.com/ChaoningZhang/MobileSAM) +- [nanosam](https://github.com/NVIDIA-AI-IOT/nanosam) +- [EasyDeployTool](https://github.com/zz990099/EasyDeployTool) diff --git a/deploy_core/CMakeLists.txt b/deploy_core/CMakeLists.txt deleted file mode 100644 index 6b92bec..0000000 --- a/deploy_core/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(deploy_core) - -add_compile_options(-std=c++17) -add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) -set(CMAKE_CXX_STANDARD 17) - - -find_package(OpenCV REQUIRED) -find_package(glog REQUIRED) - -include_directories( - include - ${OpenCV_INCLUDE_DIRS} -) - -set(source_file src/base_infer_core.cpp - src/base_detection.cpp - src/base_sam.cpp -) - -add_library(${PROJECT_NAME} SHARED ${source_file}) - - -target_link_libraries(${PROJECT_NAME} PUBLIC - ${OpenCV_LIBS} - glog::glog -) - -install(TARGETS ${PROJECT_NAME} - LIBRARY DESTINATION lib) - -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) diff --git a/deploy_core/README.md b/deploy_core/README.md deleted file mode 100644 index 6f71793..0000000 --- a/deploy_core/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Deploy Core - -The `DeployCore` module defines the abstract functionalities for all components, including core inference capabilities, 2D detection features, SAM functionalities, and more. Beyond merely defining abstract functions, DeployCore also provides external encapsulations for certain algorithms. When implementing an algorithm, developers only need to focus on completing the key processes outlined in these definitions to achieve seamless algorithm deployment. - -## Functionality - -`DeployCore` is designed to provide abstract interface definitions for the functionalities of all modules, as well as abstract base classes containing reusable code. - -- Abstract core inference functionality: `BaseInferCore` -- Abstract 2D detection functionality: `BaseDetection2DModel` -- Abstract SAM functionality: `BaseSamModel` -- Plug-and-play asynchronous pipeline base class: `BaseAsyncPipeline` - -## Structure - -The entire project code is divided into three parts: - 1. Abstract interface classes for functional modules - 2. Abstract base classes for certain functional modules - 3. Base classes for the asynchronous inference pipeline framework - -code structure: - ```bash - deploy_core - |-- CMakeLists.txt - |-- README.md - |-- include - | `-- deploy_core - | |-- base_infer_core.h - | |-- base_detection.h - | |-- base_sam.h - | |-- async_pipeline.h - | |-- async_pipeline_impl.h - | |-- block_queue.h - | |-- common_defination.h - | `-- wrapper.h - `-- src - |-- base_detection.cpp - |-- base_infer_core.cpp - `-- base_sam.cpp - ``` - - - - Abstract interface classes for functional modules - ```bash - |-- base_infer_core.h - |-- base_detection.h - |-- base_sam.h - ``` - 1. **`base_infer_core.h`**: Defines the core inference functionalities and related abstract classes, while also providing an abstract base class for the foundational features of the inference core module. - 2. **`base_detection.h`**: Defines the abstract base class for 2D detection functionalities. - 3. **`base_sam.h`**: Defines the abstract base class for SAM functionalities. - - - Base classes for the asynchronous inference pipeline framework - ```bash - |-- async_pipeline.h - |-- async_pipeline_impl.h - |-- block_queue.h - |-- common_defination.h - `-- wrapper.h - ``` - 1. **`async_pipeline.h`** and **`async_pipeline_impl.h`**: Define the asynchronous inference framework and its implementation. - 2. **`block_queue.h`**: Implements the blocking queue. - 3. **`common_defination.h`**: Contains common definitions, such as 2D bounding boxes. - 4. **`wrapper.h`**: Provides wrappers for certain classes, such as the encapsulation of OpenCV's `cv::Mat` format. - - -## TODO - diff --git a/deploy_core/include/deploy_core/async_pipeline.h b/deploy_core/include/deploy_core/async_pipeline.h deleted file mode 100644 index 8593c78..0000000 --- a/deploy_core/include/deploy_core/async_pipeline.h +++ /dev/null @@ -1,235 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:00:38 - * @LastEditTime: 2024-11-26 22:29:22 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/async_pipeline.h - */ -#ifndef ___DEPLOY_CORE_ASYNC_PIPELINE_H -#define ___DEPLOY_CORE_ASYNC_PIPELINE_H - -#include -#include -#include -#include -#include - -#include -#include - -#include "deploy_core/async_pipeline_impl.h" -#include "deploy_core/blob_buffer.h" -#include "deploy_core/block_queue.h" - -namespace async_pipeline { - -/** - * @brief A abstract class of image data. Needed by pipeline processing. Useful when data is - * on device or other location which host cant read/write data directly. Could enable the - * zero-copy feature if needed. - * - */ -class IPipelineImageData { -public: - struct ImageDataInfo { - uint8_t *data_pointer; - int image_height; - int image_width; - int image_channels; - DataLocation location; - ImageDataFormat format; - }; - virtual const ImageDataInfo &GetImageDataInfo() const = 0; - -protected: - virtual ~IPipelineImageData() = default; -}; - -/** - * @brief The basic unit pointer which is parsed in the pipeline processing. - * - */ -class IPipelinePackage { -public: - /** - * @brief `GetInferBuffer` provides the interface to get the blobs buffer instance - * which will be used to deploy inference. Case the algorithm may need multiple inference - * core and multiple blobs buffer to complete the whole processing. - * - * @return std::shared_ptr - */ - virtual std::shared_ptr GetInferBuffer() = 0; - -protected: - virtual ~IPipelinePackage() = default; -}; - -/** - * @brief This base class provides a simple implementation of the asynchronous inference - * pipeline which could be plug-and-play. - * - * `BaseAsyncPipeline` takes function instance as a basic unit `Block` of the pipeline. User should - * call the static method `BuildPipelineBlock` to construct a `Block`. Multiple `Block`s make up - * a `Context`, which pipeline deploys the whole process on. - * - * @tparam ResultType - * @tparam GenResult - */ -template -class BaseAsyncPipeline { - using ParsingType = std::shared_ptr; - using Block_t = AsyncPipelineBlock; - using Context_t = AsyncPipelineContext; - -protected: - BaseAsyncPipeline() = default; - - ~BaseAsyncPipeline() - { - ClosePipeline(); - } - - /** - * @brief The `Block` in pipeline is constructed with a function and its name. Call this method - * in the derived class to get `Block_t` instance which is used to configure the whole pipeline. - * - * @param func - * @param block_name - * @return Block_t - */ - static Block_t BuildPipelineBlock(const std::function &func, - const std::string &block_name) - { - return Block_t(func, block_name); - } - - /** - * @brief Configure the pipelien with a `pipeline_name` and multiple `Context_t` instances. One - * derived class intance could have sereral pipelines by calling `ConfigPipeline`. - * - * @param pipeline_name - * @param block_list - */ - void ConfigPipeline(const std::string &pipeline_name, const std::vector &block_list) - { - map_name2instance_.emplace(pipeline_name, block_list); - } - -public: - /** - * @brief Get the default pipeline context. Multiple instances derived from `BaseAsyncPipeline` - * could use this method to get the context from the other to generate a more complex pipeline. - * For example, in detection_2d_yolov8, we combine the algorithm process and inference_core - * process to make a integral processing pipeline. - * - * @return const Context_t& - */ - const Context_t &GetPipelineContext() const - { - if (map_name2instance_.size() != 1) - { - throw std::runtime_error("[BaseAsyncPipeline] expect one pipeline, got " + - std::to_string(map_name2instance_.size())); - } - return map_name2instance_.begin()->second.GetContext(); - } - - /** - * @brief `PushPipeline` allow user to asynchronously push the package into pipeline and wait on - * the `future` in another thread. The instance of template type `Result` is generated by functor - * `GenResult`. - * - * @param pipeline_name - * @param package - * @return std::future - */ - [[nodiscard]] std::future PushPipeline(const std::string &pipeline_name, - const ParsingType &package) noexcept - { - if (map_name2instance_.find(pipeline_name) == map_name2instance_.end()) - { - LOG(ERROR) << "[BaseAsyncPipeline] `PushPipeline` pipeline {" << pipeline_name - << "} is not valid !!!"; - return std::future(); - } - - map_index2result_[package_index_] = std::promise(); - auto ret = map_index2result_[package_index_].get_future(); - - auto callback = [this, package_index = package_index_](const ParsingType &package) -> bool { - ResultType result = gen_result_from_package_(package); - map_index2result_[package_index].set_value(std::move(result)); - map_index2result_.erase(package_index); - return true; - }; - map_name2instance_[pipeline_name].PushPipeline(package, callback); - - package_index_++; - - return std::move(ret); - } - - /** - * @brief Return if the pipeline is initialized. - * - * @param pipeline_name - * @return true - * @return false - */ - bool IsPipelineInitialized(const std::string &pipeline_name) noexcept - { - if (map_name2instance_.find(pipeline_name) == map_name2instance_.end()) - { - return false; - } - return map_name2instance_[pipeline_name].IsInitialized(); - } - - /** - * @brief Close all pipeline. The un-finished packages will be dropped. - * - */ - void ClosePipeline() - { - for (auto &p_name_ins : map_name2instance_) - { - p_name_ins.second.ClosePipeline(); - } - } - - /** - * @brief Stop all pipeline. The un-finished packages will not be dropped. - * - */ - void StopPipeline() - { - for (auto &p_name_ins : map_name2instance_) - { - p_name_ins.second.StopPipeline(); - } - } - - /** - * @brief Initialize all configured pipeline. Call this function before push packages into - * pipeline. - * - */ - void InitPipeline() - { - for (auto &p_name_ins : map_name2instance_) - { - p_name_ins.second.Init(); - } - } - -private: - std::unordered_map> map_name2instance_; - - size_t package_index_ = 0; - std::unordered_map> map_index2result_; - GenResult gen_result_from_package_; -}; - -} // namespace async_pipeline - -#endif \ No newline at end of file diff --git a/deploy_core/include/deploy_core/async_pipeline_impl.h b/deploy_core/include/deploy_core/async_pipeline_impl.h deleted file mode 100644 index a7692c1..0000000 --- a/deploy_core/include/deploy_core/async_pipeline_impl.h +++ /dev/null @@ -1,343 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:00:38 - * @LastEditTime: 2024-11-26 21:50:48 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/async_pipeline_impl.h - */ -#ifndef __EASY_DEPLOY_ASYNC_PIPELINE_IMPL_H -#define __EASY_DEPLOY_ASYNC_PIPELINE_IMPL_H - -#include -#include -#include - -#include -#include - -#include "deploy_core/block_queue.h" - -namespace async_pipeline { - -/** - * @brief Async Pipeline Block - * - * @tparam ParsingType - */ -template -class AsyncPipelineBlock { -public: - AsyncPipelineBlock() = default; - AsyncPipelineBlock(const AsyncPipelineBlock &block) - : func_(block.func_), block_name_(block.block_name_) - {} - - AsyncPipelineBlock &operator=(const AsyncPipelineBlock &block) - { - func_ = block.func_; - block_name_ = block.block_name_; - return *this; - } - - AsyncPipelineBlock(const std::function &func) : func_(func) - {} - - AsyncPipelineBlock(const std::function &func, const std::string &block_name) - : func_(func), block_name_(block_name) - {} - - const std::string &GetName() const - { - return block_name_; - } - - bool operator()(const ParsingType &pipeline_unit) const - { - return func_(pipeline_unit); - } - -private: - std::function func_; - std::string block_name_; -}; - -/** - * @brief Async Pipeline Context - * - * @tparam ParsingType - */ -template -class AsyncPipelineContext { - using Block_t = AsyncPipelineBlock; - using Context_t = AsyncPipelineContext; - -public: - AsyncPipelineContext() = default; - - AsyncPipelineContext(const Block_t &block) : blocks_({block}) - {} - - AsyncPipelineContext(const std::vector &block_vec) - { - for (const auto &block : block_vec) - { - blocks_.push_back(block); - } - } - - AsyncPipelineContext &operator=(const std::vector &block_vec) - { - for (const auto &block : block_vec) - { - blocks_.push_back(block); - } - return *this; - } - - AsyncPipelineContext(const Context_t &context) : blocks_(context.blocks_) - {} - - AsyncPipelineContext(const std::vector &context_vec) - { - for (const auto &context : context_vec) - { - for (const auto &block : context.blocks_) - { - blocks_.push_back(block); - } - } - } - - AsyncPipelineContext &operator=(const std::vector &context_vec) - { - for (const auto &context : context_vec) - { - for (const auto &block : context.blocks_) - { - blocks_.push_back(block); - } - } - return *this; - } - - AsyncPipelineContext &operator=(const Context_t &context) - { - for (const auto &block : context.blocks_) - { - blocks_.push_back(block); - } - return *this; - } - -public: - std::vector blocks_; -}; - -/** - * @brief Async Pipeline Processing Instance - * - * @tparam ParsingType - */ -template -class PipelineInstance { - using Block_t = AsyncPipelineBlock; - using Context_t = AsyncPipelineContext; - using Callback_t = std::function; - - // for inner processing - struct _InnerPackage { - ParsingType package; - Callback_t callback; - }; - using InnerParsingType = std::shared_ptr<_InnerPackage>; - using InnerBlock_t = AsyncPipelineBlock; - using InnerContext_t = AsyncPipelineContext; - -public: - PipelineInstance() = default; - - PipelineInstance(const std::vector &block_list) : context_(block_list) - { - // initialize inner context - std::vector inner_block_list; - for (const auto &block : context_.blocks_) - { - auto func = [&](InnerParsingType p) -> bool { return block(p->package); }; - InnerBlock_t inner_block(func, block.GetName()); - inner_block_list.push_back(inner_block); - } - inner_context_ = InnerContext_t(inner_block_list); - } - - ~PipelineInstance() - { - ClosePipeline(); - } - - void Init(int bq_max_size = 100) - { - // 1. for `n` blocks, construct `n+1` block queues - const auto blocks = inner_context_.blocks_; - const int n = blocks.size(); - LOG(INFO) << "[AsyncPipelineInstance] Total {" << n << "} Pipeline Blocks"; - for (int i = 0; i < n + 1; ++i) - { - block_queue_.emplace_back(std::make_shared>(bq_max_size)); - } - pipeline_close_flag_.store(false); - - async_futures_.resize(n + 1); - // 2. open `n` async threads to execute blocks - for (int i = 0; i < n; ++i) - { - async_futures_[i] = std::async(&PipelineInstance::ThreadExcuteEntry, this, block_queue_[i], - block_queue_[i + 1], blocks[i]); - } - // 3. open output threads to execute callback - async_futures_[n] = std::async(&PipelineInstance::ThreadOutputEntry, this, block_queue_[n]); - - pipeline_initialized_.store(true); - } - - void ClosePipeline() - { - if (pipeline_initialized_) - { - LOG(INFO) << "[AsyncPipelineInstance] Closing pipeline ..."; - for (const auto &bq : block_queue_) - { - bq->DisableAndClear(); - } - LOG(INFO) << "[AsyncPipelineInstance] Disabled all block queue ..."; - pipeline_close_flag_.store(true); - - for (auto &future : async_futures_) - { - auto res = future.get(); - } - LOG(INFO) << "[AsyncPipelineInstance] Join all block queue ..."; - block_queue_.clear(); - LOG(INFO) << "[AsyncPipelineInstance] Async pipeline is released successfully!!"; - pipeline_initialized_ = false; - pipeline_close_flag_.store(true); - pipeline_no_more_input_.store(true); - } - } - - void StopPipeline() - { - if (pipeline_initialized_) - { - pipeline_no_more_input_.store(true); - block_queue_[0]->SetNoMoreInput(); - } - } - - bool IsInitialized() const - { - return pipeline_initialized_; - } - - const Context_t &GetContext() const - { - return context_; - } - - void PushPipeline(const ParsingType &obj, const Callback_t &callback) - { - auto inner_pack = std::make_shared<_InnerPackage>(); - inner_pack->package = obj; - inner_pack->callback = callback; - - block_queue_[0]->BlockPush(inner_pack); - } - -private: - bool ThreadExcuteEntry(std::shared_ptr> bq_input, - std::shared_ptr> bq_output, - const InnerBlock_t &pipeline_block) - { - LOG(INFO) << "[AsyncPipelineInstance] {" << pipeline_block.GetName() << "} thread start!"; - while (!pipeline_close_flag_) - { - auto data = bq_input->Take(); - if (!data.has_value()) - { - if (pipeline_no_more_input_) - { - LOG(INFO) << "[AsyncPipelineInstance] {" << pipeline_block.GetName() - << "} set no more output ..."; - bq_output->SetNoMoreInput(); - break; - } else - { - continue; - } - } - auto start = std::chrono::high_resolution_clock::now(); - bool status = pipeline_block(data.value()); - auto end = std::chrono::high_resolution_clock::now(); - LOG(INFO) << "[AsyncPipelineInstance] {" << pipeline_block.GetName() << "} cost (us) : " - << std::chrono::duration_cast(end - start).count(); - - if (!status) - { - LOG(WARNING) << "[AsyncPipelineInstance] {" << pipeline_block.GetName() - << "}, excute block function failed! Drop package."; - continue; - } - - bq_output->BlockPush(data.value()); - } - LOG(INFO) << "[AsyncPipelineInstance] {" << pipeline_block.GetName() << "} thread quit!"; - return true; - } - - bool ThreadOutputEntry(std::shared_ptr> bq_input) - { - LOG(INFO) << "[AsyncPipelineInstance] {Output} thread start!"; - while (!pipeline_close_flag_) - { - auto data = bq_input->Take(); - if (!data.has_value()) - { - if (pipeline_no_more_input_) - { - LOG(INFO) << "[AsyncPipelineInstance] {Output} set no more output ..."; - break; - } else - { - continue; - } - } - const auto &inner_pack = data.value(); - if (inner_pack != nullptr && inner_pack->callback != nullptr) - { - inner_pack->callback(inner_pack->package); - } else - { - LOG(WARNING) - << "[AsyncPipelineInstance] {Output} package without valid callback will be dropped!!!"; - } - } - LOG(INFO) << "[AsyncPipelineInstance] {Output} thread quit!"; - - return true; - } - -private: - Context_t context_; - - InnerContext_t inner_context_; - - std::vector>> block_queue_; - std::vector> async_futures_; - - std::atomic pipeline_close_flag_{true}; - std::atomic pipeline_no_more_input_{true}; - std::atomic pipeline_initialized_{false}; -}; - -} // namespace async_pipeline - -#endif diff --git a/deploy_core/include/deploy_core/base_detection.h b/deploy_core/include/deploy_core/base_detection.h deleted file mode 100644 index fa999ff..0000000 --- a/deploy_core/include/deploy_core/base_detection.h +++ /dev/null @@ -1,212 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:24:19 - * @LastEditTime: 2024-12-02 20:03:34 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/base_detection.h - */ -#ifndef __DEPLOY_CORE_BASE_DETECTION_H -#define __DEPLOY_CORE_BASE_DETECTION_H - -#include -#include -#include -#include - -#include - -#include "deploy_core/async_pipeline.h" -#include "deploy_core/base_infer_core.h" - -namespace detection_2d { - -/** - * @brief A abstract interface class which defines the preprocess interface of detection_2d - * algorithms. - * - */ -class IDetectionPreProcess { -public: - virtual float Preprocess(std::shared_ptr input_image_data, - std::shared_ptr blob_buffer, - const std::string &blob_name, - int dst_height, - int dst_width) = 0; -}; - -/** - * @brief A abstract interface class which defines the postprocess interface of detection_2d - * algorithms. - * - */ -class IDetectionPostProcess { -public: - virtual void Postprocess(const std::vector &output_blobs_ptr, - std::vector &results, - float conf_threshold, - float transform_scale) = 0; -}; - -/** - * @brief The common detection_2d pipeline package wrapper. - * - */ -struct DetectionPipelinePackage : public async_pipeline::IPipelinePackage { - // the wrapped pipeline image data - std::shared_ptr input_image_data; - // confidence used in postprocess - float conf_thresh; - // record the transform factor during image preprocess - float transform_scale; - // the detection result - std::vector results; - - // maintain the blobs buffer instance - std::shared_ptr infer_buffer; - - // override from `IPipelinePakcage`, to provide the blobs buffer to inference_core - std::shared_ptr GetInferBuffer() override - { - if (infer_buffer == nullptr) - { - LOG(ERROR) << "[DetectionPipelinePackage] returned nullptr of infer_buffer!!!"; - } - return infer_buffer; - } -}; - -/** - * @brief A abstract class defines two pure virtual methods -- `PreProcess` and `PostProcess`. - * The derived class could only override these methods to make it work. - * - */ -class IDetectionModel { -public: - IDetectionModel() = default; - -protected: - virtual ~IDetectionModel() = default; - - /** - * @brief PreProcess-Stage. Inside the method, you should cast the `pipeline_unit` pointer to - * `DetectionPipelinePackage` type pointer, and check if the convertion works. If the package - * pointer is not valid or anything goes wrong, it should return `false` to mention the inference - * processing to drop the package. - * - * @param pipeline_unit - * @return true - * @return false - */ - virtual bool PreProcess(std::shared_ptr pipeline_unit) = 0; - - /** - * @brief PostProcess-Stage. Inside the method, you should cast the `pipeline_unit` pointer to - * `DetectionPipelinePackage` type pointer, and check if the convertion works. If the package - * pointer is not valid or anything goes wrong, it should return `false` to mention the inference - * processing to drop the package. - * - * @param pipeline_unit - * @return true - * @return false - */ - virtual bool PostProcess(std::shared_ptr pipeline_unit) = 0; -}; - -/** - * @brief A functor to generate detection results from `DetectionPipelinePackage`. Used in async - * pipeline. - * - */ -class DetectionGenResultType { -public: - std::vector operator()(const std::shared_ptr &package) - { - auto detection_package = std::dynamic_pointer_cast(package); - if (detection_package == nullptr) - { - LOG(ERROR) << "[DetectionGenResult] Got INVALID package ptr!!!"; - return {}; - } - return std::move(detection_package->results); - } -}; - -/** - * @brief The base class of detection_2d algorithms. It implements `Detect` which is the synchronous - * version of detection and `DetectAsync` which is the asynchronous version of detection. - * - * @note Call `InitPipeline()` before you intend to use `DetectAsync`. And Check if `DetectAsync` - * returns a valid `std::future<>` instance before involke `get()` method. - * - */ -class BaseDetectionModel - : public IDetectionModel, - public async_pipeline::BaseAsyncPipeline, DetectionGenResultType> { - typedef std::shared_ptr ParsingType; - -public: - BaseDetectionModel(std::shared_ptr infer_core); - - /** - * @brief Run the detection processing in synchronous mode. - * - * @param input_image input image in cv::Mat format. - * @param det_results the output results - * @param conf_thresh confidence threshold - * @param isRGB if the input is rgb format. Will flip channels if `isRGB` == false. - * @return true - * @return false - */ - bool Detect(const cv::Mat &input_image, - std::vector &det_results, - float conf_thresh, - bool isRGB = false) noexcept; - - /** - * @brief Run the detection processing in asynchronous mode. - * - * @param input_image input image in cv::Mat format. - * @param conf_thresh confidence threshold - * @param isRGB if the input is rgb format. Will flip channels if `isRGB` == false. default=false. - * @param cover_oldest whether cover the oldest package if the pipeline queue is full. - * default=false. - * @return std::future> - */ - [[nodiscard]] std::future> DetectAsync(const cv::Mat &input_image, - float conf_thresh, - bool isRGB = false, - bool cover_oldest = false) noexcept; - -protected: - // forbidden the access from outside to `BaseAsyncPipeline::PushPipeline` - using BaseAsyncPipeline::PushPipeline; - - virtual ~BaseDetectionModel(); - - std::shared_ptr infer_core_{nullptr}; - - static std::string detection_pipeline_name_; -}; - -/** - * @brief Abstract factory class of detection_2d model. - * - */ -class BaseDetection2DFactory { -public: - virtual std::shared_ptr Create() = 0; -}; - -class BaseDetectionPreprocessFactory { -public: - virtual std::shared_ptr Create() = 0; -}; - -class BaseDetectionPostprocessFactory { -public: - virtual std::shared_ptr Create() = 0; -}; - -} // namespace detection_2d - -#endif \ No newline at end of file diff --git a/deploy_core/include/deploy_core/base_infer_core.h b/deploy_core/include/deploy_core/base_infer_core.h deleted file mode 100644 index 37e9124..0000000 --- a/deploy_core/include/deploy_core/base_infer_core.h +++ /dev/null @@ -1,249 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-26 08:42:05 - * @LastEditTime: 2024-12-02 19:03:37 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/base_infer_core.h - */ -#ifndef __EASY_DEPLOY_BASE_INFER_CORE_H -#define __EASY_DEPLOY_BASE_INFER_CORE_H - -#include -#include -#include -#include - -#include "deploy_core/block_queue.h" -#include "deploy_core/async_pipeline.h" - -namespace inference_core { - -enum InferCoreType { ONNXRUNTIME, TENSORRT, RKNN, NOT_PROVIDED }; - -/** - * @brief `IRotInferCore` is abstract interface class which defines all pure virtual functions - * that the derived class should implement, e.g., `PreProcess`, `Inference` and `PostProcess`. - * - */ -class IRotInferCore { -public: - /** - * @brief `AllocBlobsBuffer` is a common interface that user could get a brand new buffer - * instance by. This pure virtual function is implemented by actual inference core, which - * may take a while to process. Use pre-allocated buffer instance in mem buffer pool could - * get better performance. See `BaseInferCore`. - * - * @return std::shared_ptr A brand new buffer instance allocated by inference - * core. - */ - virtual std::shared_ptr AllocBlobsBuffer() = 0; - - /** - * @brief Get the core type. - * - * @return InferCoreType - */ - virtual InferCoreType GetType() - { - return InferCoreType::NOT_PROVIDED; - } - - /** - * @brief Return the name of inference core. - * - * @return std::string - */ - virtual std::string GetName() - { - return ""; - } - -protected: - virtual ~IRotInferCore() = default; - - /** - * @brief `PreProcess` stage of the inference process. Return true if this is stage is not - * needed in the actual inference core implementation. Return false if something went wrong - * while doing processing. The pipeline will drop the package if `PreProcess` returns false. - * - * @param buffer a common "pipeline" package ptr. - * @return true - * @return false - */ - virtual bool PreProcess(std::shared_ptr buffer) = 0; - - /** - * @brief `Inference` stage of the inference process. Return false if something went wrong - * while doing processing. The pipeline will drop the package if `Inference` returns false. - * - * @param buffer a common "pipeline" package ptr. - * @return true - * @return false - */ - virtual bool Inference(std::shared_ptr buffer) = 0; - - /** - * @brief `PostProcess` stage of the inference process. Return false if something went wrong - * while doing processing. The pipeline will drop the package if `PostProcess` returns false. - * - * @param buffer a common "pipeline" package ptr. - * @return true - * @return false - */ - virtual bool PostProcess(std::shared_ptr buffer) = 0; -}; - -/** - * @brief A simple implementation of mem buffer pool. Using `BlockQueue` to deploy a producer- - * consumer model. It will allocate buffer using `AllocBlobsBuffer` method of `IRotInferCore` - * and provides `IBlobsBuffer` ptr when `Alloc` method is called. The "Alloced" buffer will - * return back to mem buffer pool while the customed deconstruction method of shared_ptr ptr - * is called. - * - */ -class MemBufferPool { -public: - MemBufferPool(IRotInferCore *infer_core, const int pool_size) - : pool_size_(pool_size), dynamic_pool_(pool_size) - { - for (int i = 0; i < pool_size; ++i) - { - auto blob_buffer = infer_core->AllocBlobsBuffer(); - dynamic_pool_.BlockPush(blob_buffer.get()); - static_pool_.insert({blob_buffer.get(), blob_buffer}); - } - } - - std::shared_ptr Alloc(bool block) - { - // customed deconstruction method - auto func_dealloc = [&](IBlobsBuffer *buf) { - buf->Reset(); - this->dynamic_pool_.BlockPush(buf); - }; - - auto buf = block ? dynamic_pool_.Take() : dynamic_pool_.TryTake(); - return buf.has_value() ? std::shared_ptr(buf.value(), func_dealloc) : nullptr; - } - - void Release() - { - if (dynamic_pool_.Size() != pool_size_) - { - LOG(WARNING) << "[MemBufPool] does not maintain all bufs when release func called!"; - } - static_pool_.clear(); - } - - int RemainSize() - { - return dynamic_pool_.Size(); - } - - ~MemBufferPool() - { - Release(); - } - -private: - const int pool_size_; - BlockQueue dynamic_pool_; - std::unordered_map> static_pool_; -}; - -/** - * @brief A dummy class to help `BaseInferCore` inherit from `BaseAsyncPipeline` to generate - * async pipeline framework. - * - */ -class _DummyInferCoreGenReulstType { -public: - bool operator()(const std::shared_ptr & /*package*/) - { - return true; - } -}; - -/** - * @brief `BaseInferCore` inherits `IRotInferCore` and `BaseAsyncPipeline`. `IRotInferCore` - * defines all pure virtual methods of the abstract function of the inference core. - * `BaseAsyncPipeline` provides a set of methods to help user build and utilize a async - * inference pipeline. See `BaseAsyncPipeline` defination. - * - * @note The inheritance relationship between class A and class B is modified by protected. - * And `BaseInferCore` only makes the `GetPipelineContext` method public, which means the - * derived class of `BaseInferCore` is not supported to deploy async pipeline inference - * process. It should be used by specific algorithms in its entirety. - * - */ -class BaseInferCore : public IRotInferCore, - protected async_pipeline::BaseAsyncPipeline { -protected: - BaseInferCore(); - typedef std::shared_ptr ParsingType; - -public: - using BaseAsyncPipeline::GetPipelineContext; - - /** - * @brief This function provides a sync inference process which is completely independent - * of the async inference pipeline. Through, it depends on the three stage virtual methods - * defined in `IRotInferCore`. Return false if something went wrong while inference. - * - * @param buffer - * @param batch_size default=1, multi-batch inference may not be supported. - * @return true - * @return false - */ - bool SyncInfer(std::shared_ptr buffer, const int batch_size = 1); - - /** - * @brief Get the pre-allocated blobs buffer shared pointer. The returned pointer is a - * smart pointer which will automatically return to the pool when it is released. - * - * @param block whether to block the thread if the pool is empty. - * @return std::shared_ptr - */ - std::shared_ptr GetBuffer(bool block); - - /** - * @brief Release the sources in base class. - * - * @warning The derived class should call `BaseInferCore::Release()` in its deconstruct - * function in order to release the blobs buffer before the enviroment is destroyed. - * Things go wrong if allocated memory released after their enviroment released on some - * hardware. - * - */ - virtual void Release(); - -protected: - virtual ~BaseInferCore(); - - /** - * @brief Init the base class memory pool. - * - * @warning Please call `Init()` at the derived class construct function`s end when the - * runtime enviroment is setup successfully. This method will call `AllocBlobsBuffer` - * to create a memory pool. Temporary we manually call this method to init the memory pool. - * - * @param mem_buf_size number of blobs buffers pre-allocated. - */ - void Init(int mem_buf_size = 5); - -private: - std::unique_ptr mem_buf_pool_{nullptr}; -}; - -/** - * @brief Abstract factory class of infer_core. - * - */ -class BaseInferCoreFactory { -public: - virtual std::shared_ptr Create() = 0; -}; - -} // namespace inference_core - -#endif \ No newline at end of file diff --git a/deploy_core/include/deploy_core/base_sam.h b/deploy_core/include/deploy_core/base_sam.h deleted file mode 100644 index e309f65..0000000 --- a/deploy_core/include/deploy_core/base_sam.h +++ /dev/null @@ -1,268 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 18:38:34 - * @LastEditTime: 2024-12-02 19:03:30 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/base_sam.h - */ -#ifndef __EASY_DEPLOY_BASE_SAM_H -#define __EASY_DEPLOY_BASE_SAM_H - -#include "deploy_core/base_infer_core.h" -#include "deploy_core/common_defination.h" - -#include - -namespace sam { - -/** - * @brief The common sam pipeline package wrapper. - * - */ -struct SamPipelinePackage : public async_pipeline::IPipelinePackage { - // maintain image-encoder's blobs buffer - std::shared_ptr image_encoder_blobs_buffer; - // maintain mask-decoder's blobs buffer - std::shared_ptr mask_decoder_blobs_buffer; - - // the wrapped pipeline image data - std::shared_ptr input_image_data; - // input boxes prompt - std::vector boxes; - // input points prompt - std::vector> points; - // input points labels - std::vector labels; - // record the transform factor in image preprocessing - float transform_scale; - // mask results - cv::Mat mask; - - // the blobs buffer used in inference core processing - std::shared_ptr infer_buffer; - std::shared_ptr GetInferBuffer() override - { - return infer_buffer; - } -}; - -/** - * @brief The abstract interface class of `Segment Anything Model`(SAM) which defines - * image-preprocess、prompt-preprocess、mask-postprocess interfaces. Any SAM algorithms - * implementation could override these pure virtual methods to make up a sync/async - * inference supported pipeline. - * - * workflow: - * - * `ImagePreProcess` --> `ImageEncoderInfer` --> `PromptBoxPreProcess`/`PromptPointPreProcess` - * --> `MaskDecoderInfer` --> `MaskPostProcess` - * - */ -class ISamModel { -protected: - typedef std::shared_ptr ParsingType; - virtual ~ISamModel() = default; - /** - * @brief The `ImagePreProcess` stage. Inside the method, you should cast the `pipeline_unit` - * pointer to `SamPipelinePackage` type pointer, and check if the convertion works. If the - * package pointer is not valid or anything goes wrong, it should return `false` to mention - * the inference pipelinee to drop the package. - * - * @param pipeline_unit - * @return true - * @return false - */ - virtual bool ImagePreProcess(ParsingType pipeline_unit) = 0; - - /** - * @brief The `PromptBoxPreProcess` stage. Inside the method, you should cast the `pipeline_unit` - * pointer to `SamPipelinePackage` type pointer, and check if the convertion works. If the - * package pointer is not valid or anything goes wrong, it should return `false` to mention - * the inference pipelinee to drop the package. - * - * @param pipeline_unit - * @return true - * @return false - */ - virtual bool PromptBoxPreProcess(ParsingType pipeline_unit) = 0; - - /** - * @brief The `PromptPointPreProcess` stage. Inside the method, you should cast the - * `pipeline_unit` pointer to `SamPipelinePackage` type pointer, and check if the convertion - * works. If the package pointer is not valid or anything goes wrong, it should return `false` to - * mention the inference pipelinee to drop the package. - * - * @param pipeline_unit - * @return true - * @return false - */ - virtual bool PromptPointPreProcess(ParsingType pipeline_unit) = 0; - - /** - * @brief The `MaskPostProcess` stage. Inside the method, you should cast the `pipeline_unit` - * pointer to `SamPipelinePackage` type pointer, and check if the convertion works. If the - * package pointer is not valid or anything goes wrong, it should return `false` to mention - * the inference pipelinee to drop the package. - * - * @param pipeline_unit - * @return true - * @return false - */ - virtual bool MaskPostProcess(ParsingType pipeline_unit) = 0; -}; - -/** - * @brief A functor to generate sam results from `SamPipelinePackage`. Used in async pipeline. - * - */ -class SamGenResultType { -public: - cv::Mat operator()(const std::shared_ptr &package) - { - auto sam_package = std::dynamic_pointer_cast(package); - if (sam_package == nullptr) - { - LOG(ERROR) << "[SamGenResultType] Got INVALID package ptr!!!"; - return {}; - } - return std::move(sam_package->mask); - } -}; - -/** - * @brief The base class of SAM model. It implements `GenerateMask` and `GenerateMaskAsync` - * both with `box` prompts or `points` prompts. In the asynchronous pipeline inference mode, - * the `box` pipeline and `point` pipeline could been used in the same time, cause they are - * independent. - * - */ -class BaseSamModel : public ISamModel, - public async_pipeline::BaseAsyncPipeline { -protected: - using ParsingType = std::shared_ptr; - /** - * @brief Construct `BaseSamModel` with `image_encoder_core` and at least one of `mask_points_ - * decoder_core` or `mask_boxes_decoder_core`. Will throw exception if both decoders with points - * and boxes are nullptr. - * - * @param model_name - * @param image_encoder_core - * @param mask_points_decoder_core - * @param mask_boxes_decoder_core - */ - BaseSamModel(const std::string &model_name, - std::shared_ptr image_encoder_core, - std::shared_ptr mask_points_decoder_core, - std::shared_ptr mask_boxes_decoder_core); - - virtual ~BaseSamModel(); - -public: - /** - * @brief Generate the mask with points as prompts in sync mode. - * - * @param image input image - * @param points points coords - * @param labels points labels, 0 - background; 1 - foreground - * @param cv::Mat reference to the result. 0 - background; 255 - foreground - * @param isRGB if the input image is RGB format. default=false - * @return true - * @return false - */ - bool GenerateMask(const cv::Mat &image, - const std::vector> &points, - const std::vector &labels, - cv::Mat &result, - bool isRGB = false); - /** - * @brief Generate the mask with boxes as prompts in sync mode. - * - * @note SAM model with boxes only support one box as its prompts. More boxes wont make any - * exception, but also will not take effect. - * - * @param image input image - * @param boxes boxes coords - * @param cv::Mat reference to the result. 0 - background; 255 - foreground - * @param isRGB if the input image is RGB format. default=false - * @return true - * @return false - */ - bool GenerateMask(const cv::Mat &image, - const std::vector &boxes, - cv::Mat &result, - bool isRGB = false); - - /** - * @brief Generate the mask with points as prompts in async mode. - * - * @warning The returned `std::future<>` instance could be invalid. Please make sure it is - * valid before you call `get()`. - * - * @param image input image - * @param points points coords - * @param labels points labels, 0 - background; 1 - foreground - * @param isRGB if the input image is RGB format. default=false - * @param cover_oldest whether cover the oldest package if the pipeline queue is full. - * default=false. - * @return std::future A std::future instance of the result. - */ - [[nodiscard]] std::future GenerateMaskAsync( - const cv::Mat &image, - const std::vector> &points, - const std::vector &labels, - bool isRGB = false, - bool cover_oldest = false); - - /** - * @brief Generate the mask with boxes as prompts in async mode. - * - * @note SAM model with boxes only support one box as its prompts. More boxes wont make any - * exception, but also will not take effect. - * - * @warning The returned `std::future<>` instance could be invalid. Please make sure it is - * valid before you call `get()`. - * - * @param image input image - * @param boxes boxes coords - * @param callback callback function if needed. default=nullptr. - * @param isRGB if the input image is RGB format. default=false - * @param cover_oldest whether cover the oldest package if the pipeline queue is full. - * default=false. - * @return std::future A std::future instance of the result. - */ - [[nodiscard]] std::future GenerateMaskAsync(const cv::Mat &image, - const std::vector &boxes, - bool isRGB = false, - bool cover_oldest = false); - -private: - // forbidden the access from outside to `BaseAsyncPipeline::PushPipeline` - using BaseAsyncPipeline::PushPipeline; - - void ConfigureBoxPipeline(); - - void ConfigurePointPipeline(); - -protected: - std::shared_ptr image_encoder_core_; - std::shared_ptr mask_points_decoder_core_; - std::shared_ptr mask_boxes_decoder_core_; - - const std::string box_pipeline_name_; - const std::string point_pipeline_name_; - const std::string model_name_; -}; - - -/** - * @brief Abstract factory base class of Sam model. - * - */ -class BaseSamFactory { -public: - virtual std::shared_ptr Create() = 0; -}; - -} // namespace sam - -#endif \ No newline at end of file diff --git a/deploy_core/include/deploy_core/blob_buffer.h b/deploy_core/include/deploy_core/blob_buffer.h deleted file mode 100644 index c503bbf..0000000 --- a/deploy_core/include/deploy_core/blob_buffer.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 15:27:59 - * @LastEditTime: 2024-11-26 21:57:59 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/blob_buffer.h - */ -#ifndef __EASY_DEPLOY_BLOB_BUFFER_H -#define __EASY_DEPLOY_BLOB_BUFFER_H - -#include -#include - -#include "deploy_core/common_defination.h" - -namespace inference_core { - -/** - * @brief The key to abstracting and shielding the inference framework and hardware characteristics - * lies in how the management of inference buffer is abstracted. Considering the requirements - * of asynchronous inference framework, we encapsulated the buffer used during inference into a - * dedicated class and abstracted its functionality by developing the `IBlobsBuffer` interface - * class. The implementation of this interface must provide the following functionalities: - * - * 1. Set the buffer address to be used for inference. - * - * 2. Get the default buffer address. - * - * 3. Set the shape of the model blob. - * - * 4. Get the default blob shape. - * - * And Other base functionalities as declared below. - * - */ -class IBlobsBuffer { -public: - /** - * @brief The `BlobsBuffer` instance should provide the buffer ptr which will be used in the - * inference process. This buffer is allocated by certain inference_core by default. User could - * customize the buffer ptr by calling `SetBlobBuffer`. - * - * @param blob_name The name of the blob. - * @return std::pair Will return {nullptr, UNKOWN} if `blob_name` is - * invalid. - */ - virtual std::pair GetOuterBlobBuffer( - const std::string &blob_name) noexcept = 0; - - /** - * @brief The `BlobsBuffer` instance should provide the functionality to accept a customized - * data buffer ptr which could be on host or device. Some inference frameworks based on - * heterogeneous architecture hardware (e.g. CUDA) use buffer on device to deploy inference. There - * is no need to copy data from host to device if the device buffer ptr is provided to - * `BlobsBuffer`. - * - * @param blob_name The name of the blob. - * @param data_ptr The ptr of the customized data buffer. - * @param location Location of the customized data buffer. - * @return true - * @return false Will return false if `blob_name` is invalid. - */ - virtual bool SetBlobBuffer(const std::string &blob_name, - void *data_ptr, - DataLocation location) noexcept = 0; - - /** - * @brief `SetBlobBuffer` provides the functionality to change the default using data buffer - * on host size or device side. After calling this method, `GetOuterBlobBuffer` will return - * the buffer ptr on the certain side. - * - * @note Some inference frameworks (e.g. onnxruntime, rknn) do not distinguish buffer between - * the host side and the device side. So this method will not change their default buffer ptr. - * - * @param blob_name The name of the blob. - * @param location Location of the customized data buffer. - * @return true - * @return false Will return false if `blob_name` is invalid. - */ - virtual bool SetBlobBuffer(const std::string &blob_name, DataLocation location) noexcept = 0; - - /** - * @brief `SetBlobShape` provides the functionality to change the dynamic blob shape in the - * inference processing if the model engine allows. - * - * @note Some inference framework (e.g. rknn) do not support dynamic blob shape. And make sure - * your model supports dynamic blob shape before you call this method. - * - * @param blob_name The name of the blob. - * @param shape The dynamic blob shape. - * @return true - * @return false Will return false if `blob_name` is invalid. - */ - virtual bool SetBlobShape(const std::string &blob_name, - const std::vector &shape) noexcept = 0; - - /** - * @brief `GetBlobShape` provides the functionality to get the dynamic blob shape in the - * inference processing. By default, this will return the max blob shape which is parsed - * in `inference_core` construction. - * - * @param blob_name The name of the blob. - * @return const std::vector& The const reference of blob shape vector maintained. - */ - virtual const std::vector &GetBlobShape(const std::string &blob_name) const noexcept = 0; - - /** - * @brief Return the total number of blobs. - * - * @return size_t - */ - virtual size_t Size() const noexcept = 0; - - /** - * @brief Reset the `BlobsBuffer` which will not release the buffer memory. - * - */ - virtual void Reset() noexcept = 0; - -protected: - virtual ~IBlobsBuffer() noexcept = default; - - /** - * @brief Release the whole `BlobsBuffer` instance. - * - */ - virtual void Release() noexcept = 0; -}; - -} // namespace inference_core - -#endif \ No newline at end of file diff --git a/deploy_core/include/deploy_core/block_queue.h b/deploy_core/include/deploy_core/block_queue.h deleted file mode 100644 index c73f36f..0000000 --- a/deploy_core/include/deploy_core/block_queue.h +++ /dev/null @@ -1,294 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:00:38 - * @LastEditTime: 2024-11-26 09:29:20 - * @FilePath: /EasyDeploy/deploy_core/include/deploy_core/block_queue.h - */ -#ifndef __EASY_DEPLOY_BLOCK_QUEUE_H -#define __EASY_DEPLOY_BLOCK_QUEUE_H - -#include -#include -#include -#include - -/** - * @brief A simple implementation of block queue. - * - * @tparam T - */ -template -class BlockQueue { -public: - BlockQueue(const size_t max_size) : max_size_(max_size) - {} - - /** - * @brief Push a obj into the queue. Will block the thread if the queue is full. - * - * @param obj - * @return true - * @return false - */ - bool BlockPush(const T &obj) noexcept; - - /** - * @brief Push a obj into the queue. Will cover the oldest element if the queue is full. - * - * @param obj - * @return true - * @return false - */ - bool CoverPush(const T &obj) noexcept; - - /** - * @brief Get and pop the oldest element in the queue. Will block the thread if the queue is - * empty. - * - * @return std::optional - */ - std::optional Take() noexcept; - - /** - * @brief Get and pop the oldest element in the queue. Will return `nullopt` if the queue is - * empty. - * - * @return std::optional - */ - std::optional TryTake() noexcept; - - /** - * @brief Get the size of the queue. - * - * @return int - */ - int Size() noexcept; - - /** - * @brief Return if the queue is empty. - * - * @return true - * @return false - */ - bool Empty() noexcept; - - /** - * @brief Set the `push` process disabled. After called this method, all `push` calling will - * return `false`, which means this block queue no longer accept new elements. - * - */ - void DisablePush() noexcept; - - /** - * @brief Set the `push` process enabled. - * - */ - void EnablePush() noexcept; - - /** - * @brief Set the `take` process disabled. After called this method, all `take` calling will - * return `false`, which means this block queue no longer provides elements. - * - */ - void DisableTake() noexcept; - - /** - * @brief Set the `take` process enabled. - * - */ - void EnableTake() noexcept; - - /** - * @brief Set the `push` and `take` process disabled. - * - */ - void Disable() noexcept; - - /** - * @brief Get the max size of the block queue. - * - * @return int - */ - int GetMaxSize() const noexcept; - - /** - * @brief Set the `push` and `take` process disabled, and clear all elements in it. - * - */ - void DisableAndClear() noexcept; - - /** - * @brief Set the `push` process will no longer be called. The consumer threads which were - * blocked will be notified and quit blocking, when this method is called. - * - */ - void SetNoMoreInput() noexcept; - - ~BlockQueue() noexcept; - -private: - const size_t max_size_; - std::queue q_; - std::atomic push_enabled_{true}; - std::atomic take_enabled_{true}; - std::condition_variable producer_cv_; - std::condition_variable consumer_cv_; - std::mutex lck_; - - std::atomic no_more_input_{false}; -}; - -template -BlockQueue::~BlockQueue() noexcept -{ - Disable(); -} - -template -bool BlockQueue::BlockPush(const T &obj) noexcept -{ - std::unique_lock u_lck(lck_); - while (q_.size() >= max_size_ && push_enabled_.load()) - { - producer_cv_.wait(u_lck); - } - if (!push_enabled_.load()) - { - return false; - } - q_.push(obj); - consumer_cv_.notify_one(); - return true; -} - -template -bool BlockQueue::CoverPush(const T &obj) noexcept -{ - std::unique_lock u_lck(lck_); - if (!push_enabled_.load()) - { - return false; - } - if (q_.size() == max_size_) - { - q_.pop(); - } - q_.push(obj); - consumer_cv_.notify_one(); - return true; -} - -template -std::optional BlockQueue::Take() noexcept -{ - std::unique_lock u_lck(lck_); - // block until: 1. take disabled; 2. no more input set; 3. new elements - while (q_.size() == 0 && take_enabled_ && no_more_input_ == false) - { - consumer_cv_.wait(u_lck); - } - if (!take_enabled_ || (no_more_input_ && q_.size() == 0)) - { - return std::nullopt; - } - T ret = q_.front(); - q_.pop(); - producer_cv_.notify_one(); - - if (no_more_input_) - { - consumer_cv_.notify_all(); - } - return ret; -} - -template -std::optional BlockQueue::TryTake() noexcept -{ - std::unique_lock u_lck(lck_); - if (q_.size() == 0) - { - return std::nullopt; - } else - { - T ret = q_.front(); - q_.pop(); - producer_cv_.notify_all(); - if (no_more_input_) - { - consumer_cv_.notify_all(); - } - return ret; - } -} - -template -int BlockQueue::Size() noexcept -{ - std::unique_lock u_lck(lck_); - return q_.size(); -} - -template -bool BlockQueue::Empty() noexcept -{ - std::unique_lock u_lck(lck_); - return q_.size() == 0; -} - -template -int BlockQueue::GetMaxSize() const noexcept -{ - return max_size_; -} - -template -void BlockQueue::Disable() noexcept -{ - DisablePush(); - DisableTake(); -} - -template -void BlockQueue::DisableAndClear() noexcept -{ - Disable(); - std::unique_lock u_lck(lck_); - while (!q_.empty()) q_.pop(); -} - -template -void BlockQueue::DisablePush() noexcept -{ - push_enabled_.store(false); - producer_cv_.notify_all(); -} - -template -void BlockQueue::EnablePush() noexcept -{ - push_enabled_.store(true); -} - -template -void BlockQueue::DisableTake() noexcept -{ - take_enabled_.store(false); - consumer_cv_.notify_all(); -} - -template -void BlockQueue::EnableTake() noexcept -{ - take_enabled_.store(true); -} - -template -void BlockQueue::SetNoMoreInput() noexcept -{ - no_more_input_.store(true); - consumer_cv_.notify_all(); -} - -#endif \ No newline at end of file diff --git a/deploy_core/include/deploy_core/common_defination.h b/deploy_core/include/deploy_core/common_defination.h deleted file mode 100644 index 612cde9..0000000 --- a/deploy_core/include/deploy_core/common_defination.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:00:38 - * @LastEditTime: 2024-11-26 22:07:03 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/common_defination.h - */ -#ifndef __EASY_DEPLOY_COMMON_DEFINATION_H -#define __EASY_DEPLOY_COMMON_DEFINATION_H - -/** - * @brief Defination of common 2D bounding box - * - * @param x center of bbox `x` - * @param y center of bbox `y` - * @param w width of bbox - * @param h height of bbox - * @param conf confidence of bbox - * @param cls classification of bbox - */ -struct BBox2D { - float x; - float y; - float w; - float h; - float conf; - float cls; -}; - -/** - * @brief Enum of data loacation - * - * @param HOST data is host accessable - * @param DEVICE data is device accessable, means host cant read/write the data buffer directly - * @param UNKOWN some other condition - * - */ -enum DataLocation { HOST = 0, DEVICE = 1, UNKOWN = 2 }; - -/** - * @brief Defination of common image format. - * - */ -enum ImageDataFormat { YUV = 0, RGB = 1, BGR = 2, GRAY = 3 }; - -// some macro -#define CHECK_STATE(state, hint) \ - { \ - if (!(state)) \ - { \ - LOG(ERROR) << (hint); \ - return false; \ - } \ - } - -#define MESSURE_DURATION(run) \ - { \ - auto start = std::chrono::high_resolution_clock::now(); \ - (run); \ - auto end = std::chrono::high_resolution_clock::now(); \ - LOG(INFO) << #run << " cost(us): " \ - << std::chrono::duration_cast(end - start).count(); \ - } - -#define MESSURE_DURATION_AND_CHECK_STATE(run, hint) \ - { \ - auto start = std::chrono::high_resolution_clock::now(); \ - CHECK_STATE((run), hint); \ - auto end = std::chrono::high_resolution_clock::now(); \ - LOG(INFO) << #run << " cost(us): " \ - << std::chrono::duration_cast(end - start).count(); \ - } - -#endif \ No newline at end of file diff --git a/deploy_core/include/deploy_core/wrapper.h b/deploy_core/include/deploy_core/wrapper.h deleted file mode 100644 index 6c9b5b0..0000000 --- a/deploy_core/include/deploy_core/wrapper.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:00:38 - * @LastEditTime: 2024-11-26 21:58:32 - * @FilePath: /easy_deploy/deploy_core/include/deploy_core/wrapper.h - */ -#ifndef __EASY_DEPLOY_WRAPPER_H -#define __EASY_DEPLOY_WRAPPER_H - -#include "deploy_core/async_pipeline.h" - -#include - -#include - -/** - * @brief A simple wrapper of cv::Mat. Used in pipeline. - * - */ -class PipelineCvImageWrapper : public async_pipeline::IPipelineImageData { -public: - PipelineCvImageWrapper(const cv::Mat &cv_image, bool isRGB = false) : inner_cv_image(cv_image) - { - image_data_info.data_pointer = cv_image.data; - image_data_info.format = isRGB ? ImageDataFormat::RGB : ImageDataFormat::BGR; - image_data_info.image_height = cv_image.rows; - image_data_info.image_width = cv_image.cols; - image_data_info.image_channels = cv_image.channels(); - image_data_info.location = DataLocation::HOST; - } - - const ImageDataInfo &GetImageDataInfo() const - { - return image_data_info; - } - -private: - IPipelineImageData::ImageDataInfo image_data_info; - const cv::Mat inner_cv_image; -}; - -#endif \ No newline at end of file diff --git a/deploy_core/src/base_detection.cpp b/deploy_core/src/base_detection.cpp deleted file mode 100644 index 5e2628b..0000000 --- a/deploy_core/src/base_detection.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:24:19 - * @LastEditTime: 2024-11-26 21:58:50 - * @FilePath: /easy_deploy/deploy_core/src/base_detection.cpp - */ -#include "deploy_core/base_detection.h" - -#include "deploy_core/wrapper.h" - -namespace detection_2d { - -std::string BaseDetectionModel::detection_pipeline_name_ = "DetectionPipeline"; - -/** - * @brief construct a `DetectionPipelinePackage` - * - * @param input_image - * @param conf_thresh - * @param isRGB - * @param blob_buffers - * @return std::shared_ptr - */ -static std::shared_ptr CreateDetectionPipelineUnit( - const cv::Mat &input_image, - float conf_thresh, - bool isRGB, - std::shared_ptr blob_buffers) -{ - // 1. construct the image wrapper - auto image_wrapper = std::make_shared(input_image, isRGB); - // 2. construct `DetectionPipelinePakcage` - auto package = std::make_shared(); - package->input_image_data = image_wrapper; - package->conf_thresh = conf_thresh; - package->infer_buffer = blob_buffers; - - return package; -} - -BaseDetectionModel::BaseDetectionModel(std::shared_ptr infer_core) - : infer_core_(infer_core) -{ - // 1. check infer_core - if (infer_core == nullptr) - { - throw std::invalid_argument("[BaseDetectionModel] Input argument `infer_core` is nullptr!!!"); - } - - // 2. configure pipeline - auto preprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [=](ParsingType unit) -> bool { return PreProcess(unit); }, "BaseDet PreProcess"); - - auto infer_core_context = infer_core->GetPipelineContext(); - - auto postprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [=](ParsingType unit) -> bool { return PostProcess(unit); }, "BaseDet PostProcess"); - - BaseAsyncPipeline::ConfigPipeline(detection_pipeline_name_, - {preprocess_block, infer_core_context, postprocess_block}); -} - -bool BaseDetectionModel::Detect(const cv::Mat &input_image, - std::vector &det_results, // todo - float conf_thresh, - bool isRGB) noexcept -{ - // 1. Get blobs buffer - auto blob_buffers = infer_core_->GetBuffer(false); - if (blob_buffers == nullptr) - { - LOG(ERROR) << "[BaseDetectionModel] Inference Core run out buffer!!!"; - return false; - } - - // 2. Create a dummy pipeline package - auto package = CreateDetectionPipelineUnit(input_image, conf_thresh, isRGB, blob_buffers); - - // 3. preprocess by derived class - MESSURE_DURATION_AND_CHECK_STATE(PreProcess(package), - "[BaseDetectionModel] Preprocess execute failed!!!"); - - // 4. network inference - MESSURE_DURATION_AND_CHECK_STATE(infer_core_->SyncInfer(blob_buffers), - "[BaseDetectionModel] SyncInfer execute failed!!!"); - - // 5. postprocess by derived class - MESSURE_DURATION_AND_CHECK_STATE(PostProcess(package), - "[BaseDetectionModel] PostProcess execute failed!!!"); - - // 6. take output - det_results = std::move(package->results); - - return true; -} - -std::future> BaseDetectionModel::DetectAsync(const cv::Mat &input_image, - float conf_thresh, - bool isRGB, - bool cover_oldest) noexcept -{ - // 1. check if the pipeline is initialized - if (!IsPipelineInitialized(detection_pipeline_name_)) - { - LOG(ERROR) << "[BaseDetectionModel] Async Pipeline is not init yet!!!"; - return std::future>(); - } - - // 2. get blob buffer - auto blob_buffers = infer_core_->GetBuffer(true); - if (blob_buffers == nullptr) - { - LOG(ERROR) << "[BaseDetectionModel] Failed to get buffer from inference core!!!"; - return std::future>(); - } - - // 3. create a pipeline package - auto package = CreateDetectionPipelineUnit(input_image, conf_thresh, isRGB, blob_buffers); - - // 4. push package into pipeline and return `std::future` - return PushPipeline(detection_pipeline_name_, package); -} - -BaseDetectionModel::~BaseDetectionModel() -{ - ClosePipeline(); - infer_core_->Release(); -} - - -} // namespace detection_2d \ No newline at end of file diff --git a/deploy_core/src/base_infer_core.cpp b/deploy_core/src/base_infer_core.cpp deleted file mode 100644 index 8981805..0000000 --- a/deploy_core/src/base_infer_core.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:00 - * @LastEditTime: 2024-11-26 21:56:31 - * @FilePath: /easy_deploy/deploy_core/src/base_infer_core.cpp - */ -#include "deploy_core/base_infer_core.h" - -namespace inference_core { - -// used in sync infer -struct _InnerSyncInferPackage : public async_pipeline::IPipelinePackage { -public: - std::shared_ptr GetInferBuffer() override - { - return buffer; - } - std::shared_ptr buffer; -}; - -BaseInferCore::BaseInferCore() -{ - auto preprocess_block = BuildPipelineBlock( - [&](ParsingType unit) -> bool { return PreProcess(unit); }, "BaseInferCore PreProcess"); - auto inference_block = BuildPipelineBlock( - [&](ParsingType unit) -> bool { return Inference(unit); }, "BaseInferCore Inference"); - auto postprocess_block = BuildPipelineBlock( - [&](ParsingType unit) -> bool { return PostProcess(unit); }, "BaseInferCore PostProcess"); - ConfigPipeline("InferCore Pipieline", {preprocess_block, inference_block, postprocess_block}); -} - -bool BaseInferCore::SyncInfer(std::shared_ptr buffer, const int batch_size) -{ - auto inner_package = std::make_shared<_InnerSyncInferPackage>(); - inner_package->buffer = buffer; - CHECK_STATE(PreProcess(inner_package), "[BaseInferCore] SyncInfer Preprocess Failed!!!"); - CHECK_STATE(Inference(inner_package), "[BaseInferCore] SyncInfer Inference Failed!!!"); - CHECK_STATE(PostProcess(inner_package), "[BaseInferCore] SyncInfer PostProcess Failed!!!"); - return true; -} - -std::shared_ptr BaseInferCore::GetBuffer(bool block) -{ - return mem_buf_pool_->Alloc(block); -} - -void BaseInferCore::Release() -{ - BaseAsyncPipeline::ClosePipeline(); - mem_buf_pool_.reset(); -} - -void BaseInferCore::Init(int mem_buf_size) -{ - if (mem_buf_size <= 0 || mem_buf_size > 100) - { - throw std::invalid_argument("mem_buf_size should be between [1,100], Got: " + - std::to_string(mem_buf_size)); - } - mem_buf_pool_ = std::make_unique(this, mem_buf_size); - LOG(INFO) << "successfully init mem buf pool with pool_size : " << mem_buf_size; -} - -BaseInferCore::~BaseInferCore() -{ - Release(); -} - -} // namespace inference_core \ No newline at end of file diff --git a/deploy_core/src/base_sam.cpp b/deploy_core/src/base_sam.cpp deleted file mode 100644 index 9ff533e..0000000 --- a/deploy_core/src/base_sam.cpp +++ /dev/null @@ -1,323 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-24 20:05:41 - * @LastEditTime: 2024-11-26 21:57:33 - * @FilePath: /easy_deploy/deploy_core/src/base_sam.cpp - */ -#include "deploy_core/base_sam.h" - -#include "deploy_core/wrapper.h" - -namespace sam { - -/** - * @brief Check if the input arguments are valid - * - * @param image - * @param infer_core - * @param points - * @param labels - */ -static bool CheckValidArguments(const cv::Mat &image, - const std::shared_ptr &infer_core, - const std::vector> &points, - const std::vector &labels) noexcept -{ - if (image.empty()) - { - LOG(ERROR) << "[BaseSamModel] Got empty image!!!"; - return false; - } else if (infer_core == nullptr) - { - LOG(ERROR) << "[BaseSamModel] Infer_core with points as prompt is null!!!"; - return false; - } else if (points.size() != labels.size() || points.size() < 1) - { - LOG(ERROR) << "[BaseSamModel] points/labels size is not valid!!! " - << "points.size: " << points.size() << ", labels.size: " << labels.size(); - return false; - } - - return true; -} - -/** - * @brief Check if the input arguments are valid - * - * @param image - * @param infer_core - * @param boxes - */ -static bool CheckValidArguments(const cv::Mat &image, - const std::shared_ptr &infer_core, - const std::vector &boxes) noexcept -{ - if (image.empty()) - { - LOG(ERROR) << "[BaseSamModel] Got empty image!!!"; - return false; - } else if (infer_core == nullptr) - { - LOG(ERROR) << "[BaseSamModel] Infer_core with boxes as prompt is null!!!"; - return false; - } else if (boxes.size() < 1) - { - LOG(ERROR) << "[BaseSamModel] boxes size is not valid!!! " - << "boxes.size: " << boxes.size(); - return false; - } else if (boxes.size() > 1) - { - LOG(WARNING) << "[BaseSamModel] More than one boxes is not support in sam model!!"; - } - - return true; -} - -BaseSamModel::BaseSamModel(const std::string &model_name, - std::shared_ptr image_encoder_core, - std::shared_ptr mask_points_decoder_core, - std::shared_ptr mask_boxes_decoder_core) - : model_name_(model_name), - image_encoder_core_(image_encoder_core), - mask_points_decoder_core_(mask_points_decoder_core), - mask_boxes_decoder_core_(mask_boxes_decoder_core), - box_pipeline_name_(model_name + "_SamWithBoxPipeline"), - point_pipeline_name_(model_name + "_SamWithPointPipeline") -{ - if (image_encoder_core == nullptr) - { - throw std::invalid_argument("`image_encoder_core` should not be null"); - } - - if (mask_points_decoder_core == nullptr && mask_boxes_decoder_core == nullptr) - { - throw std::invalid_argument("one of `point/box` decoder should be non-nullptr"); - } - - if (mask_points_decoder_core_ != nullptr) - { - ConfigurePointPipeline(); - } - if (mask_boxes_decoder_core_ != nullptr) - { - ConfigureBoxPipeline(); - } -} - -BaseSamModel::~BaseSamModel() -{ - BaseAsyncPipeline::ClosePipeline(); - - if (image_encoder_core_ != nullptr) - { - image_encoder_core_->Release(); - } - if (mask_points_decoder_core_ != nullptr) - { - mask_points_decoder_core_->Release(); - } - if (mask_boxes_decoder_core_ != nullptr) - { - mask_boxes_decoder_core_->Release(); - } -} - -void BaseSamModel::ConfigureBoxPipeline() -{ - auto image_preprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [&](ParsingType unit) -> bool { return ImagePreProcess(unit); }, - "[MobileSam Image PreProcess]"); - - auto prompt_preprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [&](ParsingType unit) -> bool { return PromptBoxPreProcess(unit); }, - "[MobileSam Prompt PreProcess]"); - - auto mask_postprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [&](ParsingType unit) -> bool { return MaskPostProcess(unit); }, - "[MobileSam Mask PostProcess]"); - - const auto &image_encoder_context = image_encoder_core_->GetPipelineContext(); - - const auto &mask_decoder_context = mask_boxes_decoder_core_->GetPipelineContext(); - - BaseAsyncPipeline::ConfigPipeline( - box_pipeline_name_, {image_preprocess_block, image_encoder_context, prompt_preprocess_block, - mask_decoder_context, mask_postprocess_block}); -} - -void BaseSamModel::ConfigurePointPipeline() -{ - auto image_preprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [&](ParsingType unit) -> bool { return ImagePreProcess(unit); }, - "[MobileSam Image PreProcess]"); - - auto prompt_preprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [&](ParsingType unit) -> bool { return PromptPointPreProcess(unit); }, - "[MobileSam Prompt PreProcess]"); - - auto mask_postprocess_block = BaseAsyncPipeline::BuildPipelineBlock( - [&](ParsingType unit) -> bool { return MaskPostProcess(unit); }, - "[MobileSam Mask PostProcess]"); - - const auto &image_encoder_context = image_encoder_core_->GetPipelineContext(); - - const auto &mask_decoder_context = mask_points_decoder_core_->GetPipelineContext(); - - BaseAsyncPipeline::ConfigPipeline( - point_pipeline_name_, {image_preprocess_block, image_encoder_context, prompt_preprocess_block, - mask_decoder_context, mask_postprocess_block}); -} - -bool BaseSamModel::GenerateMask(const cv::Mat &image, - const std::vector> &points, - const std::vector &labels, - cv::Mat &result, - bool isRGB) -{ - // 0. check - CHECK_STATE(CheckValidArguments(image, mask_points_decoder_core_, points, labels), - "[BaseSamModel] `GenerateMask` with points got invalid arguments"); - - // 1. Get blobs buffers - auto encoder_blob_buffers = image_encoder_core_->GetBuffer(true); - auto decoder_blob_buffers = mask_points_decoder_core_->GetBuffer(true); - - // 2. Construct `SamPipelinePackage` - auto package = std::make_shared(); - package->input_image_data = std::make_shared(image, isRGB); - package->points = points; - package->labels = labels; - package->image_encoder_blobs_buffer = encoder_blob_buffers; - package->mask_decoder_blobs_buffer = decoder_blob_buffers; - - // 3. Carry out workflow - MESSURE_DURATION_AND_CHECK_STATE(ImagePreProcess(package), - "[BaseSamModel] Image-Preprocess execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(image_encoder_core_->SyncInfer(package->GetInferBuffer()), - "[BaseSamModel] Image-encoder sync infer execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(PromptPointPreProcess(package), - "[BaseSamModel] Prompt-preprocess execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(mask_points_decoder_core_->SyncInfer(package->GetInferBuffer()), - "[BaseSamModel] Prompt-decoder sync infer execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(MaskPostProcess(package), - "[BaseSamModel] Mask-postprocess execute failed!!!"); - - // 4. output the result - result = package->mask; - return true; -} - -bool BaseSamModel::GenerateMask(const cv::Mat &image, - const std::vector &boxes, - cv::Mat &result, - bool isRGB) -{ - // 0. check - CHECK_STATE(CheckValidArguments(image, mask_boxes_decoder_core_, boxes), - "[BaseSamModel] `GenerateMask` with boxes got invalid arguments"); - - // 1. Get blobs buffers - auto encoder_blob_buffers = image_encoder_core_->GetBuffer(true); - auto decoder_blob_buffers = mask_boxes_decoder_core_->GetBuffer(true); - - // 2. Construct `SamPipelinePackage` - auto package = std::make_shared(); - package->input_image_data = std::make_shared(image, isRGB); - package->boxes = boxes; - package->image_encoder_blobs_buffer = encoder_blob_buffers; - package->mask_decoder_blobs_buffer = decoder_blob_buffers; - - // 3. Carry out workflow - MESSURE_DURATION_AND_CHECK_STATE(ImagePreProcess(package), - "[BaseSamModel] Image-Preprocess execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(image_encoder_core_->SyncInfer(package->GetInferBuffer()), - "[BaseSamModel] Image-encoder sync infer execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(PromptBoxPreProcess(package), - "[BaseSamModel] Prompt-preprocess execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(mask_boxes_decoder_core_->SyncInfer(package->GetInferBuffer()), - "[BaseSamModel] Prompt-decoder sync infer execute failed!!!"); - - MESSURE_DURATION_AND_CHECK_STATE(MaskPostProcess(package), - "[BaseSamModel] Mask-postprocess execute failed!!!"); - - // 4. output the result - result = package->mask; - return true; -} - -std::future BaseSamModel::GenerateMaskAsync(const cv::Mat &image, - const std::vector> &points, - const std::vector &labels, - bool isRGB, - bool cover_oldest) -{ - // 0. Check - if (!CheckValidArguments(image, mask_points_decoder_core_, points, labels)) - { - LOG(ERROR) << "[BaseSamModel] `GenerateMask` with points got invalid arguments"; - return std::future(); - } - if (!BaseAsyncPipeline::IsPipelineInitialized(point_pipeline_name_)) - { - LOG(ERROR) << "[BaseSamModel] Async pipeline with points as prompt is not initialized yet!!!"; - return std::future(); - } - - // 1. Get blobs buffers - auto encoder_blob_buffers = image_encoder_core_->GetBuffer(true); - auto decoder_blob_buffers = mask_points_decoder_core_->GetBuffer(true); - - // 2. Construct `SamPipelinePackage` - auto package = std::make_shared(); - package->input_image_data = std::make_shared(image, isRGB); - package->points = points; - package->labels = labels; - package->image_encoder_blobs_buffer = encoder_blob_buffers; - package->mask_decoder_blobs_buffer = decoder_blob_buffers; - - // 3. return `std::future` instance - return BaseAsyncPipeline::PushPipeline(point_pipeline_name_, package); -} - -std::future BaseSamModel::GenerateMaskAsync(const cv::Mat &image, - const std::vector &boxes, - bool isRGB, - bool cover_oldest) -{ - // 0. check - if (!CheckValidArguments(image, mask_boxes_decoder_core_, boxes)) - { - LOG(ERROR) << "[BaseSamModel] `GenerateMask` with boxes got invalid arguments"; - return std::future(); - } - - if (!BaseAsyncPipeline::IsPipelineInitialized(box_pipeline_name_)) - { - LOG(ERROR) << "[BaseSamModel] Async pipeline with boxes as prompt is not initialized yet!!!"; - return std::future(); - } - - // 1. Get blobs buffers - auto encoder_blob_buffers = image_encoder_core_->GetBuffer(true); - auto decoder_blob_buffers = mask_boxes_decoder_core_->GetBuffer(true); - - // 2. Construct `SamPipelinePackage` - auto package = std::make_shared(); - package->input_image_data = std::make_shared(image, isRGB); - package->boxes = boxes; - package->image_encoder_blobs_buffer = encoder_blob_buffers; - package->mask_decoder_blobs_buffer = decoder_blob_buffers; - - // 3. return `std::future` instance - return BaseAsyncPipeline::PushPipeline(box_pipeline_name_, package); -} - -} // namespace sam \ No newline at end of file diff --git a/deploy_utils/CMakeLists.txt b/deploy_utils/CMakeLists.txt deleted file mode 100644 index 6a0ed77..0000000 --- a/deploy_utils/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(deploy_utils) - - -add_subdirectory(image_processing_utils) diff --git a/deploy_utils/image_processing_utils/CMakeLists.txt b/deploy_utils/image_processing_utils/CMakeLists.txt deleted file mode 100644 index 6e359b5..0000000 --- a/deploy_utils/image_processing_utils/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(image_processing_utils) - -add_compile_options(-std=c++17) -add_compile_options(-O3) -set(CMAKE_CXX_STANDARD 17) - -if(ENABLE_TENSORRT) - find_package(CUDA REQUIRED) - set(CMAKE_CUDA_COMPILER /usr/local/cuda/bin/nvcc) - enable_language(CUDA) - include_directories(${CUDA_INCLUDE_DIRS}) -endif() - -find_package(OpenCV REQUIRED) -find_package(glog REQUIRED) - -set(source_file - src/detection_2d_preprocess_cpu.cpp - src/detection_2d_postprocess_cpu.cpp -) - -if(ENABLE_TENSORRT) - list(APPEND source_file - src/detection_2d_preprocess_cuda.cpp - src/detection_2d_cuda_preprocess.cu - ) -endif() - -include_directories( - include - ${OpenCV_INCLUDE_DIRS} -) - -add_library(${PROJECT_NAME} SHARED ${source_file}) - -target_link_libraries(${PROJECT_NAME} PUBLIC - glog::glog - ${OpenCV_LIBS} - deploy_core -) - -install(TARGETS ${PROJECT_NAME} - LIBRARY DESTINATION lib) - -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file diff --git a/deploy_utils/image_processing_utils/include/detection_2d_util/detection_2d_util.h b/deploy_utils/image_processing_utils/include/detection_2d_util/detection_2d_util.h deleted file mode 100644 index 4eeac9d..0000000 --- a/deploy_utils/image_processing_utils/include/detection_2d_util/detection_2d_util.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:00 - * @LastEditTime: 2024-12-03 15:51:37 - * @FilePath: /EasyDeploy/deploy_utils/image_processing_utils/include/detection_2d_util/detection_2d_util.h - */ -#ifndef __EASY_DEPLOY_DETECTION_2D_UTIL_H -#define __EASY_DEPLOY_DETECTION_2D_UTIL_H - -#include "deploy_core/base_detection.h" -#include "deploy_core/base_infer_core.h" - -namespace detection_2d { - -/** - * @brief Create a Cpu based Det Pre Process object - * - */ -std::shared_ptr CreateCpuDetPreProcess( - const std::vector &mean = {0, 0, 0}, - const std::vector &val = {255, 255, 255}, - bool do_transpose = true, - bool do_norm = true); - -std::shared_ptr CreateCpuDetPreProcessFactory( - const std::vector &mean = {0, 0, 0}, - const std::vector &val = {255, 255, 255}, - bool do_transpose = true, - bool do_norm = true); - -/** - * @brief Create a Cuda based Det Pre Process object - * - */ -std::shared_ptr CreateCudaDetPreProcess(const int max_src_height = 1920, - const int max_src_width = 1920, - const int max_src_channels = 3); - -std::shared_ptr CreateCudaDetPreProcessFactory( - const int max_src_height = 1920, - const int max_src_width = 1920, - const int max_src_channels = 3); - -/** - * @brief Refer to `ultralytics` official project. - * - */ -std::shared_ptr CreateYolov8PostProcessCpuOrigin( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales = {8, 16, 32}); - -std::shared_ptr CreateYolov8PostProcessCpuOriginFactory( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales = {8, 16, 32}); - -/** - * @brief Refer to `ultralytics` official project. - * - * In ultralytics v8.2.2, original yolov8 output shape: `[batch, 4 + cls_number, 8400]`. - * We Modify output shape to `[batch, 8400, 4 + cls_number]` - * - */ -std::shared_ptr CreateYolov8PostProcessCpuTranspose( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales = {8, 16, 32}); - -std::shared_ptr CreateYolov8PostProcessCpuTransposeFactory( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales = {8, 16, 32}); - -/** - * @brief refer to modified `ultralytics` project by rk team。 - * @brief `https://github.com/airockchip/rknn_model_zoo/tree/main/examples/yolov8` - * @brief - * `https://github.com/airockchip/ultralytics_yolov8/blob/main/RKOPT_README.md` - * - */ -std::shared_ptr CreateYolov8PostProcessCpuDivide( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales = {8, 16, 32}); - -std::shared_ptr CreateYolov8PostProcessCpuDivideFactory( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales = {8, 16, 32}); - -} // namespace detection_2d - -#endif \ No newline at end of file diff --git a/deploy_utils/image_processing_utils/src/detection_2d_cuda_preprocess.cu b/deploy_utils/image_processing_utils/src/detection_2d_cuda_preprocess.cu deleted file mode 100755 index c5bc864..0000000 --- a/deploy_utils/image_processing_utils/src/detection_2d_cuda_preprocess.cu +++ /dev/null @@ -1,148 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:00 - * @LastEditTime: 2024-11-26 21:47:33 - * @FilePath: /easy_deploy/deploy_utils/image_processing_utils/src/detection_2d_cuda_preprocess.cu - */ -#include -#include - -#include - -struct AffineMatrix { - float value[6]; -}; - -extern "C" { - -__global__ void WarpaffineKernel(uint8_t *src, - int src_line_size, - int src_width, - int src_height, - float *dst, - int dst_width, - int dst_height, - uint8_t const_value_st, - AffineMatrix d2s, - int edge) -{ - int position = blockDim.x * blockIdx.x + threadIdx.x; - if (position >= edge) - return; - - float m_x1 = d2s.value[0]; - float m_y1 = d2s.value[1]; - float m_z1 = 0.; - float m_x2 = d2s.value[3]; - float m_y2 = d2s.value[4]; - float m_z2 = 0.; - - int dx = position % dst_width; - int dy = position / dst_width; - float src_x = m_x1 * dx + m_y1 * dy + m_z1 + 0.5f; - float src_y = m_x2 * dx + m_y2 * dy + m_z2 + 0.5f; - float c0, c1, c2; - - if (src_x <= -1 || src_x >= src_width || src_y <= -1 || src_y >= src_height) - { - // out of range - c0 = const_value_st; - c1 = const_value_st; - c2 = const_value_st; - } else - { - int y_low = floorf(src_y); - int x_low = floorf(src_x); - int y_high = y_low + 1; - int x_high = x_low + 1; - - uint8_t const_value[] = {const_value_st, const_value_st, const_value_st}; - float ly = src_y - y_low; - float lx = src_x - x_low; - float hy = 1 - ly; - float hx = 1 - lx; - float w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; - uint8_t *v1 = const_value; - uint8_t *v2 = const_value; - uint8_t *v3 = const_value; - uint8_t *v4 = const_value; - - if (y_low >= 0) - { - if (x_low >= 0) - v1 = src + y_low * src_line_size + x_low * 3; - - if (x_high < src_width) - v2 = src + y_low * src_line_size + x_high * 3; - } - - if (y_high < src_height) - { - if (x_low >= 0) - v3 = src + y_high * src_line_size + x_low * 3; - - if (x_high < src_width) - v4 = src + y_high * src_line_size + x_high * 3; - } - - c0 = w1 * v1[0] + w2 * v2[0] + w3 * v3[0] + w4 * v4[0]; - c1 = w1 * v1[1] + w2 * v2[1] + w3 * v3[1] + w4 * v4[1]; - c2 = w1 * v1[2] + w2 * v2[2] + w3 * v3[2] + w4 * v4[2]; - } - - // bgr to rgb - float t = c2; - c2 = c0; - c0 = t; - - // normalization - c0 = c0 / 255.0f; - c1 = c1 / 255.0f; - c2 = c2 / 255.0f; - - // rgbrgbrgb to rrrgggbbb - int area = dst_width * dst_height; - float *pdst_c0 = dst + dy * dst_width + dx; - float *pdst_c1 = pdst_c0 + area; - float *pdst_c2 = pdst_c1 + area; - *pdst_c0 = c0; - *pdst_c1 = c1; - *pdst_c2 = c2; -} - -float CallCudaPreprocess(const uint8_t *src, - int src_width, - int src_height, - float *dst, - int dst_width, - int dst_height, - void *device_mem_buffer) -{ - int img_size = src_width * src_height * 3; - cudaMemcpy(device_mem_buffer, src, img_size, cudaMemcpyHostToDevice); - AffineMatrix s2d, d2s; - float scale = std::min(dst_height / (float)src_height, dst_width / (float)src_width); - - s2d.value[0] = scale; - s2d.value[1] = 0; - s2d.value[2] = -scale * src_width * 0.5 + dst_width * 0.5; - s2d.value[3] = 0; - s2d.value[4] = scale; - s2d.value[5] = -scale * src_height * 0.5 + dst_height * 0.5; - cv::Mat m2x3_s2d(2, 3, CV_32F, s2d.value); - cv::Mat m2x3_d2s(2, 3, CV_32F, d2s.value); - cv::invertAffineTransform(m2x3_s2d, m2x3_d2s); - - memcpy(d2s.value, m2x3_d2s.ptr(0), sizeof(d2s.value)); - - int jobs = dst_height * dst_width; - int threads = 256; - int blocks = ceil(jobs / (float)threads); - WarpaffineKernel<<>>(static_cast(device_mem_buffer), src_width * 3, - src_width, src_height, dst, dst_width, dst_height, 0, - d2s, jobs); - - return scale; -} -} diff --git a/deploy_utils/image_processing_utils/src/detection_2d_postprocess_cpu.cpp b/deploy_utils/image_processing_utils/src/detection_2d_postprocess_cpu.cpp deleted file mode 100644 index 791b632..0000000 --- a/deploy_utils/image_processing_utils/src/detection_2d_postprocess_cpu.cpp +++ /dev/null @@ -1,493 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:00 - * @LastEditTime: 2024-12-03 15:51:39 - * @FilePath: /EasyDeploy/deploy_utils/image_processing_utils/src/detection_2d_postprocess_cpu.cpp - */ -#include "detection_2d_util/detection_2d_util.h" - -namespace detection_2d { - -static void DetectionNmsProcess(std::vector &candidates, std::vector &picked_idxes) -{ - const int candidate_objs_total = candidates.size(); - - auto func_rect_inter_area_size = [](const BBox2D &obj_a, const BBox2D &obj_b) { - cv::Rect_ inter = cv::Rect_(obj_a.x, obj_a.y, obj_a.w, obj_a.h) & - cv::Rect_(obj_b.x, obj_b.y, obj_b.w, obj_b.h); - return inter.area(); - }; - - std::sort(candidates.begin(), candidates.end(), - [](const BBox2D &a, const BBox2D &b) { return a.conf > b.conf; }); - - for (int i = 0; i < candidate_objs_total; ++i) - { - const BBox2D &obj_a = candidates[i]; - bool keep = true; - for (const int picked_idx : picked_idxes) - { - const BBox2D &obj_b = candidates[picked_idx]; - const float inter_area_size = func_rect_inter_area_size(obj_a, obj_b); - const float union_area_size = obj_a.w * obj_a.h + obj_b.w * obj_b.h - inter_area_size; - - if (inter_area_size / union_area_size > 0.6) - { - keep = false; - break; - } - } - - if (keep) - { - picked_idxes.push_back(i); - } - } -} - -/** - * @brief Original Yolov8 output blob shape : [batch, 4 + cls, 8400] - * - */ -class DetectionPostProcessCPU_Origin : public IDetectionPostProcess { -public: - DetectionPostProcessCPU_Origin(const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales); - - void Postprocess(const std::vector &output_blobs_ptr, - std::vector &results, - float conf_threshold, - float transform_scale) override; - -private: - const int cls_number_; - int bboxes_number_; -}; - -DetectionPostProcessCPU_Origin::DetectionPostProcessCPU_Origin( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) - : cls_number_(cls_number) -{ - bboxes_number_ = 0; - for (const int s : downsample_scales) - { - bboxes_number_ += (input_height / s) * (input_width / s); - } -} - -void DetectionPostProcessCPU_Origin::Postprocess(const std::vector &_output_blobs_ptr, - std::vector &results, - float conf_threshold, - float transform_scale) -{ - float *output_blob_ptr = static_cast(_output_blobs_ptr[0]); - const int bbox_info_number = cls_number_ + 4; - - std::vector candidates; - for (int i = 0; i < bboxes_number_; ++i) - { - BBox2D box; - box.conf = 0; - for (int j = 0; j < cls_number_; ++j) - { - const size_t offset = (j + 4) * bboxes_number_ + i; - const float local_conf = output_blob_ptr[offset]; - if (local_conf > box.conf) - { - box.conf = local_conf; - box.cls = j; - } - } - - if (box.conf >= conf_threshold) - { - box.x = output_blob_ptr[0 * bboxes_number_ + i] / transform_scale; - box.y = output_blob_ptr[1 * bboxes_number_ + i] / transform_scale; - box.w = output_blob_ptr[2 * bboxes_number_ + i] / transform_scale; - box.h = output_blob_ptr[3 * bboxes_number_ + i] / transform_scale; - candidates.push_back(box); - } - } - - std::vector picked_indexes; - DetectionNmsProcess(candidates, picked_indexes); - for (const int keep_index : picked_indexes) - { - const BBox2D &box = candidates[keep_index]; - results.push_back(box); - } -} - -/** - * @brief Modified Yolov8 export processing, tranpose the output blob shape to [batch, 8400, 4 + - * cls] - * - */ -class DetectionPostProcessCPU_Transpose : public IDetectionPostProcess { -public: - DetectionPostProcessCPU_Transpose(const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales); - - void Postprocess(const std::vector &output_blobs_ptr, - std::vector &results, - float conf_threshold, - float transform_scale) override; - -private: - const int cls_number_; - int bboxes_number_; -}; - -DetectionPostProcessCPU_Transpose::DetectionPostProcessCPU_Transpose( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) - : cls_number_(cls_number) -{ - bboxes_number_ = 0; - for (const int s : downsample_scales) - { - bboxes_number_ += (input_height / s) * (input_width / s); - } -} - -void DetectionPostProcessCPU_Transpose::Postprocess(const std::vector &_output_blobs_ptr, - std::vector &results, - float conf_threshold, - float transform_scale) -{ - float *output_blob_ptr = static_cast(_output_blobs_ptr[0]); - const int bbox_info_number = cls_number_ + 4; - /** - * @brief In ultralytics v8.2.2, original yolov8 output shape: [batch, 4 + cls_number, 8400] - * @brief Modify output shape to [batch, 8400, 4 + cls_number] - */ - std::vector candidates; - for (int i = 0; i < bboxes_number_; ++i) - { - BBox2D box; - const int offset = i * bbox_info_number; - - box.conf = 0; - for (int j = 0; j < cls_number_; ++j) - { - const float local_conf = output_blob_ptr[offset + 4 + j]; - if (local_conf > box.conf) - { - box.conf = local_conf; - box.cls = j; - } - } - if (box.conf >= conf_threshold) - { - box.x = output_blob_ptr[offset] / transform_scale; - box.y = output_blob_ptr[offset + 1] / transform_scale; - box.w = output_blob_ptr[offset + 2] / transform_scale; - box.h = output_blob_ptr[offset + 3] / transform_scale; - candidates.push_back(box); - } - } - std::vector picked_indexes; - DetectionNmsProcess(candidates, picked_indexes); - for (const int keep_index : picked_indexes) - { - const BBox2D &box = candidates[keep_index]; - results.push_back(box); - } -} - -/** - * @brief refer to modified `ultralytics` project by rk team。 - * @brief `https://github.com/airockchip/rknn_model_zoo/tree/main/examples/yolov8` - * @brief - * `https://github.com/airockchip/ultralytics_yolov8/blob/main/RKOPT_README.md` - * - */ -class Yolov8PostProcessCPU_Divide : public IDetectionPostProcess { -public: - Yolov8PostProcessCPU_Divide(const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales); - - void Postprocess(const std::vector &output_blobs_ptr, - std::vector &results, - float conf_threshold, - float transform_scale) override; - -private: - void GenerateCandidates(const std::vector &output_blobs_ptr, - std::vector &candidates, - float conf_threshold, - float transform_scale); - -private: - const int cls_number_; - int bboxes_number_; - const std::vector downsample_scales_; - const int input_height_; - const int input_width_; -}; - -Yolov8PostProcessCPU_Divide::Yolov8PostProcessCPU_Divide(const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) - : cls_number_(cls_number), - downsample_scales_(downsample_scales), - input_height_(input_height), - input_width_(input_width) -{ - bboxes_number_ = 0; - for (const int s : downsample_scales) - { - bboxes_number_ += (input_height / s) * (input_width / s); - } -} - -void Yolov8PostProcessCPU_Divide::GenerateCandidates(const std::vector &output_blobs_ptr, - std::vector &candidates, - float conf_threshold, - float transform_scale) -{ - const int level_count = downsample_scales_.size(); - for (int l = 0; l < level_count; ++l) - { - const int l_h = input_height_ / downsample_scales_[l]; - const int l_w = input_width_ / downsample_scales_[l]; - const int total_element_size = l_h * l_w; - const float *box_ptr = static_cast(output_blobs_ptr[l * level_count]); - const float *cls_ptr = static_cast(output_blobs_ptr[l * level_count + 1]); - const float *cls_reduce_ptr = static_cast(output_blobs_ptr[l * level_count + 2]); - - for (int r = 0; r < l_h; ++r) - { - for (int c = 0; c < l_w; ++c) - { - const size_t grid_offset = r * l_w + c; - // use cls_reduce - if (cls_reduce_ptr[grid_offset] < conf_threshold) - continue; - - // find max confidence - float max_confidence = 0; - int max_cls_index = 0; - for (int i = 0; i < cls_number_; ++i) - { - float conf = cls_ptr[grid_offset + total_element_size * i]; - if (conf > conf_threshold && conf > max_confidence) - { - max_confidence = conf; - max_cls_index = i; - } - } - - // calculate bbox - if (max_confidence > conf_threshold) - { - static const int dfl_len = 16; // TODO - static const int bbox_params_number = 64; // TODO - std::vector before_dfl(bbox_params_number); - for (int i = 0; i < bbox_params_number; ++i) - { - before_dfl[i] = box_ptr[grid_offset + total_element_size * i]; - } - // compute dfl - std::vector box(4); - for (int i = 0; i < 4; ++i) - { - std::vector exp_t(dfl_len); - float exp_sum = 0; - float acc_sum = 0; - for (int j = 0; j < dfl_len; ++j) - { - exp_t[j] = exp(before_dfl[j + i * dfl_len]); - exp_sum += exp_t[j]; - } - for (int j = 0; j < dfl_len; ++j) - { - acc_sum += exp_t[j] / exp_sum * j; - } - box[i] = acc_sum; - } - float x1, y1, x2, y2, w, h; - x1 = (-box[0] + c + 0.5) * downsample_scales_[l] / transform_scale; - y1 = (-box[1] + r + 0.5) * downsample_scales_[l] / transform_scale; - x2 = (box[2] + c + 0.5) * downsample_scales_[l] / transform_scale; - y2 = (box[3] + r + 0.5) * downsample_scales_[l] / transform_scale; - w = x2 - x1; - h = y2 - y1; - BBox2D candidate; - candidate.x = (x1 + x2) / 2; - candidate.y = (y1 + y2) / 2; - candidate.w = w; - candidate.h = h; - candidate.conf = max_confidence; - candidate.cls = max_cls_index; - candidates.push_back(candidate); - } - } - } - } -} - -void Yolov8PostProcessCPU_Divide::Postprocess(const std::vector &output_blobs_ptr, - std::vector &results, - float conf_threshold, - float transform_scale) -{ - // box, cls, cls_reduce - CHECK(output_blobs_ptr.size() == downsample_scales_.size() * 3); - // generate candidates - std::vector candidates; - GenerateCandidates(output_blobs_ptr, candidates, conf_threshold, transform_scale); - - std::vector picked_indexes; - DetectionNmsProcess(candidates, picked_indexes); - for (const int keep_index : picked_indexes) - { - const BBox2D &box = candidates[keep_index]; - results.push_back(box); - } -} - -std::shared_ptr CreateYolov8PostProcessCpuOrigin( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) -{ - return std::make_shared(input_height, input_width, cls_number, - downsample_scales); -} - -std::shared_ptr CreateYolov8PostProcessCpuTranspose( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) -{ - return std::make_shared(input_height, input_width, cls_number, - downsample_scales); -} - -std::shared_ptr CreateYolov8PostProcessCpuDivide( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) -{ - return std::make_shared(input_height, input_width, cls_number, - downsample_scales); -} - -struct Detection2DYolov8PostprocessParams { - int input_height; - int input_width; - int cls_number; - std::vector downsample_scales; -}; - -class Detection2DYolov8PostprocessOriginFactory : public BaseDetectionPostprocessFactory { -public: - Detection2DYolov8PostprocessOriginFactory(const Detection2DYolov8PostprocessParams ¶ms) - : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateYolov8PostProcessCpuOrigin(params_.input_height, params_.input_width, - params_.cls_number, params_.downsample_scales); - } - -private: - const Detection2DYolov8PostprocessParams params_; -}; - -std::shared_ptr CreateYolov8PostProcessCpuOriginFactory( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) -{ - Detection2DYolov8PostprocessParams params; - params.input_height = input_height; - params.input_width = input_width; - params.cls_number = cls_number; - params.downsample_scales = downsample_scales; - - return std::make_shared(params); -} - -class Detection2DYolov8PostprocessTransposeFactory : public BaseDetectionPostprocessFactory { -public: - Detection2DYolov8PostprocessTransposeFactory(const Detection2DYolov8PostprocessParams ¶ms) - : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateYolov8PostProcessCpuTranspose(params_.input_height, params_.input_width, - params_.cls_number, params_.downsample_scales); - } - -private: - const Detection2DYolov8PostprocessParams params_; -}; - -std::shared_ptr CreateYolov8PostProcessCpuTransposeFactory( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) -{ - Detection2DYolov8PostprocessParams params; - params.input_height = input_height; - params.input_width = input_width; - params.cls_number = cls_number; - params.downsample_scales = downsample_scales; - - return std::make_shared(params); -} - -class Detection2DYolov8PostprocessDivideFactory : public BaseDetectionPostprocessFactory { -public: - Detection2DYolov8PostprocessDivideFactory(const Detection2DYolov8PostprocessParams ¶ms) - : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateYolov8PostProcessCpuDivide(params_.input_height, params_.input_width, - params_.cls_number, params_.downsample_scales); - } - -private: - const Detection2DYolov8PostprocessParams params_; -}; - -std::shared_ptr CreateYolov8PostProcessCpuDivideFactory( - const int input_height, - const int input_width, - const int cls_number, - const std::vector &downsample_scales) -{ - Detection2DYolov8PostprocessParams params; - params.input_height = input_height; - params.input_width = input_width; - params.cls_number = cls_number; - params.downsample_scales = downsample_scales; - - return std::make_shared(params); -} - -} // namespace detection_2d \ No newline at end of file diff --git a/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cpu.cpp b/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cpu.cpp deleted file mode 100644 index 50318a2..0000000 --- a/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cpu.cpp +++ /dev/null @@ -1,253 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:00 - * @LastEditTime: 2024-12-02 20:14:25 - * @FilePath: /easy_deploy/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cpu.cpp - */ -#include "detection_2d_util/detection_2d_util.h" - -namespace detection_2d { - -class DetPreProcessCPU : public IDetectionPreProcess { -public: - DetPreProcessCPU(const std::vector &mean, - const std::vector &val, - bool do_transpose = true, - bool do_norm = true); - - float Preprocess(std::shared_ptr input_image_data, - std::shared_ptr blob_buffer, - const std::string &blob_name, - int dst_height, - int dst_width) override; - -private: - void FlipChannelsWithNorm(const cv::Mat &image, float *dst_ptr, bool flip); - void FlipChannelsWithoutNorm(const cv::Mat &image, u_char *dst_ptr, bool flip); - void TransposeAndFilpWithNorm(const cv::Mat &image, float *dst_ptr, bool flip); - void TransposeAndFilpWithoutNorm(const cv::Mat &image, u_char *dst_ptr, bool flip); - -private: - const std::vector mean_, val_; - const bool do_transpose_, do_norm_; -}; - -DetPreProcessCPU::DetPreProcessCPU(const std::vector &mean, - const std::vector &val, - bool do_transpose, - bool do_norm) - : mean_(mean), val_(val), do_transpose_(do_transpose), do_norm_(do_norm) -{} - -float DetPreProcessCPU::Preprocess( - std::shared_ptr input_image_data, - std::shared_ptr blob_buffer, - const std::string &blob_name, - int dst_height, - int dst_width) -{ - // 0. Make sure read/write on the host-side memory buffer - blob_buffer->SetBlobBuffer(blob_name, DataLocation::HOST); - auto _dst_ptr = blob_buffer->GetOuterBlobBuffer(blob_name); - - // 1. Caculate transform factor - float *dst_ptr = static_cast(_dst_ptr.first); - const auto &image_data_info = input_image_data->GetImageDataInfo(); - const int image_height = image_data_info.image_height; - const int image_width = image_data_info.image_width; - int fix_height, fix_width; - float scale; - - const float s_w = static_cast(dst_width) / image_width; - const float s_h = static_cast(dst_height) / image_height; - - if (s_h < s_w) - { - fix_height = dst_height; - scale = s_h; - fix_width = static_cast(image_width * scale); - } else - { - fix_width = dst_width; - scale = s_w; - fix_height = static_cast(image_height * scale); - } - - // 2. rebuild the cv::Mat format image - cv::Mat input_image(image_height, image_width, CV_8UC3, image_data_info.data_pointer); - - // 3. resize and padding to the left-top - cv::Mat resized_image; - cv::resize(input_image, resized_image, {fix_width, fix_height}); - cv::Mat dst_image = cv::Mat(dst_height, dst_width, CV_8UC3, cv::Scalar{0, 0, 0}); - resized_image.copyTo(dst_image(cv::Rect(0, 0, fix_width, fix_height))); - - if (!do_transpose_) - { - // 4. flip and norm - if (do_norm_) - { - FlipChannelsWithNorm(dst_image, dst_ptr, image_data_info.format == ImageDataFormat::BGR); - } else - { - FlipChannelsWithoutNorm(dst_image, reinterpret_cast(dst_ptr), - image_data_info.format == ImageDataFormat::BGR); - } - } else - { - // 5. transpose flip and norm - if (do_norm_) - { - TransposeAndFilpWithNorm(dst_image, dst_ptr, image_data_info.format == ImageDataFormat::BGR); - } else - { - TransposeAndFilpWithoutNorm(dst_image, reinterpret_cast(dst_ptr), - image_data_info.format == ImageDataFormat::BGR); - } - } - - return scale; -} - -void DetPreProcessCPU::FlipChannelsWithNorm(const cv::Mat &image, float *dst_ptr, bool flip) -{ - const int rows = image.rows, cols = image.cols; - - const int r_idx = flip ? 2 : 0; - const int b_idx = flip ? 0 : 2; - const int g_idx = 1; - - for (int r = 0; r < rows; ++r) - { - size_t offset = (r * cols) * 3; - for (int c = 0; c < cols; ++c) - { - size_t idx = offset + c * 3; - dst_ptr[idx + 0] = (image.data[idx + r_idx] - mean_[r_idx]) / val_[r_idx]; - dst_ptr[idx + 1] = (image.data[idx + g_idx] - mean_[g_idx]) / val_[g_idx]; - dst_ptr[idx + 2] = (image.data[idx + b_idx] - mean_[b_idx]) / val_[b_idx]; - } - } -} - -void DetPreProcessCPU::FlipChannelsWithoutNorm(const cv::Mat &image, u_char *dst_ptr, bool flip) -{ - const int rows = image.rows, cols = image.cols; - - const int r_idx = flip ? 2 : 0; - const int b_idx = flip ? 0 : 2; - const int g_idx = 1; - - for (int r = 0; r < rows; ++r) - { - size_t offset = r * cols * 3; - for (int c = 0; c < cols; ++c) - { - size_t idx = offset + c * 3; - dst_ptr[idx + 0] = image.data[idx + r_idx]; - dst_ptr[idx + 1] = image.data[idx + g_idx]; - dst_ptr[idx + 2] = image.data[idx + b_idx]; - } - } -} - -void DetPreProcessCPU::TransposeAndFilpWithNorm(const cv::Mat &image, float *dst_ptr, bool flip) -{ - const int rows = image.rows, cols = image.cols; - - const int r_idx = flip ? 2 : 0; - const int b_idx = flip ? 0 : 2; - const int g_idx = 1; - - const int single_channel_pixel_size = rows * cols; - for (int r = 0; r < rows; ++r) - { - uchar *pixel_ptr = image.data + r * image.step; - int offset_start = cols * r; - for (int c = 0; c < cols; ++c) - { - int offset_idx = offset_start + c; - - dst_ptr[offset_idx + 0 * single_channel_pixel_size] = - (pixel_ptr[r_idx] - mean_[r_idx]) / val_[r_idx]; - - dst_ptr[offset_idx + 1 * single_channel_pixel_size] = - (pixel_ptr[g_idx] - mean_[g_idx]) / val_[g_idx]; - - dst_ptr[offset_idx + 2 * single_channel_pixel_size] = - (pixel_ptr[b_idx] - mean_[b_idx]) / val_[b_idx]; - - pixel_ptr += 3; - } - } -} - -void DetPreProcessCPU::TransposeAndFilpWithoutNorm(const cv::Mat &image, u_char *dst_ptr, bool flip) -{ - const int rows = image.rows, cols = image.cols; - - const int r_idx = flip ? 2 : 0; - const int b_idx = flip ? 0 : 2; - const int g_idx = 1; - - const int single_channel_pixel_size = rows * cols; - for (int r = 0; r < rows; ++r) - { - uchar *pixel_ptr = image.data + r * image.step; - int offset_start = cols * r; - for (int c = 0; c < cols; ++c) - { - int offset_idx = offset_start + c; - - dst_ptr[offset_idx + 0 * single_channel_pixel_size] = pixel_ptr[r_idx]; - dst_ptr[offset_idx + 1 * single_channel_pixel_size] = pixel_ptr[g_idx]; - dst_ptr[offset_idx + 2 * single_channel_pixel_size] = pixel_ptr[b_idx]; - - pixel_ptr += 3; - } - } -} - -std::shared_ptr CreateCpuDetPreProcess(const std::vector &mean, - const std::vector &val, - bool do_transpose, - bool do_norm) -{ - return std::make_shared(mean, val, do_transpose, do_norm); -} - -struct Detection2DPreprocessCpuParams { - std::vector mean; - std::vector val; - bool do_transpose; - bool do_norm; -}; - -class Detection2DPreprocessCpuFactory : public BaseDetectionPreprocessFactory { -public: - Detection2DPreprocessCpuFactory(const Detection2DPreprocessCpuParams ¶ms) : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateCpuDetPreProcess(params_.mean, params_.val, params_.do_transpose, params_.do_norm); - } - -private: - const Detection2DPreprocessCpuParams params_; -}; - -std::shared_ptr CreateCpuDetPreProcessFactory( - const std::vector &mean, const std::vector &val, bool do_transpose, bool do_norm) -{ - Detection2DPreprocessCpuParams params; - params.mean = mean; - params.val = val; - params.do_transpose = do_transpose; - params.do_norm = do_norm; - - return std::make_shared(params); -} - -} // namespace detection_2d diff --git a/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cuda.cpp b/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cuda.cpp deleted file mode 100644 index 75ac5db..0000000 --- a/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cuda.cpp +++ /dev/null @@ -1,134 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:00 - * @LastEditTime: 2024-12-02 20:14:47 - * @FilePath: /easy_deploy/deploy_utils/image_processing_utils/src/detection_2d_preprocess_cuda.cpp - */ -#include "detection_2d_util/detection_2d_util.h" - -#include - -extern "C" float CallCudaPreprocess(const uint8_t *src, - int src_width, - int src_height, - float *dst, - int dst_width, - int dst_height, - void *unified_mem_buffer); - -namespace detection_2d { - -class DetPreProcessCUDA : public IDetectionPreProcess { -public: - DetPreProcessCUDA(const int max_src_height = 1920, - const int max_src_width = 1920, - const int max_src_channels = 3); - - float Preprocess(std::shared_ptr input_image_data, - std::shared_ptr blob_buffer, - const std::string &blob_name, - int dst_height, - int dst_width) override; - - ~DetPreProcessCUDA(); - -private: - float CudaPreprocess( - const uint8_t *src, int src_width, int src_height, float *dst, int dst_width, int dst_height); - -private: - const int max_src_width_; - const int max_src_height_; - void *device_mem_buffer_ = nullptr; -}; - -DetPreProcessCUDA::DetPreProcessCUDA(const int max_src_height, - const int max_src_width, - const int src_channels) - : max_src_height_(max_src_height), max_src_width_(max_src_width) -{ - const int max_input_byte_size = max_src_height_ * max_src_width_ * src_channels; - - auto malloc_ret = cudaMalloc(&device_mem_buffer_, max_input_byte_size); - if (malloc_ret != cudaSuccess) - { - throw std::runtime_error("[DetPreProcessCUDA] CudaMalloc failed to alloc memory on cuda!!!"); - } -} - -float DetPreProcessCUDA::Preprocess( - std::shared_ptr input_image_data, - std::shared_ptr blob_buffer, - const std::string &blob_name, - int dst_height, - int dst_width) -{ - // 1. Make sure the buffer ptr is on device side - blob_buffer->SetBlobBuffer(blob_name, DataLocation::DEVICE); - auto _dst_ptr = blob_buffer->GetOuterBlobBuffer(blob_name); - - float *dst_ptr = static_cast(_dst_ptr.first); - const auto &image_data_info = input_image_data->GetImageDataInfo(); - - // 2. Call cuda kernel function - return CudaPreprocess(image_data_info.data_pointer, image_data_info.image_width, - image_data_info.image_height, dst_ptr, dst_width, dst_height); -} - -float DetPreProcessCUDA::CudaPreprocess( - const uint8_t *src, int src_width, int src_height, float *dst, int dst_width, int dst_height) -{ - return CallCudaPreprocess(src, src_width, src_height, dst, dst_width, dst_height, - device_mem_buffer_); -} - -DetPreProcessCUDA::~DetPreProcessCUDA() -{ - if (device_mem_buffer_ != nullptr) - { - cudaFree(device_mem_buffer_); - device_mem_buffer_ = nullptr; - } -} - -std::shared_ptr CreateCudaDetPreProcess(const int max_src_height, - const int max_src_width, - const int max_src_channels) -{ - return std::make_shared(max_src_height, max_src_width, max_src_channels); -} - -struct Detection2DPreprocessCudaParams { - int max_src_height; - int max_src_width; - int max_src_channels; -}; - -class Detection2DPreprocessCudaFactory : public BaseDetectionPreprocessFactory { -public: - Detection2DPreprocessCudaFactory(const Detection2DPreprocessCudaParams ¶ms) : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateCudaDetPreProcess(params_.max_src_height, params_.max_src_width, - params_.max_src_channels); - } - -private: - const Detection2DPreprocessCudaParams params_; -}; - -std::shared_ptr CreateCudaDetPreProcessFactory( - const int max_src_height, const int max_src_width, const int max_src_channels) -{ - Detection2DPreprocessCudaParams params; - params.max_src_height = max_src_height; - params.max_src_width = max_src_width; - params.max_src_channels = max_src_channels; - - return std::make_shared(params); -} - -} // namespace detection_2d \ No newline at end of file diff --git a/detection_2d/CMakeLists.txt b/detection_2d/CMakeLists.txt index ab827dc..6e5361c 100644 --- a/detection_2d/CMakeLists.txt +++ b/detection_2d/CMakeLists.txt @@ -3,4 +3,4 @@ project(detection_2d) add_subdirectory(detection_2d_yolov8) -add_subdirectory(detection_2d_rt_detr) \ No newline at end of file +add_subdirectory(detection_2d_rt_detr) diff --git a/detection_2d/README.md b/detection_2d/README.md index 93b1736..7c958e2 100644 --- a/detection_2d/README.md +++ b/detection_2d/README.md @@ -6,4 +6,4 @@ Multiple 2D detectors have been implemented based on the `BaseDetection2DModel` - [x] Yolov8 - [x] RT-Detr -- [ ] ... \ No newline at end of file +- [ ] ... diff --git a/detection_2d/detection_2d_rt_detr/CMakeLists.txt b/detection_2d/detection_2d_rt_detr/CMakeLists.txt index 2644305..10cc562 100644 --- a/detection_2d/detection_2d_rt_detr/CMakeLists.txt +++ b/detection_2d/detection_2d_rt_detr/CMakeLists.txt @@ -19,9 +19,9 @@ include_directories( set(source_file src/rt_detr.cpp src/rt_detr_factory.cpp) -add_library(${PROJECT_NAME} SHARED ${source_file}) +add_library(${PROJECT_NAME} SHARED ${source_file}) -target_link_libraries(${PROJECT_NAME} PUBLIC +target_link_libraries(${PROJECT_NAME} PUBLIC glog::glog ${OpenCV_LIBS} deploy_core @@ -30,4 +30,8 @@ target_link_libraries(${PROJECT_NAME} PUBLIC install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION lib) -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file +target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) + +if (BUILD_TESTING) + add_subdirectory(test) +endif() diff --git a/detection_2d/detection_2d_rt_detr/include/detection_2d_rt_detr/rt_detr.h b/detection_2d/detection_2d_rt_detr/include/detection_2d_rt_detr/rt_detr.h index 058d08c..2951eef 100644 --- a/detection_2d/detection_2d_rt_detr/include/detection_2d_rt_detr/rt_detr.h +++ b/detection_2d/detection_2d_rt_detr/include/detection_2d_rt_detr/rt_detr.h @@ -33,4 +33,4 @@ std::shared_ptr CreateRTDetrDetectionModelFactory( const std::vector &output_blob_name); } // namespace detection_2d -#endif \ No newline at end of file +#endif diff --git a/detection_2d/detection_2d_rt_detr/src/rt_detr.cpp b/detection_2d/detection_2d_rt_detr/src/rt_detr.cpp index bcc4d42..5dabc22 100644 --- a/detection_2d/detection_2d_rt_detr/src/rt_detr.cpp +++ b/detection_2d/detection_2d_rt_detr/src/rt_detr.cpp @@ -5,13 +5,13 @@ namespace detection_2d { class RTDetrDetection : public BaseDetectionModel { public: RTDetrDetection(const std::shared_ptr &infer_core, - const std::shared_ptr &preprocess_block, - const int input_height, - const int input_width, - const int input_channel, - const int cls_number, - const std::vector &input_blobs_name, - const std::vector &output_blobs_name); + const std::shared_ptr &preprocess_block, + const int input_height, + const int input_width, + const int input_channel, + const int cls_number, + const std::vector &input_blobs_name, + const std::vector &output_blobs_name); ~RTDetrDetection() = default; @@ -32,15 +32,14 @@ class RTDetrDetection : public BaseDetectionModel { std::shared_ptr preprocess_block_; }; -RTDetrDetection::RTDetrDetection( - const std::shared_ptr &infer_core, - const std::shared_ptr &preprocess_block, - const int input_height, - const int input_width, - const int input_channel, - const int cls_number, - const std::vector &input_blobs_name, - const std::vector &output_blobs_name) +RTDetrDetection::RTDetrDetection(const std::shared_ptr &infer_core, + const std::shared_ptr &preprocess_block, + const int input_height, + const int input_width, + const int input_channel, + const int cls_number, + const std::vector &input_blobs_name, + const std::vector &output_blobs_name) : BaseDetectionModel(infer_core), input_blobs_name_(input_blobs_name), output_blobs_name_(output_blobs_name), @@ -52,69 +51,58 @@ RTDetrDetection::RTDetrDetection( preprocess_block_(preprocess_block) { // 创建并获取一个缓存句柄，用于校验模型与算法的一致性 - auto p_map_buffer2ptr = infer_core_->AllocBlobsBuffer(); - if (p_map_buffer2ptr->Size() != input_blobs_name_.size() + output_blobs_name_.size()) + auto blobs_tensor = infer_core_->AllocBlobsBuffer(); + if (blobs_tensor->Size() != input_blobs_name_.size() + output_blobs_name_.size()) { LOG(ERROR) << "[RTDetrDetection] Infer core should has {" << input_blobs_name_.size() + output_blobs_name_.size() << "} blobs !" - << " but got " << p_map_buffer2ptr->Size() << " blobs"; + << " but got " << blobs_tensor->Size() << " blobs"; throw std::runtime_error( "[RTDetrDetection] Construction Failed!!! Got invalid blobs_num size!!!"); } for (const std::string &input_blob_name : input_blobs_name) { - if (p_map_buffer2ptr->GetOuterBlobBuffer(input_blob_name).first == nullptr) - { - LOG(ERROR) << "[RTDetrDetection] Input_blob_name_ {" << input_blob_name - << "input blob name does not match `infer_core_` !"; - throw std::runtime_error( - "[RTDetrDetection] Construction Failed!!! Got invalid input_blob_name!!!"); - } + blobs_tensor->GetTensor(input_blob_name); } for (const std::string &output_blob_name : output_blobs_name) { - if (p_map_buffer2ptr->GetOuterBlobBuffer(output_blob_name).first == nullptr) - { - LOG(ERROR) << "[RTDetrDetection] Output_blob_name_ {" << output_blob_name - << "output blob name does not match `infer_core_` !"; - throw std::runtime_error( - "[RTDetrDetection] Construction Failed!!! Got invalid output_blob_name!!!"); - } + blobs_tensor->GetTensor(output_blob_name); } } -bool RTDetrDetection::PreProcess( - std::shared_ptr _package) +bool RTDetrDetection::PreProcess(std::shared_ptr _package) { auto package = std::dynamic_pointer_cast(_package); CHECK_STATE(package != nullptr, "[RTDetrDetection] PreProcess the `_package` instance does not belong to " "`DetectionPipelinePackage`"); - const auto &blobs_buffer = package->GetInferBuffer(); - float scale = preprocess_block_->Preprocess(package->input_image_data, blobs_buffer, - input_blobs_name_[0], input_height_, input_width_); + const auto &blobs_tensor = package->GetInferBuffer(); + + float scale = preprocess_block_->Preprocess(package->input_image_data, + blobs_tensor->GetTensor(input_blobs_name_[0]), + input_height_, input_width_); + package->transform_scale = scale; return true; } -bool RTDetrDetection::PostProcess( - std::shared_ptr _package) +bool RTDetrDetection::PostProcess(std::shared_ptr _package) { auto package = std::dynamic_pointer_cast(_package); CHECK_STATE(package != nullptr, "[RTDetrDetection] PostProcess the `_package` instance does not belong to " "`DetectionPipelinePackage`"); - const auto &blobs_buffer = package->GetInferBuffer(); + const auto &blobs_tensor = package->GetInferBuffer(); // RTDetrDetection outputs: labels (1, 300); boxes (1, 300, 4); scores (1, 300) - float *labels_ptr = reinterpret_cast(blobs_buffer->GetOuterBlobBuffer("labels").first); - float *boxes_ptr = reinterpret_cast(blobs_buffer->GetOuterBlobBuffer("boxes").first); - float *scores_ptr = reinterpret_cast(blobs_buffer->GetOuterBlobBuffer("scores").first); + float *labels_ptr = blobs_tensor->GetTensor(output_blobs_name_[0])->Cast(); + float *boxes_ptr = blobs_tensor->GetTensor(output_blobs_name_[1])->Cast(); + float *scores_ptr = blobs_tensor->GetTensor(output_blobs_name_[2])->Cast(); const int CANDIDATES_NUM = 300; const float conf_thresh = package->conf_thresh; @@ -156,9 +144,9 @@ std::shared_ptr CreateRTDetrDetectionModel( const std::vector &input_blobs_name, const std::vector &output_blobs_name) { - return std::make_shared(infer_core, preprocess_block, input_height, - input_width, input_channel, cls_number, - input_blobs_name, output_blobs_name); + return std::make_shared(infer_core, preprocess_block, input_height, input_width, + input_channel, cls_number, input_blobs_name, + output_blobs_name); } -} // namespace detection_2d \ No newline at end of file +} // namespace detection_2d diff --git a/detection_2d/detection_2d_rt_detr/src/rt_detr_factory.cpp b/detection_2d/detection_2d_rt_detr/src/rt_detr_factory.cpp index 5c53935..b8d03d6 100644 --- a/detection_2d/detection_2d_rt_detr/src/rt_detr_factory.cpp +++ b/detection_2d/detection_2d_rt_detr/src/rt_detr_factory.cpp @@ -64,4 +64,4 @@ std::shared_ptr CreateRTDetrDetectionModelFactory( return std::make_shared(params); } -} // namespace detection_2d \ No newline at end of file +} // namespace detection_2d diff --git a/simple_tests/CMakeLists.txt b/detection_2d/detection_2d_rt_detr/test/CMakeLists.txt similarity index 50% rename from simple_tests/CMakeLists.txt rename to detection_2d/detection_2d_rt_detr/test/CMakeLists.txt index 977f051..0e23d52 100644 --- a/simple_tests/CMakeLists.txt +++ b/detection_2d/detection_2d_rt_detr/test/CMakeLists.txt @@ -1,21 +1,7 @@ -cmake_minimum_required(VERSION 3.8) -project(simple_tests) - add_compile_options(-std=c++17) add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) set(CMAKE_CXX_STANDARD 17) -set(easy_deploy_packages - deploy_core - image_processing_utils -) - -set(algorithm_packages - detection_2d_yolov8 - detection_2d_rt_detr - sam_mobilesam -) - if(ENABLE_TENSORRT) list(APPEND platform_core_packages trt_core) endif() @@ -28,48 +14,42 @@ if(ENABLE_ORT) list(APPEND platform_core_packages ort_core) endif() - - find_package(GTest REQUIRED) find_package(glog REQUIRED) find_package(OpenCV REQUIRED) -set(source_file - # src/gtest_main.cpp - src/fs_util.cpp - src/test_func.cpp +set(source_file + test_detection_2d_rt_detr.cpp ) -if(ENABLE_TENSORRT) - list(APPEND source_file src/test_jetson_devkit.cpp) -endif() - -if(ENABLE_RKNN) - list(APPEND source_file src/test_rk_devkit.cpp) -endif() - -if(ENABLE_ORT) - list(APPEND source_file src/test_cpu_devkit.cpp) -endif() - -enable_testing() - include_directories( include ${OpenCV_INCLUDE_DIRS} ) -add_executable(${PROJECT_NAME} ${source_file}) +add_executable(test_detection_2d_rt_detr ${source_file}) -target_link_libraries(${PROJECT_NAME} PUBLIC +target_link_libraries(test_detection_2d_rt_detr PUBLIC GTest::gtest_main glog::glog ${OpenCV_LIBS} - ${algorithm_packages} - ${easy_deploy_packages} + deploy_core + image_processing_utils + detection_2d_rt_detr + test_utils ${platform_core_packages} ) +if(ENABLE_TENSORRT) + target_compile_definitions(test_detection_2d_rt_detr PRIVATE ENABLE_TENSORRT) +endif() +if(ENABLE_RKNN) + target_compile_definitions(test_detection_2d_rt_detr PRIVATE ENABLE_RKNN) +endif() + +if(ENABLE_ORT) + target_compile_definitions(test_detection_2d_rt_detr PRIVATE ENABLE_ORT) +endif() -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file +gtest_discover_tests(test_detection_2d_rt_detr) diff --git a/detection_2d/detection_2d_rt_detr/test/test_detection_2d_rt_detr.cpp b/detection_2d/detection_2d_rt_detr/test/test_detection_2d_rt_detr.cpp new file mode 100644 index 0000000..d9faa0c --- /dev/null +++ b/detection_2d/detection_2d_rt_detr/test/test_detection_2d_rt_detr.cpp @@ -0,0 +1,100 @@ +#include + +#include "detection_2d_util/detection_2d_util.h" +#include "detection_2d_rt_detr/rt_detr.h" +#include "test_utils/detection_2d_test_utils.hpp" + +using namespace inference_core; +using namespace detection_2d; +using namespace test_utils; + +#define GEN_TEST_CASES(Tag, FixtureClass) \ + TEST_F(FixtureClass, test_rt_detr_##Tag##_correctness) \ + { \ + test_detection_2d_algorithm_correctness(rt_detr_model_, test_image_path_, conf_threshold_, \ + expected_obj_num_, test_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_rt_detr_##Tag##_async_correctness) \ + { \ + test_detection_2d_algorithm_async_correctness(rt_detr_model_, test_image_path_, \ + conf_threshold_, expected_obj_num_, \ + test_visual_result_save_path_); \ + } + +class BaseRTDetrFixture : public testing::Test { +protected: + std::shared_ptr rt_detr_model_; + + std::string test_image_path_; + std::string test_visual_result_save_path_; + float conf_threshold_; + size_t expected_obj_num_; +}; + +#ifdef ENABLE_TENSORRT + +#include "trt_core/trt_core.h" + +class RTDetr_TensorRT_Fixture : public BaseRTDetrFixture { +public: + void SetUp() override + { + std::string model_path = "/workspace/models/rt_detr_v2_single_input.engine"; + const int input_height = 640; + const int input_width = 640; + const int input_channels = 3; + const int cls_number = 80; + const std::vector input_blobs_name = {"images"}; + const std::vector output_blobs_name = {"labels", "boxes", "scores"}; + + auto infer_core = CreateTrtInferCore(model_path); + auto preprocess = CreateCudaDetPreProcess(); + + rt_detr_model_ = + CreateRTDetrDetectionModel(infer_core, preprocess, input_height, input_width, + input_channels, cls_number, input_blobs_name, output_blobs_name); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_visual_result_save_path_ = "/workspace/test_data/rt_detr_tensorrt_test_result.jpg"; + conf_threshold_ = 0.4; + expected_obj_num_ = 22ul; + } +}; + +GEN_TEST_CASES(tensorrt, RTDetr_TensorRT_Fixture); + +#endif + +#ifdef ENABLE_ORT + +#include "ort_core/ort_core.h" + +class RTDetr_OnnxRuntime_Fixture : public BaseRTDetrFixture { +public: + void SetUp() override + { + std::string model_path = "/workspace/models/rt_detr_v2_single_input.onnx"; + const int input_height = 640; + const int input_width = 640; + const int input_channels = 3; + const int cls_number = 80; + const std::vector input_blobs_name = {"images"}; + const std::vector output_blobs_name = {"labels", "boxes", "scores"}; + + auto infer_core = CreateOrtInferCore(model_path); + auto preprocess = CreateCpuDetPreProcess({0, 0, 0}, {255, 255, 255}, true, true); + + rt_detr_model_ = + CreateRTDetrDetectionModel(infer_core, preprocess, input_height, input_width, + input_channels, cls_number, input_blobs_name, output_blobs_name); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_visual_result_save_path_ = "/workspace/test_data/rt_detr_tensorrt_test_result.jpg"; + conf_threshold_ = 0.4; + expected_obj_num_ = 23ul; + } +}; + +GEN_TEST_CASES(onnxruntime, RTDetr_OnnxRuntime_Fixture); + +#endif diff --git a/detection_2d/detection_2d_yolov8/CMakeLists.txt b/detection_2d/detection_2d_yolov8/CMakeLists.txt index 1e5fb96..512ec51 100644 --- a/detection_2d/detection_2d_yolov8/CMakeLists.txt +++ b/detection_2d/detection_2d_yolov8/CMakeLists.txt @@ -5,12 +5,9 @@ add_compile_options(-std=c++17) add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) set(CMAKE_CXX_STANDARD 17) - find_package(OpenCV REQUIRED) find_package(glog REQUIRED) - - include_directories( include ${OpenCV_INCLUDE_DIRS} @@ -19,9 +16,9 @@ include_directories( set(source_file src/yolov8.cpp src/yolov8_factory.cpp) -add_library(${PROJECT_NAME} SHARED ${source_file}) +add_library(${PROJECT_NAME} SHARED ${source_file}) -target_link_libraries(${PROJECT_NAME} PUBLIC +target_link_libraries(${PROJECT_NAME} PUBLIC glog::glog ${OpenCV_LIBS} deploy_core @@ -30,4 +27,8 @@ target_link_libraries(${PROJECT_NAME} PUBLIC install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION lib) -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file +target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) + +if (BUILD_TESTING) + add_subdirectory(test) +endif() diff --git a/detection_2d/detection_2d_yolov8/include/detection_2d_yolov8/yolov8.h b/detection_2d/detection_2d_yolov8/include/detection_2d_yolov8/yolov8.h index 97b5cc7..65e2a37 100644 --- a/detection_2d/detection_2d_yolov8/include/detection_2d_yolov8/yolov8.h +++ b/detection_2d/detection_2d_yolov8/include/detection_2d_yolov8/yolov8.h @@ -53,4 +53,4 @@ std::shared_ptr CreateYolov8DetectionModelFactory( const std::vector &downsample_scales = {8, 16, 32}); } // namespace detection_2d -#endif \ No newline at end of file +#endif diff --git a/detection_2d/detection_2d_yolov8/src/yolov8.cpp b/detection_2d/detection_2d_yolov8/src/yolov8.cpp index 6143a2c..7d96bfd 100644 --- a/detection_2d/detection_2d_yolov8/src/yolov8.cpp +++ b/detection_2d/detection_2d_yolov8/src/yolov8.cpp @@ -66,34 +66,24 @@ Yolov8Detection::Yolov8Detection(const std::shared_ptrAllocBlobsBuffer(); - if (p_map_buffer2ptr->Size() != input_blobs_name_.size() + output_blobs_name_.size()) + // Check if the input arguments and inference_core matches + auto blobs_tensor = infer_core_->AllocBlobsBuffer(); + if (blobs_tensor->Size() != input_blobs_name_.size() + output_blobs_name_.size()) { LOG(ERROR) << "[Yolov8Detection] Infer core should has {" << input_blobs_name_.size() + output_blobs_name_.size() << "} blobs !" - << " but got " << p_map_buffer2ptr->Size() << " blobs"; + << " but got " << blobs_tensor->Size() << " blobs"; throw std::runtime_error("[Yolov8Detection] Got invalid input arguments!!"); } for (const std::string &input_blob_name : input_blobs_name) { - if (p_map_buffer2ptr->GetOuterBlobBuffer(input_blob_name).first == nullptr) - { - LOG(ERROR) << "[Yolov8Detection] Input_blob_name_ {" << input_blob_name - << "input blob name does not match `infer_core_` !"; - throw std::runtime_error("[Yolov8Detection] Got invalid input arguments!!"); - } + blobs_tensor->GetTensor(input_blob_name); } for (const std::string &output_blob_name : output_blobs_name) { - if (p_map_buffer2ptr->GetOuterBlobBuffer(output_blob_name).first == nullptr) - { - LOG(ERROR) << "[Yolov8Detection] Output_blob_name_ {" << output_blob_name - << "} does not match name in infer_core_ !"; - throw std::runtime_error("[Yolov8Detection] Got invalid input arguments!!"); - } + blobs_tensor->GetTensor(output_blob_name); } for (const int s : downsample_scales_) @@ -118,9 +108,12 @@ bool Yolov8Detection::PreProcess(std::shared_ptrGetInferBuffer(); - float scale = preprocess_block_->Preprocess(package->input_image_data, p_blob_buffers, - input_blobs_name_[0], input_height_, input_width_); + const auto &blobs_tensor = package->GetInferBuffer(); + + float scale = preprocess_block_->Preprocess(package->input_image_data, + blobs_tensor->GetTensor(input_blobs_name_[0]), + input_height_, input_width_); + package->transform_scale = scale; return true; } @@ -136,7 +129,7 @@ bool Yolov8Detection::PostProcess(std::shared_ptr output_blobs_ptr; for (const std::string &output_blob_name : output_blobs_name_) { - void *blob_ptr = p_blob_buffers->GetOuterBlobBuffer(output_blob_name).first; + void *blob_ptr = p_blob_buffers->GetTensor(output_blob_name)->RawPtr(); output_blobs_ptr.push_back(blob_ptr); } @@ -163,4 +156,4 @@ std::shared_ptr CreateYolov8DetectionModel( input_blobs_name, output_blobs_name, downsample_scales); } -} // namespace detection_2d \ No newline at end of file +} // namespace detection_2d diff --git a/detection_2d/detection_2d_yolov8/src/yolov8_factory.cpp b/detection_2d/detection_2d_yolov8/src/yolov8_factory.cpp index 2401e82..903197f 100644 --- a/detection_2d/detection_2d_yolov8/src/yolov8_factory.cpp +++ b/detection_2d/detection_2d_yolov8/src/yolov8_factory.cpp @@ -72,4 +72,4 @@ std::shared_ptr CreateYolov8DetectionModelFactory( return std::make_shared(params); } -} // namespace detection_2d \ No newline at end of file +} // namespace detection_2d diff --git a/detection_2d/detection_2d_yolov8/test/CMakeLists.txt b/detection_2d/detection_2d_yolov8/test/CMakeLists.txt new file mode 100644 index 0000000..e919a8a --- /dev/null +++ b/detection_2d/detection_2d_yolov8/test/CMakeLists.txt @@ -0,0 +1,55 @@ +add_compile_options(-std=c++17) +add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) +set(CMAKE_CXX_STANDARD 17) + +if(ENABLE_TENSORRT) + list(APPEND platform_core_packages trt_core) +endif() + +if(ENABLE_RKNN) + list(APPEND platform_core_packages rknn_core) +endif() + +if(ENABLE_ORT) + list(APPEND platform_core_packages ort_core) +endif() + +find_package(GTest REQUIRED) +find_package(glog REQUIRED) +find_package(OpenCV REQUIRED) + +set(source_file + test_detection_2d_yolov8.cpp +) + +include_directories( + include + ${OpenCV_INCLUDE_DIRS} +) + +add_executable(test_detection_2d_yolov8 ${source_file}) + +target_link_libraries(test_detection_2d_yolov8 PUBLIC + GTest::gtest_main + glog::glog + ${OpenCV_LIBS} + deploy_core + image_processing_utils + detection_2d_yolov8 + test_utils + ${platform_core_packages} +) + +if(ENABLE_TENSORRT) + target_compile_definitions(test_detection_2d_yolov8 PRIVATE ENABLE_TENSORRT) +endif() + +if(ENABLE_RKNN) + target_compile_definitions(test_detection_2d_yolov8 PRIVATE ENABLE_RKNN) +endif() + +if(ENABLE_ORT) + target_compile_definitions(test_detection_2d_yolov8 PRIVATE ENABLE_ORT) +endif() + +gtest_discover_tests(test_detection_2d_yolov8) diff --git a/detection_2d/detection_2d_yolov8/test/test_detection_2d_yolov8.cpp b/detection_2d/detection_2d_yolov8/test/test_detection_2d_yolov8.cpp new file mode 100644 index 0000000..f0f9501 --- /dev/null +++ b/detection_2d/detection_2d_yolov8/test/test_detection_2d_yolov8.cpp @@ -0,0 +1,139 @@ +#include + +#include "detection_2d_util/detection_2d_util.h" +#include "detection_2d_yolov8/yolov8.h" +#include "test_utils/detection_2d_test_utils.hpp" + +using namespace inference_core; +using namespace detection_2d; +using namespace test_utils; + +#define GEN_TEST_CASES(Tag, FixtureClass) \ + TEST_F(FixtureClass, test_yolov8_##Tag##_correctness) \ + { \ + test_detection_2d_algorithm_correctness(yolov8_model_, test_image_path_, conf_threshold_, \ + expected_obj_num_, test_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_yolov8_##Tag##_async_correctness) \ + { \ + test_detection_2d_algorithm_async_correctness(yolov8_model_, test_image_path_, \ + conf_threshold_, expected_obj_num_, \ + test_visual_result_save_path_); \ + } + +class BaseYolov8Fixture : public testing::Test { +protected: + std::shared_ptr yolov8_model_; + + std::string test_image_path_; + std::string test_visual_result_save_path_; + float conf_threshold_; + size_t expected_obj_num_; +}; + +#ifdef ENABLE_TENSORRT + +#include "trt_core/trt_core.h" + +class Yolov8_TensorRT_Fixture : public BaseYolov8Fixture { +public: + void SetUp() override + { + std::string model_path = "/workspace/models/yolov8n.engine"; + const int input_height = 640; + const int input_width = 640; + const int input_channels = 3; + const int cls_number = 80; + const std::vector input_blobs_name = {"images"}; + const std::vector output_blobs_name = {"output0"}; + + auto infer_core = CreateTrtInferCore(model_path); + auto preprocess = CreateCudaDetPreProcess(); + auto postprocess = CreateYolov8PostProcessCpuOrigin(input_height, input_width, cls_number); + + yolov8_model_ = + CreateYolov8DetectionModel(infer_core, preprocess, postprocess, input_height, input_width, + input_channels, cls_number, input_blobs_name, output_blobs_name); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_visual_result_save_path_ = "/workspace/test_data/yolov8_tensorrt_test_result.jpg"; + conf_threshold_ = 0.4; + expected_obj_num_ = 10ul; + } +}; + +GEN_TEST_CASES(tensorrt, Yolov8_TensorRT_Fixture); + +#endif + +#ifdef ENABLE_ORT + +#include "ort_core/ort_core.h" + +class Yolov8_OnnxRuntime_Fixture : public BaseYolov8Fixture { +public: + void SetUp() override + { + std::string model_path = "/workspace/models/yolov8n.onnx"; + const int input_height = 640; + const int input_width = 640; + const int input_channels = 3; + const int cls_number = 80; + const std::vector input_blobs_name = {"images"}; + const std::vector output_blobs_name = {"output0"}; + + auto infer_core = CreateOrtInferCore(model_path); + auto preprocess = CreateCpuDetPreProcess({0, 0, 0}, {255, 255, 255}, true, true); + auto postprocess = CreateYolov8PostProcessCpuOrigin(input_height, input_width, cls_number); + + yolov8_model_ = + CreateYolov8DetectionModel(infer_core, preprocess, postprocess, input_height, input_width, + input_channels, cls_number, input_blobs_name, output_blobs_name); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_visual_result_save_path_ = "/workspace/test_data/yolov8_tensorrt_test_result.jpg"; + conf_threshold_ = 0.4; + expected_obj_num_ = 11ul; + } +}; + +GEN_TEST_CASES(onnxruntime, Yolov8_OnnxRuntime_Fixture); + +#endif + +#ifdef ENABLE_RKNN + +#include "rknn_core/rknn_core.h" + +class Yolov8_Rknn_Fixture : public BaseYolov8Fixture { +public: + void SetUp() override + { + std::string model_path = "/workspace/models/yolov8n_divide_opset11.rknn"; + const int input_height = 640; + const int input_width = 640; + const int input_channels = 3; + const int cls_number = 80; + const std::vector input_blobs_name = {"images"}; + const std::vector output_blobs_name = {"318", "onnx::ReduceSum_326", "331", + "338", "onnx::ReduceSum_346", "350", + "357", "onnx::ReduceSum_365", "369"}; + + auto infer_core = CreateRknnInferCore(model_path, {{"images", RknnInputTensorType::RK_UINT8}}); + auto preprocess = CreateCpuDetPreProcess({0, 0, 0}, {1, 1, 1}, false, false); + auto postprocess = CreateYolov8PostProcessCpuDivide(input_height, input_width, cls_number); + + yolov8_model_ = + CreateYolov8DetectionModel(infer_core, preprocess, postprocess, input_height, input_width, + input_channels, cls_number, input_blobs_name, output_blobs_name); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_visual_result_save_path_ = "/workspace/test_data/yolov8_tensorrt_test_result.jpg"; + conf_threshold_ = 0.4; + expected_obj_num_ = 10ul; + } +}; + +GEN_TEST_CASES(rknn, Yolov8_Rknn_Fixture); + +#endif diff --git a/doc/EnviromentSetup.md b/doc/EnviromentSetup.md deleted file mode 100644 index d84c675..0000000 --- a/doc/EnviromentSetup.md +++ /dev/null @@ -1,32 +0,0 @@ -# EasyDeploy Enviroment Setup - -We use docker to setup enviroment. - -## Scripts - -- Just run docker container building script. - ```bash - cd docker - bash build_docker.sh --platform=nvidia_gpu # rk3588/jetson - ``` - -- Run script to get into container. - ```bash - bash docker/into_docker.sh - ``` - -## Notes - -- `docker` is needed on your machine. You could use scripts to easily install docker. - ```bash - wget http://fishros.com/install -O fishros && . fishros - ``` - -- On `jetson` platform, `docker` is pre-installed with jetpack. There is no need to install docker again. - -- On `nvidia_gpu` platform, `nvidia-container-runtime` is needed by docker to use gpu and cuda. Please make sure it is installed and configured with docker daemon. - ```bash - cat /etc/docker/daemon.json - # `nvidia` should be in runtimes list - ``` -- Downloading packages from `github` during docker image building may break up. Use `github` proxy from repo [gh-proxy](https://github.com/hunshcn/gh-proxy) to speed it up. diff --git a/doc/HowToDeployModels.md b/doc/HowToDeployModels.md deleted file mode 100644 index a6885c4..0000000 --- a/doc/HowToDeployModels.md +++ /dev/null @@ -1,37 +0,0 @@ -# How to Deploy Models with `EasyDeploy` - -We designed `BaseInferCore` and `IBlobsBuffer` to unify inference across frameworks, handling memory transport and enabling easy read/write without framework or hardware concerns. - -This allows running multiple inference cores (e.g., `onnxruntime` and `rknn` on `RK3588`) to maximize throughput. The async pipeline further boosts performance. - -Key details and virtual functions are in `base_infer_core.h`; developers only implement data processing and `blob_buffer` allocation. - -## `inference_core::BaseInferCore` - -When using BaseInferCore, algorithm developers only need to: -1. Implement the key abstract functions defined by BaseInferCore. -2. Use the IBlobsBuffer interface for managing inference buffers, without worrying about the underlying buffer logic. -3. Focus on the deployment of the model inference process while leaving the specifics of the inference framework and algorithm type abstracted away. - -- Example: - ```cpp - auto infer_core = std::make_shared(...); - auto blob_buffer = infer_core->GetBuffer(true); // get blob buffer from pool - auto input_buffer_and_loc = blob_buffer->GetOuterBlobBuffer("images"); - // input_buffer_and_loc contains buffer ptr and location - // the location is on host-side by default - void* input_buffer_ptr = input_buffer_and_loc.first; - memcpy(input_buffer_ptr, src, len); - // do inference - infer_core->SyncInfer(blob_buffer); - // postprocess - auto output_buffer_and_loc = blob_buffer->GetOuterBlobBuffer("output"); - void* output_buffer_ptr = output_buffer_and_loc.first; - ... - ``` - -- Through the `IBlobsBuffer` interface, **dynamic shape** of a specific blob can be configured, or an existing buffer can be reused, enabling zero-copy in sequential model inference workflows. - -- Some inference frameworks or hardware platforms are heterogeneous, requiring data transfers between the host and device (e.g., platforms related to the TensorRT framework). These tasks are handled in the concrete implementation classes of `BaseInferCore`. During usage, users can select host or device-side buffers via the `SetBlobBuffer` method. The buffers retrieved through the `GetOuterBlobBuffer` method will adapt accordingly. - -- The overloaded `SetBlobBuffer` method allows users to specify external buffers, enabling zero-copy functionality. \ No newline at end of file diff --git a/doc/QuickStart.md b/doc/QuickStart.md deleted file mode 100644 index 75f174f..0000000 --- a/doc/QuickStart.md +++ /dev/null @@ -1,221 +0,0 @@ -# EasyDeploy Quick Start Demo - -This documentation will show how to use `EasyDeploy` with the `yolov8`, `rt-detr`, `mobile-sam`, `nano-sam` algorithms on multiple inference frameworks. - -Before this, you should follow [Setup](./EnviromentSetup.md) - -## 0. QuickStart - -### 0.1 Download all onnx model - - All `onnx` models could be downloaded from [goolge driver](https://drive.google.com/drive/folders/1yVEOzo59aob_1uXwv343oeh0dTKuHT58?usp=drive_link). - - Put all onnx models into `EasyDeploy/models` directory. - -### 0.2 Convert models - - Use `tools/cvt_onnx2*_all.sh` scripts to convert onnx models. - ```bash - bash tools/cvt_onnx2rknn_all.sh - # bash tools/cvt_onnx2trt_all.sh - ``` -### 0.3 Run demo - - Build and run simple_tests demo. See test cases for detail. - ```bash - cd /workspace - mkdir build && cd build - cmake .. -DBUILD_TESTING=ON -DENABLE_TENSORRT=ON - make -j - - # test on yolo correctness - ./bin/simple_tests --gtest_filter=*yolo*correctness - # test on yolo speed - GLOG_minloglevel=1 ./bin/simple_tests --gtest_filter=*yolo*speed - # for other tests, please see `EasyDeploy/simple_tests/src/` - ``` - - - -## 1. Yolov8 - -### 1.1 Yolov8-with-TensorRT - -#### 1.1.1 Export yolov8 onnx model from `ultralytics` project. - - - Install `ultralytics` by pip - ```bash - pip install ultralytics - ``` - - - Export Yolov8n model to onnx, and copy it to `path/to/EasyDeploy/models`. - ```bash - # download the official model online - yolo export model=yolov8n.pt format=onnx - # use the local pre-downloaded model or your custom model - yolo export model=./yolov8n.pt format=onnx - ``` - -#### 1.1.2 Build TensorRT engine with the yolov8 onnx model. - - - Go into your pre-built tensorrt docker container. - ```bash - cd path/to/EasyDeploy/ - bash docker/into_docker.sh - ``` - - - Use `trtexec` official tool to convert onnx model to tensorrt engine. - ```bash - /usr/src/tensorrt/bin/trtexec --onnx=/workspace/models/yolov8n.onnx \ - --saveEngine=/workspace/models/yolov8n.engine \ - --fp16 - ``` - -#### 1.1.3 Run test demo - - Now we have a `yolov8n.engine` file under `/workspace/models/`. Run simple tests on yolov8. - ```bash - cd /workspace - mkdir build && cd build - cmake .. -DBUILD_TESTING=ON -DENABLE_TENSORRT=ON - make -j - - # test on yolo correctness - ./bin/simple_tests --gtest_filter=*yolo*correctness - # test on yolo speed - GLOG_minloglevel=1 ./bin/simple_tests --gtest_filter=*yolo*speed - ``` - - -### 1.2 Yolov8-with-RKNN - - - - - - -## 2. RT-Detr - -### 2.1 RT-Detr-with-TensorRT - -#### 2.1.1 Export RT-Detr-V2 onnx model - - Reference to official [RT-Detr-v2](https://github.com/lyuwenyu/RT-DETR/tree/main/rtdetrv2_pytorch). The original model exported by `tools/export_onnx.py` supports dynamic input shape. However, we simply want a static input shape model for inference. So, the script we used to export the onnx model with single static input image blob is `EasyDeploy/tools/rt_detr_v2_export_onnx.py`. Replace the official export script with our script. - - Download the official rt-detr code from [rt-detr](https://github.com/lyuwenyu/RT-DETR/tree/main). - ```bash - git clone git@github.com:lyuwenyu/RT-DETR.git - ``` - - - Setup enviroment and download models - ```bash - cd rtdetrv2_pytorch/ - pip install -r requirements.txt - - mkdir weights && cd weights - wget https://github.com/lyuwenyu/storage/releases/download/v0.2/rtdetrv2_r18vd_120e_coco_rerun_48.1.pth - ``` - - - Export rt-detr-v2 model with single static input image blob. - ```bash - cp EasyDeploy/tools/rt_detr_v2_export_onnx.py RT-DETR/rtdetrv2/tools/ - cd RT-DETR/rtdetrv2 - - python3 tools/rt_detr_v2_export_onnx.py \ - -c ./configs/rtdetrv2/rtdetrv2_r18vd_120e_coco.yml \ - -r weights/rtdetrv2_r18vd_120e_coco_rerun_48.1.pth \ - -o rt_detr_v2_single_input.onnx --check --simplify - ``` - - - Visualize the output onnx model with [Netron](https://netron.app/). The image input blob should be static shape and the image original shape blob should be removed. - -#### 2.1.2 Build TensorRT engine with rt-detr-v2 onnx model - - Now we get the rt-detr-v2 onnx model. Convert it to tensorrt engine with `trtexec` tool. Note that `fp16` optimization will greatly affect the model's accuracy. So, we do not convert the onnx model with `--fp16` flag. - ```bash - /usr/src/tensorrt/bin/trtexec --onnx=/workspace/models/rt_detr_v2_single_input.onnx \ - --saveEngine=/workspace/models/rt_detr_v2_single_input.engine - ``` - -#### 2.1.3 Run test demo - - Then we get the rt-detr-v2 tensorrt engine model. Run simple tests. - ```bash - cd /workspace - mkdir build && cd build - cmake .. -DBUILD_TESTING=ON -DENABLE_TENSORRT=ON - make -j - - # test on rt-detr-v2 correctness - ./bin/simple_tests --gtest_filter=*rtdetr*correctness - # test on rt-detr-v2 speed - GLOG_minloglevel=1 ./bin/simple_tests --gtest_filter=*rtdetr*speed - ``` - - -## 3. MobileSAM and NanoSAM - -### 3.1 Mobile/Nano-SAM-with-TensorRT - -#### 3.1.1 Get the onnx model - - - Download the onnx models from [goolge driver](https://drive.google.com/drive/folders/1yVEOzo59aob_1uXwv343oeh0dTKuHT58?usp=drive_link). Put models under `EasyDeploy/models/`. - -#### 3.1.2 Build TensorRT engine with onnx model - - Convert models with `trtexec` tool. - ```bash - bash docker/into_docker.sh - - # convert mobile-sam image encoder - /usr/src/tensorrt/bin/trtexec --onnx=/workspace/models/mobile_sam_encoder.onnx \ - --saveEngine=/workspace/models/mobile_sam_encoder.engine - # convert nano-sam image encoder - /usr/src/tensorrt/bin/trtexec --onnx=/workspace/models/nanosam_image_encoder_opset11.onnx \ - --saveEngine=/workspace/models/nanosam_image_encoder_opset11.engine \ - --fp16 - # convert decoder-with-point - /usr/src/tensorrt/bin/trtexec --onnx=/workspace/models/modified_mobile_sam_point.onnx \ - --saveEngine=/workspace/models/modified_mobile_sam_point.engine \ - --fp16 - # convert decoder-with-box - /usr/src/tensorrt/bin/trtexec --onnx=/workspace/models/modified_mobile_sam_box.onnx \ - --saveEngine=/workspace/models/modified_mobile_sam_box.engine \ - --fp16 - ``` - -#### 3.1.3 Run test demo - - Now we get all tensorrt engine models used in mobilesam/nanosam. Run simple tests. - ```bash - cd /workspace - mkdir build && cd build - cmake .. -DBUILD_TESTING=ON -DENABLE_TENSORRT=ON - make -j - - # test on mobilesam correctness - ./bin/simple_tests --gtest_filter=*mobilesam*correctness - # test on mobilesam speed - GLOG_minloglevel=1 ./bin/simple_tests --gtest_filter=*mobilesam*speed - ``` - - - **Note** You may want to change the image encoder path in `/workspace/simple_tests/src/test_jetson_devkit.cpp` to `/workspace/models/nanosam_image_encoder_opset11.engine`, which could turn the mobilesam algorithm into nanosam algorithm. - -### 3.2 Nano-SAM-with-RKNN - -#### 3.2.1 Get the onnx model - - - Download the onnx models from [goolge driver](https://drive.google.com/drive/folders/1yVEOzo59aob_1uXwv343oeh0dTKuHT58?usp=drive_link). Put models under `EasyDeploy/models/`. - -#### 3.2.2 Build rknn model from onnx model - - - Use python scripts in `EasyDeploy/tools/`. The python enviroment should been setup in docker already. - ```bash - cd tools - python3 cvt_onnx2rknn_nanosam.py - python3 cvt_onnx2rknn_mobilesam_point_decoder.py - python3 cvt_onnx2rknn_mobilesam_box_decoder.py - ``` - -#### 3.2.3 Run test demo - - Run demos under `simple_tests/src/test_rk_devkit.cpp`. - ```bash - cd /workspace - mkdir build && cd build - cmake .. -DBUILD_TESTING=ON -DENABLE_RKNN=ON - make -j - - # test on mobilesam correctness - ./bin/simple_tests --gtest_filter=*mobilesam*correctness - # test on mobilesam speed - GLOG_minloglevel=1 ./bin/simple_tests --gtest_filter=*mobilesam*speed - ``` \ No newline at end of file diff --git a/docker/build_docker.sh b/docker/build_docker.sh deleted file mode 100644 index b3ac441..0000000 --- a/docker/build_docker.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash - -IMAGE_BASE_NAME="easy_deploy_base_dev" -BUILT_IMAGE_TAG="" - -CONTAINER_NAME="easy_deploy" - -usage() { - echo "Usage: $0 --platform=" - echo "Available platforms: jetson, nvidia_gpu, rk3588" - exit 1 -} - -parse_args() { - if [ "$#" -ne 1 ]; then - usage - fi - # 解析参数 - for i in "$@"; do - case $i in - --platform=*) - PLATFORM="${i#*=}" - shift - ;; - *) - usage - ;; - esac - done -} - -is_image_exist() { - local name="$1" - if docker images --filter "reference=$name" \ - --format "{{.Repository}}:{{.Tag}}" | grep -q "$name"; then - return 0 - else - return 1 - fi -} - -is_container_exist() { - local name="$1" - if docker ps -a --filter "name=$name" | grep -q "$name"; then - return 0 - else - return 1 - fi -} - -build_rk3588_image() { - BUILT_IMAGE_TAG=${IMAGE_BASE_NAME}:rknn_u2204 - if is_image_exist ${BUILT_IMAGE_TAG}; then - echo Image: ${BUILT_IMAGE_TAG} exists! Skip image building process ... - else - docker build -f rknn_u2204.dockerfile -t ${BUILT_IMAGE_TAG} . - fi -} - -build_jetson_image() { - BUILT_IMAGE_TAG=${IMAGE_BASE_NAME}:jetson_tensorrt_u2004 - if is_image_exist ${BUILT_IMAGE_TAG}; then - echo Image: ${BUILT_IMAGE_TAG} exists! Skip image building process ... - else - docker build -f jetson_tensorrt_u2004.dockerfile -t ${BUILT_IMAGE_TAG} . - fi -} - -build_nvidia_gpu_image() { - BUILT_IMAGE_TAG=${IMAGE_BASE_NAME}:nvidia_gpu_tensorrt_u2204 - if is_image_exist ${BUILT_IMAGE_TAG}; then - echo Image: ${BUILT_IMAGE_TAG} exists! Skip image building process ... - else - docker build -f nvidia_gpu_tensorrt_u2204.dockerfile -t ${BUILT_IMAGE_TAG} . - fi -} - -build_image() { - case $PLATFORM in - jetson) - echo "Start Building Docker image for Jetson platform..." - build_jetson_image - ;; - nvidia_gpu) - echo "Start Building Docker image for nvidia_gpu platform..." - build_nvidia_gpu_image - ;; - rk3588) - echo "Start Building Docker image for rk3588 platform..." - build_rk3588_image - ;; - *) - echo "Unknown platform: $PLATFORM" - usage - ;; - esac -} - -add_user() { - echo Adding User: ${USER} into container - -} - -create_container() { - echo "Creating docker container ..." - - if ! is_image_exist ${BUILT_IMAGE_TAG}; then - echo Image: ${BUILT_IMAGE_TAG} does not exist, quit creating ... - exit 1 - fi - - if is_container_exist ${CONTAINER_NAME}; then - echo Container: ${CONTAINER_NAME} exists! Skip container building process ... - return 0 - fi - - EXTERNAL_TAG="" - case $PLATFORM in - jetson) - EXTERNAL_TAG="--runtime nvidia" - ;; - nvidia_gpu) - EXTERNAL_TAG="--runtime nvidia" - ;; - rk3588) - - ;; - *) - ;; - esac - - docker run -itd --privileged \ - --device /dev/dri \ - --group-add video \ - -v /tmp/.X11-unix:/tmp/.X11-unix \ - --network host \ - --ipc host \ - -v $(dirname "$(pwd)"):/workspace \ - -w /workspace \ - -v /dev/bus/usb:/dev/bus/usb \ - -e DISPLAY=${DISPLAY} \ - -e DOCKER_USER=${USER} \ - -e USER=${USER} \ - --name ${CONTAINER_NAME} \ - ${EXTERNAL_TAG} \ - ${BUILT_IMAGE_TAG} \ - /bin/bash -} - -parse_args "$@" - -build_image - -create_container - -echo "EasyDeploy Base Dev Enviroment Built Successfully!!!" -echo "Now Run into_docker.sh" \ No newline at end of file diff --git a/docker/into_docker.sh b/docker/into_docker.sh deleted file mode 100644 index da6a324..0000000 --- a/docker/into_docker.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -CONTAINER_NAME="easy_deploy" -docker start $CONTAINER_NAME -docker exec -it $CONTAINER_NAME /bin/bash \ No newline at end of file diff --git a/docker/jetson_tensorrt_u2004.dockerfile b/docker/jetson_tensorrt_u2004.dockerfile deleted file mode 100644 index 88e2b77..0000000 --- a/docker/jetson_tensorrt_u2004.dockerfile +++ /dev/null @@ -1,74 +0,0 @@ - -# Base image starts with CUDA -ARG BASE_IMG=nvcr.io/nvidia/l4t-tensorrt:r8.5.2.2-devel -FROM ${BASE_IMG} as base -ENV BASE_IMG=nvcr.io/nvidia/l4t-tensorrt:r8.5.2.2-devel - -ENV TENSORRT_VERSION=8.5.2.2 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN rm /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ focal main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ focal-security main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ focal-updates main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ focal-backports main restricted universe multiverse" >> /etc/apt/sources.list && \ - apt-get update - - -# Install basic dependencies -RUN apt install -y \ - build-essential \ - manpages-dev \ - wget \ - zlib1g \ - software-properties-common \ - git \ - libssl-dev \ - zlib1g-dev \ - libbz2-dev \ - libreadline-dev \ - libsqlite3-dev \ - wget \ - ca-certificates \ - curl \ - llvm \ - libncurses5-dev \ - xz-utils tk-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libffi-dev \ - liblzma-dev \ - mecab-ipadic-utf8 \ - libopencv-dev \ - cmake - -# install cmake-3.22.6 -RUN cd /tmp && \ - wget https://gp.zz990099.cn/https://github.com/Kitware/CMake/releases/download/v3.22.6/cmake-3.22.6.tar.gz && \ - tar -xzvf cmake-3.22.6.tar.gz && \ - rm cmake-3.22.6.tar.gz -RUN cd /tmp/cmake-3.22.6/ && \ - ./configure && \ - make -j4 && \ - make install - -# install glog -RUN cd /tmp && \ - wget https://gp.zz990099.cn/https://github.com/google/glog/archive/refs/tags/v0.5.0.tar.gz && \ - tar -xzvf v0.5.0.tar.gz && \ - rm v0.5.0.tar.gz -RUN cd /tmp/glog-0.5.0 && \ - mkdir build && cd build && \ - cmake .. && make -j4 && \ - make install - -# install gtest -RUN cd /tmp && \ - wget https://gp.zz990099.cn/https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz && \ - tar -xzvf release-1.11.0.tar.gz && \ - rm release-1.11.0.tar.gz -RUN cd /tmp/googletest-release-1.11.0 && \ - mkdir build && cd build && \ - cmake .. && make -j4 && \ - make install \ No newline at end of file diff --git a/docker/nvidia_gpu_tensorrt_u2204.dockerfile b/docker/nvidia_gpu_tensorrt_u2204.dockerfile deleted file mode 100644 index df08b87..0000000 --- a/docker/nvidia_gpu_tensorrt_u2204.dockerfile +++ /dev/null @@ -1,74 +0,0 @@ - -# Base image starts with CUDA -ARG BASE_IMG=nvcr.io/nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04 -FROM ${BASE_IMG} as base -ENV BASE_IMG=nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04 - -ENV TENSORRT_VERSION=8.6.1 -ENV TENSORRT_PACAKGE_VERSION=8.6.1.6-1+cuda11.8 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN rm /etc/apt/sources.list && \ - echo "deb https://mirrors.ustc.edu.cn/ubuntu/ jammy main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb https://mirrors.ustc.edu.cn/ubuntu/ jammy-security main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb https://mirrors.ustc.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb https://mirrors.ustc.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse" >> /etc/apt/sources.list && \ - apt-get update - - -# Install basic dependencies -RUN apt install -y \ - build-essential \ - manpages-dev \ - wget \ - zlib1g \ - software-properties-common \ - git \ - libssl-dev \ - zlib1g-dev \ - libbz2-dev \ - libreadline-dev \ - libsqlite3-dev \ - wget \ - ca-certificates \ - curl \ - llvm \ - libncurses5-dev \ - xz-utils tk-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libffi-dev \ - liblzma-dev \ - mecab-ipadic-utf8 \ - cmake \ - libopencv-dev \ - libeigen3-dev \ - libgoogle-glog-dev \ - libgtest-dev - -# Install TensorRT + dependencies -RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub -RUN add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" -RUN apt-get update -RUN TENSORRT_MAJOR_VERSION=`echo ${TENSORRT_VERSION} | cut -d '.' -f 1` && \ - apt-get install -y libnvinfer${TENSORRT_MAJOR_VERSION}=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-plugin${TENSORRT_MAJOR_VERSION}=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-headers-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-headers-plugin-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-plugin-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvonnxparsers${TENSORRT_MAJOR_VERSION}=${TENSORRT_PACAKGE_VERSION} \ - libnvonnxparsers-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvparsers${TENSORRT_MAJOR_VERSION}=${TENSORRT_PACAKGE_VERSION} \ - libnvparsers-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-lean${TENSORRT_MAJOR_VERSION}=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-lean-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-dispatch${TENSORRT_MAJOR_VERSION}=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-dispatch-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-vc-plugin${TENSORRT_MAJOR_VERSION}=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-vc-plugin-dev=${TENSORRT_PACAKGE_VERSION} \ - libnvinfer-samples=${TENSORRT_PACAKGE_VERSION} - -RUN cd /usr/src/tensorrt/samples \ - && make -j \ No newline at end of file diff --git a/docker/rknn_u2204.dockerfile b/docker/rknn_u2204.dockerfile deleted file mode 100644 index d53c4d4..0000000 --- a/docker/rknn_u2204.dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -FROM ubuntu:22.04 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN rm /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ jammy main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ jammy-updates main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ jammy-backports main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.ustc.edu.cn/ubuntu-ports/ jammy-security main restricted universe multiverse" >> /etc/apt/sources.list && \ - apt-get update - -RUN apt-get install -y \ - build-essential \ - manpages-dev \ - wget \ - software-properties-common \ - git \ - libssl-dev \ - libbz2-dev \ - libreadline-dev \ - libsqlite3-dev \ - wget \ - ca-certificates \ - curl \ - llvm \ - libncurses5-dev \ - xz-utils tk-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libffi-dev \ - mecab-ipadic-utf8 \ - sudo - -RUN apt-get install -y \ - cmake \ - libopencv-dev \ - libeigen3-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - libassimp-dev \ - assimp-utils - -# use github release proxy speedup -# see [https://github.com/hunshcn/gh-proxy] -RUN cd /tmp && \ - wget https://gp.zz990099.cn/https://github.com/airockchip/rknn-toolkit2/archive/refs/tags/v2.3.0.tar.gz - -RUN cd /tmp && \ - tar -xzvf v2.3.0.tar.gz && \ - rm v2.3.0.tar.gz - -RUN cd /tmp/rknn-toolkit2-2.3.0/rknpu2/runtime/Linux/librknn_api/ && \ - cp include/* /usr/include/ && \ - cp aarch64/* /usr/lib - -# rknn-toolkit2 python package on arm64 -RUN apt-get install python3-pip -y && \ - pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple && \ - pip install pip --upgrade - -RUN cd /tmp/rknn-toolkit2-2.3.0/rknn-toolkit2/packages/arm64 && \ - pip install rknn_toolkit2-2.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl diff --git a/easy_deploy_tool b/easy_deploy_tool new file mode 160000 index 0000000..4a01290 --- /dev/null +++ b/easy_deploy_tool @@ -0,0 +1 @@ +Subproject commit 4a012904f39be0c35f0da9921e5c761f3ef1e2bb diff --git a/inference_core/CMakeLists.txt b/inference_core/CMakeLists.txt deleted file mode 100644 index 33f3cf4..0000000 --- a/inference_core/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(inference_core) - -if (ENABLE_ORT) - add_subdirectory(ort_core) -endif() - -if (ENABLE_TENSORRT) - add_subdirectory(trt_core) -endif() - -if (ENABLE_RKNN) - add_subdirectory(rknn_core) -endif() \ No newline at end of file diff --git a/inference_core/ort_core/CMakeLists.txt b/inference_core/ort_core/CMakeLists.txt deleted file mode 100644 index e7a5dd7..0000000 --- a/inference_core/ort_core/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(ort_core) - -add_compile_options(-std=c++17) -add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) -set(CMAKE_CXX_STANDARD 17) - -find_package(glog REQUIRED) - -if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") - message(STATUS "Building OnnxRuntime-Core on x86_64 platform ...") - set(ONNXRUNTIME_INCLUDE_DIRS onnxruntime-pkg-x86/include) - FILE(GLOB ONNXRUNTIME_LIBS onnxruntime-pkg-x86/lib/*.so) -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") - message(STATUS "Building OnnxRuntime-Core on aarch64 platform ...") - set(ONNXRUNTIME_INCLUDE_DIRS onnxruntime-pkg-aarch64/include) - FILE(GLOB ONNXRUNTIME_LIBS onnxruntime-pkg-aarch64/lib/*.so) -else() - message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}. \ - Supported architectures are x86_64 and aarch64.") -endif() - - -set(source_file - src/ort_core.cpp - src/ort_core_factory.cpp -) - -include_directories( - include - ${ONNXRUNTIME_INCLUDE_DIRS} -) - -add_library(${PROJECT_NAME} SHARED ${source_file}) - -target_link_libraries(${PROJECT_NAME} PUBLIC - ${ONNXRUNTIME_LIBS} - glog::glog - deploy_core -) - -install(TARGETS ${PROJECT_NAME} - LIBRARY DESTINATION lib) - -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file diff --git a/inference_core/ort_core/include/ort_core/ort_core.h b/inference_core/ort_core/include/ort_core/ort_core.h deleted file mode 100644 index 1305baa..0000000 --- a/inference_core/ort_core/include/ort_core/ort_core.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:05 - * @LastEditTime: 2024-12-02 19:43:10 - * @FilePath: /easy_deploy/inference_core/rknn_core/include/rknn_core/rknn_core.h - */ -#ifndef __EASY_DEPLOY_INFERENCE_CORE_ORT_CORE_H -#define __EASY_DEPLOY_INFERENCE_CORE_ORT_CORE_H - -#include "deploy_core/base_infer_core.h" - -namespace inference_core { - -std::shared_ptr CreateOrtInferCore( - const std::string onnx_path, - const std::unordered_map> &input_blobs_shape = {}, - const std::unordered_map> &output_blobs_shape = {}, - const int num_threads = 0); - -std::shared_ptr CreateOrtInferCoreFactory( - const std::string onnx_path, - const std::unordered_map> &input_blobs_shape = {}, - const std::unordered_map> &output_blobs_shape = {}, - const int num_threads = 0); - -} // namespace inference_core - -#endif \ No newline at end of file diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/GIT_COMMIT_ID b/inference_core/ort_core/onnxruntime-pkg-aarch64/GIT_COMMIT_ID deleted file mode 100644 index a1dfcbc..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/GIT_COMMIT_ID +++ /dev/null @@ -1 +0,0 @@ -387127404e6c1d84b3468c387d864877ed1c67fe diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/LICENSE b/inference_core/ort_core/onnxruntime-pkg-aarch64/LICENSE deleted file mode 100644 index 48bc6bb..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) Microsoft Corporation - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/Privacy.md b/inference_core/ort_core/onnxruntime-pkg-aarch64/Privacy.md deleted file mode 100644 index fcc8468..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/Privacy.md +++ /dev/null @@ -1,21 +0,0 @@ -# Privacy - -## Data Collection -The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft's privacy statement. Our privacy statement is located at https://go.microsoft.com/fwlink/?LinkID=824704. You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. - -*** - -### Private Builds -No data collection is performed when using your private builds built from source code. - -### Official Builds -ONNX Runtime does not maintain any independent telemetry collection mechanisms outside of what is provided by the platforms it supports. However, where applicable, ONNX Runtime will take advantage of platform-supported telemetry systems to collect trace events with the goal of improving product quality. - -Currently telemetry is only implemented for Windows builds and is turned **ON** by default in the official builds distributed in their respective package management repositories ([see here](../README.md#binaries)). This may be expanded to cover other platforms in the future. Data collection is implemented via 'Platform Telemetry' per vendor platform providers (see [telemetry.h](../onnxruntime/core/platform/telemetry.h)). - -#### Technical Details -The Windows provider uses the [TraceLogging](https://docs.microsoft.com/en-us/windows/win32/tracelogging/trace-logging-about) API for its implementation. This enables ONNX Runtime trace events to be collected by the operating system, and based on user consent, this data may be periodically sent to Microsoft servers following GDPR and privacy regulations for anonymity and data access controls. - -Windows ML and onnxruntime C APIs allow Trace Logging to be turned on/off (see [API pages](../README.md#api-documentation) for details). -For information on how to enable and disable telemetry, see [C API: Telemetry](./C_API.md#telemetry). -There are equivalent APIs in the C#, Python, and Java language bindings as well. diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/README.md b/inference_core/ort_core/onnxruntime-pkg-aarch64/README.md deleted file mode 100644 index 24c3e19..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/README.md +++ /dev/null @@ -1,61 +0,0 @@ -

- -**ONNX Runtime is a cross-platform inference and training machine-learning accelerator**. - -**ONNX Runtime inference** can enable faster customer experiences and lower costs, supporting models from deep learning frameworks such as PyTorch and TensorFlow/Keras as well as classical machine learning libraries such as scikit-learn, LightGBM, XGBoost, etc. ONNX Runtime is compatible with different hardware, drivers, and operating systems, and provides optimal performance by leveraging hardware accelerators where applicable alongside graph optimizations and transforms. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-inferencing) - -**ONNX Runtime training** can accelerate the model training time on multi-node NVIDIA GPUs for transformer models with a one-line addition for existing PyTorch training scripts. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-training) - -## Get Started & Resources - -* **General Information**: [onnxruntime.ai](https://onnxruntime.ai) - -* **Usage documentation and tutorials**: [onnxruntime.ai/docs](https://onnxruntime.ai/docs) - -* **YouTube video tutorials**: [youtube.com/@ONNXRuntime](https://www.youtube.com/@ONNXRuntime) - -* [**Upcoming Release Roadmap**](https://github.com/microsoft/onnxruntime/wiki/Upcoming-Release-Roadmap) - -* **Companion sample repositories**: - - ONNX Runtime Inferencing: [microsoft/onnxruntime-inference-examples](https://github.com/microsoft/onnxruntime-inference-examples) - - ONNX Runtime Training: [microsoft/onnxruntime-training-examples](https://github.com/microsoft/onnxruntime-training-examples) - -## Builtin Pipeline Status - -|System|Inference|Training| -|---|---|---| -|Windows|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20CPU%20CI%20Pipeline?label=Windows+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=9)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20GPU%20CI%20Pipeline?label=Windows+GPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=10)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20GPU%20TensorRT%20CI%20Pipeline?label=Windows+GPU+TensorRT)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=47)|| -|Linux|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20CPU%20CI%20Pipeline?label=Linux+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=11)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20CPU%20Minimal%20Build%20E2E%20CI%20Pipeline?label=Linux+CPU+Minimal+Build)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=64)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20GPU%20CI%20Pipeline?label=Linux+GPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=12)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20GPU%20TensorRT%20CI%20Pipeline?label=Linux+GPU+TensorRT)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=45)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20OpenVINO%20CI%20Pipeline?label=Linux+OpenVINO)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=55)|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining-linux-ci-pipeline?label=Linux+CPU+Training)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=86)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining-linux-gpu-ci-pipeline?label=Linux+GPU+Training)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=84)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining/orttraining-ortmodule-distributed?label=Training+Distributed)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=148)| -|Mac|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/MacOS%20CI%20Pipeline?label=MacOS+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=13)|| -|Android|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Android%20CI%20Pipeline?label=Android)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=53)|| -|iOS|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/iOS%20CI%20Pipeline?label=iOS)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=134)|| -|Web|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/ONNX%20Runtime%20Web%20CI%20Pipeline?label=Web)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=161)|| -|Other|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/onnxruntime-binary-size-checks-ci-pipeline?repoName=microsoft%2Fonnxruntime&label=Binary+Size+Check)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=187&repoName=microsoft%2Fonnxruntime)|| - -## Third-party Pipeline Status - -|System|Inference|Training| -|---|---|---| -|Linux|[![Build Status](https://github.com/Ascend/onnxruntime/actions/workflows/build-and-test.yaml/badge.svg)](https://github.com/Ascend/onnxruntime/actions/workflows/build-and-test.yaml)|| - -## Data/Telemetry - -Windows distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. - -## Contributions and Feedback - -We welcome contributions! Please see the [contribution guidelines](CONTRIBUTING.md). - -For feature requests or bug reports, please file a [GitHub Issue](https://github.com/Microsoft/onnxruntime/issues). - -For general discussion or questions, please use [GitHub Discussions](https://github.com/microsoft/onnxruntime/discussions). - -## Code of Conduct - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) -or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - -## License - -This project is licensed under the [MIT License](LICENSE). diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/ThirdPartyNotices.txt b/inference_core/ort_core/onnxruntime-pkg-aarch64/ThirdPartyNotices.txt deleted file mode 100644 index 8ec770d..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/ThirdPartyNotices.txt +++ /dev/null @@ -1,6508 +0,0 @@ -THIRD PARTY SOFTWARE NOTICES AND INFORMATION - -Do Not Translate or Localize - -This software incorporates material from third parties. Microsoft makes certain -open source code available at http://3rdpartysource.microsoft.com, or you may -send a check or money order for US $5.00, including the product name, the open -source component name, and version number, to: - -Source Code Compliance Team -Microsoft Corporation -One Microsoft Way -Redmond, WA 98052 -USA - -Notwithstanding any other terms, you may reverse engineer this software to the -extent required to debug changes to any libraries licensed under the GNU Lesser -General Public License. - -_____ - -Intel Math Kernel Library (Intel MKL) - -Intel Simplified Software License (Version April 2018) - -Copyright (c) 2018 Intel Corporation. - -Use and Redistribution. You may use and redistribute the software (the “Software”), without modification, -provided the following conditions are met: - -* Redistributions must reproduce the above copyright notice and the following terms of use in the Software -and in the documentation and/or other materials provided with the distribution. - -* Neither the name of Intel nor the names of its suppliers may be used to endorse or promote products -derived from this Software without specific prior written permission. - -* No reverse engineering, decompilation, or disassembly of this Software is permitted. - -Limited patent license. Intel grants you a world-wide, royalty-free, non-exclusive license under patents it now -or hereafter owns or controls to make, have made, use, import, offer to sell and sell (“Utilize”) this Software, -but solely to the extent that any such patent is necessary to Utilize the Software alone. The patent license -shall not apply to any combinations which include this software. No hardware per se is licensed hereunder. - -Third party and other Intel programs. “Third Party Programs” are the files listed in the “third-party-programs.txt” -text file that is included with the Software and may include Intel programs under separate license terms. -Third Party Programs, even if included with the distribution of the Materials, are governed by -separate license terms and those license terms solely govern your use of those programs. - -DISCLAIMER. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT ARE DISCLAIMED. THIS SOFTWARE IS -NOT INTENDED FOR USE IN SYSTEMS OR APPLICATIONS WHERE FAILURE OF THE SOFTWARE -MAY CAUSE PERSONAL INJURY OR DEATH AND YOU AGREE THAT YOU ARE FULLY RESPONSIBLE FOR ANY -CLAIMS, COSTS, DAMAGES, EXPENSES, AND ATTORNEYS’ FEES ARISING OUT OF ANY SUCH USE, -EVEN IF ANY CLAIM ALLEGES THAT INTEL WAS NEGLIGENT REGARDING THE DESIGN OR MANUFACTURE OF -THE MATERIALS. - -LIMITATION OF LIABILITY. IN NO EVENT WILL INTEL BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -OF SUCH DAMAGE. YOU AGREE TO INDEMNIFY AND HOLD INTEL HARMLESS AGAINST ANY CLAIMS -AND EXPENSES RESULTING FROM YOUR USE OR UNAUTHORIZED USE OF THE SOFTWARE. - -No support. Intel may make changes to the Software, at any time without notice, and is not obligated to -support, update or provide training for the Software. - -Termination. Intel may terminate your right to use the Software in the event of your breach of this Agreement -and you fail to cure the breach within a reasonable period of time. - -Feedback. Should you provide Intel with comments, modifications, corrections, enhancements or other input -(“Feedback”) related to the Software Intel will be free to use, disclose, reproduce, license or otherwise -distribute or exploit the Feedback in its sole discretion without any obligations or restrictions of any kind, -including without limitation, intellectual property rights or licensing obligations. - -Compliance with laws. You agree to comply with all relevant laws and regulations governing your use, -transfer, import or export (or prohibition thereof) of the Software. - -Governing law. All disputes will be governed by the laws of the United States of America and the State of -Delaware without reference to conflict of law principles and subject to the exclusive jurisdiction of the state or -federal courts sitting in the State of Delaware, and each party agrees that it submits to the personal -jurisdiction and venue of those courts and waives any objections. The United Nations Convention on -Contracts for the International Sale of Goods (1980) is specifically excluded and will not apply to the -Software. - -*Other names and brands may be claimed as the property of others. - -_____ - -protocolbuffers/protobuf - -Copyright 2008 Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -_____ - -madler/zlib - -The deflate format used by zlib was defined by Phil Katz. The deflate and -zlib specifications were written by L. Peter Deutsch. Thanks to all the -people who reported problems and suggested various improvements in zlib; they -are too numerous to cite here. - -Copyright notice: - - (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - -If you use the zlib library in a product, we would appreciate *not* receiving -lengthy legal documents to sign. The sources are provided for free but without -warranty of any kind. The library has been entirely written by Jean-loup -Gailly and Mark Adler; it does not include third-party code. - -If you redistribute modified sources, we would appreciate that you include in -the file ChangeLog history information documenting your changes. Please read -the FAQ for more information on the distribution of modified source versions. - -_____ - -pybind/pybind11 - -Copyright (c) 2016 Wenzel Jakob , All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Please also refer to the file CONTRIBUTING.md, which clarifies licensing of -external contributions to this project including patches, pull requests, etc. - -_____ - -onnx -Open Neural Network Exchange - -Copyright (c) Facebook, Inc. and Microsoft Corporation. All rights reserved. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -Eigen - -MPL v2.0 -Mozilla Public License Version 2.0 - - -================================== - -1. Definitions - --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions - --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities - -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination - --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. - -_____ - -intel/dnnl - -Copyright 2016-2018 Intel Corporation - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -sub-components: - -xbyak - -Copyright (c) 2007 MITSUNARI Shigeo. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. -Neither the name of the copyright owner nor the names of its contributors may -be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -Microsoft GSL - -Copyright (c) 2015 Microsoft Corporation. All rights reserved. - -This code is licensed under the MIT License (MIT). - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -_____ - -Tensorflow - -Copyright 2018 The TensorFlow Authors. All rights reserved. - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, -and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by -the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all -other entities that control, are controlled by, or are under common -control with that entity. For the purposes of this definition, -"control" means (i) the power, direct or indirect, to cause the -direction or management of such entity, whether by contract or -otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity -exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, -including but not limited to software source code, documentation -source, and configuration files. - -"Object" form shall mean any form resulting from mechanical -transformation or translation of a Source form, including but -not limited to compiled object code, generated documentation, -and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or -Object form, made available under the License, as indicated by a -copyright notice that is included in or attached to the work -(an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object -form, that is based on (or derived from) the Work and for which the -editorial revisions, annotations, elaborations, or other modifications -represent, as a whole, an original work of authorship. For the purposes -of this License, Derivative Works shall not include works that remain -separable from, or merely link (or bind by name) to the interfaces of, -the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including -the original version of the Work and any modifications or additions -to that Work or Derivative Works thereof, that is intentionally -submitted to Licensor for inclusion in the Work by the copyright owner -or by an individual or Legal Entity authorized to submit on behalf of -the copyright owner. For the purposes of this definition, "submitted" -means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, -and issue tracking systems that are managed by, or on behalf of, the -Licensor for the purpose of discussing and improving the Work, but -excluding communication that is conspicuously marked or otherwise -designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity -on behalf of whom a Contribution has been received by Licensor and -subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of -this License, each Contributor hereby grants to You a perpetual, -worldwide, non-exclusive, no-charge, royalty-free, irrevocable -copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the -Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of -this License, each Contributor hereby grants to You a perpetual, -worldwide, non-exclusive, no-charge, royalty-free, irrevocable -(except as stated in this section) patent license to make, have made, -use, offer to sell, sell, import, and otherwise transfer the Work, -where such license applies only to those patent claims licensable -by such Contributor that are necessarily infringed by their -Contribution(s) alone or by combination of their Contribution(s) -with the Work to which such Contribution(s) was submitted. If You -institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work -or a Contribution incorporated within the Work constitutes direct -or contributory patent infringement, then any patent licenses -granted to You under this License for that Work shall terminate -as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the -Work or Derivative Works thereof in any medium, with or without -modifications, and in Source or Object form, provided that You -meet the following conditions: - -(a) You must give any other recipients of the Work or -Derivative Works a copy of this License; and - -(b) You must cause any modified files to carry prominent notices -stating that You changed the files; and - -(c) You must retain, in the Source form of any Derivative Works -that You distribute, all copyright, patent, trademark, and -attribution notices from the Source form of the Work, -excluding those notices that do not pertain to any part of -the Derivative Works; and - -(d) If the Work includes a "NOTICE" text file as part of its -distribution, then any Derivative Works that You distribute must -include a readable copy of the attribution notices contained -within such NOTICE file, excluding those notices that do not -pertain to any part of the Derivative Works, in at least one -of the following places: within a NOTICE text file distributed -as part of the Derivative Works; within the Source form or -documentation, if provided along with the Derivative Works; or, -within a display generated by the Derivative Works, if and -wherever such third-party notices normally appear. The contents -of the NOTICE file are for informational purposes only and -do not modify the License. You may add Your own attribution -notices within Derivative Works that You distribute, alongside -or as an addendum to the NOTICE text from the Work, provided -that such additional attribution notices cannot be construed -as modifying the License. - -You may add Your own copyright statement to Your modifications and -may provide additional or different license terms and conditions -for use, reproduction, or distribution of Your modifications, or -for any such Derivative Works as a whole, provided Your use, -reproduction, and distribution of the Work otherwise complies with -the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, -any Contribution intentionally submitted for inclusion in the Work -by You to the Licensor shall be under the terms and conditions of -this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify -the terms of any separate license agreement you may have executed -with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade -names, trademarks, service marks, or product names of the Licensor, -except as required for reasonable and customary use in describing the -origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or -agreed to in writing, Licensor provides the Work (and each -Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -implied, including, without limitation, any warranties or conditions -of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A -PARTICULAR PURPOSE. You are solely responsible for determining the -appropriateness of using or redistributing the Work and assume any -risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, -whether in tort (including negligence), contract, or otherwise, -unless required by applicable law (such as deliberate and grossly -negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, -incidental, or consequential damages of any character arising as a -result of this License or out of the use or inability to use the -Work (including but not limited to damages for loss of goodwill, -work stoppage, computer failure or malfunction, or any and all -other commercial damages or losses), even if such Contributor -has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing -the Work or Derivative Works thereof, You may choose to offer, -and charge a fee for, acceptance of support, warranty, indemnity, -or other liability obligations and/or rights consistent with this -License. However, in accepting such obligations, You may act only -on Your own behalf and on Your sole responsibility, not on behalf -of any other Contributor, and only if You agree to indemnify, -defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason -of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017, The TensorFlow Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -Microsoft Cognitive Toolkit (CNTK) - -Copyright (c) Microsoft Corporation. All rights reserved. - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT -LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -NumPy License - -Copyright (c) 2005, NumPy Developers - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name of the NumPy Developers nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -Pytorch / Caffe2 - -From PyTorch: - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) -Copyright (c) 2006 Idiap Research Institute (Samy Bengio) -Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) - -From Caffe2: - -Copyright (c) 2016-present, Facebook Inc. All rights reserved. - -All contributions by Facebook: -Copyright (c) 2016 Facebook Inc. - -All contributions by Google: -Copyright (c) 2015 Google Inc. -All rights reserved. - -All contributions by Yangqing Jia: -Copyright (c) 2015 Yangqing Jia -All rights reserved. - -All contributions from Caffe: -Copyright(c) 2013, 2014, 2015, the respective contributors -All rights reserved. - -All other contributions: -Copyright(c) 2015, 2016 the respective contributors -All rights reserved. - -Caffe2 uses a copyright model similar to Caffe: each contributor holds -copyright over their contributions to Caffe2. The project versioning records -all such contribution and copyright details. If a contributor wants to further -mark their specific copyright on a particular contribution, they should -indicate their copyright solely in the commit message of the change when it is -committed. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America - and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -_____ - -Caffe - -COPYRIGHT - -All contributions by the University of California: -Copyright (c) 2014-2017 The Regents of the University of California (Regents) -All rights reserved. - -All other contributions: -Copyright (c) 2014-2017, the respective contributors -All rights reserved. - -Caffe uses a shared copyright model: each contributor holds copyright over -their contributions to Caffe. The project versioning records all such -contribution and copyright details. If a contributor wants to further mark -their specific copyright on a particular contribution, they should indicate -their copyright solely in the commit message of the change when it is -committed. - -LICENSE - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -CONTRIBUTION AGREEMENT - -By contributing to the BVLC/caffe repository through pull-request, comment, -or otherwise, the contributor releases their content to the -license and copyright terms herein. - -_____ - -The LLVM Compiler Infrastructure - -============================================================================== -LLVM Release License -============================================================================== -University of Illinois/NCSA -Open Source License - -Copyright (c) 2003-2017 University of Illinois at Urbana-Champaign. -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== -Copyrights and Licenses for Third Party Software Distributed with LLVM: -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- -Google Test llvm/utils/unittest/googletest -OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} -pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} -ARM contributions llvm/lib/Target/ARM/LICENSE.TXT -md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h - -_____ - -google/benchmark - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -CONTRIBUTORS - -# People who have agreed to one of the CLAs and can contribute patches. -# The AUTHORS file lists the copyright holders; this file -# lists people. For example, Google employees are listed here -# but not in AUTHORS, because Google holds the copyright. -# -# Names should be added to this file only after verifying that -# the individual or the individual's organization has agreed to -# the appropriate Contributor License Agreement, found here: -# -# https://developers.google.com/open-source/cla/individual -# https://developers.google.com/open-source/cla/corporate -# -# The agreement for individuals can be filled out on the web. -# -# When adding J Random Contributor's name to this file, -# either J's name or J's organization's name should be -# added to the AUTHORS file, depending on whether the -# individual or corporate CLA was used. -# -# Names should be added to this file as: -# Name -# -# Please keep the list sorted. - -Albert Pretorius -Arne Beer -Billy Robert O'Neal III -Chris Kennelly -Christopher Seymour -David Coeurjolly -Deniz Evrenci -Dominic Hamon -Dominik Czarnota -Eric Fiselier -Eugene Zhuk -Evgeny Safronov -Federico Ficarelli -Felix Homann -Ismael Jimenez Martinez -Jern-Kuan Leong -JianXiong Zhou -Joao Paulo Magalhaes -John Millikin -Jussi Knuuttila -Kai Wolf -Kishan Kumar -Kaito Udagawa -Lei Xu -Matt Clarkson -Maxim Vafin -Nick Hutchinson -Oleksandr Sochka -Pascal Leroy -Paul Redmond -Pierre Phaneuf -Radoslav Yovchev -Raul Marin -Ray Glover -Robert Guo -Roman Lebedev -Shuo Chen -Tobias Ulvgård -Tom Madams -Yixuan Qiu -Yusuke Suzuki -Zbigniew Skowron - -AUTHORS - -# This is the official list of benchmark authors for copyright purposes. -# This file is distinct from the CONTRIBUTORS files. -# See the latter for an explanation. -# -# Names should be added to this file as: -# Name or Organization -# The email address is not required for organizations. -# -# Please keep the list sorted. - -Albert Pretorius -Arne Beer -Carto -Christopher Seymour -David Coeurjolly -Deniz Evrenci -Dirac Research -Dominik Czarnota -Eric Fiselier -Eugene Zhuk -Evgeny Safronov -Federico Ficarelli -Felix Homann -Google Inc. -International Business Machines Corporation -Ismael Jimenez Martinez -Jern-Kuan Leong -JianXiong Zhou -Joao Paulo Magalhaes -Jussi Knuuttila -Kaito Udagawa -Kishan Kumar -Lei Xu -Matt Clarkson -Maxim Vafin -MongoDB Inc. -Nick Hutchinson -Oleksandr Sochka -Paul Redmond -Radoslav Yovchev -Roman Lebedev -Shuo Chen -Steinar H. Gunderson -Stripe, Inc. -Yixuan Qiu -Yusuke Suzuki -Zbigniew Skowron - -_____ - -HalideIR - -Copyright (c) 2016 HalideIR contributors -Copyright (c) 2012-2014 MIT CSAIL, Google Inc., and other contributors -HalideIR is derived from the Halide project. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -Distributed Machine Learning Common Codebase - -Copyright (c) 2015 by Contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -_____ - -DLPack: Open In Memory Tensor Structure - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 by Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -HowardHinnant/date - -The source code in this project is released using the MIT License. There is no -global license for the project because each file is licensed individually with -different author names and/or dates. - -If you contribute to this project, please add your name to the license of each -file you modify. If you have already contributed to this project and forgot to -add your name to the license, please feel free to submit a new P/R to add your -name to the license in each file you modified. - -For convenience, here is a copy of the MIT license found in each file except -without author names or dates: - -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -TVM Open Deep Learning Compiler Stack - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -CONTRIBUTORS - -TVM Contributors -================ -TVM adopts the Apache style model and governs by merit. We believe that it is important to create an inclusive community where everyone can use, -contribute to, and influence the direction of the project. We actively invite contributors who have earned the merit to be part of the development community. - -See the [community structure document](http://docs.tvm.ai/contribute/community.html) for the explanation of community structure and contribution guidelines. - -## Committers -- [Tianqi Chen](https://github.com/tqchen) (PMC) -- [Thierry Moreau](http://homes.cs.washington.edu/~moreau/) -- [Ziheng Jiang](https://github.com/ZihengJiang) -- [Haichen Shen](http://homes.cs.washington.edu/~haichen/) -- [Yizhi Liu](https://github.com/yzhliu) - -## Code Owners -- [Aditya Atluri](https://github.com/adityaatluri) ROCM -- [Leyuan Wang](https://github.com/Laurawly) TOPI -- [Yuwei Hu](https://github.com/Huyuwei) TOPI -- [Zhixun Tan](https://github.com/phisiart) OpenGL/WebGL backend -- [Nick Hynes](https://github.com/nhynes) SGX and secured computing -- [Lianmin Zheng](https://github.com/merrymercy) AutoTVM - -## Reviewers -- [Zhi Chen](https://github.com/zhiics) -- [Xiaoqiang Dan](https://github.com/xqdan) -- [Liangfu Chen](https://github.com/liangfu) -- [Masahiro Masuda](https://github.com/masahi) -- [Kazutaka Morita](https://github.com/kazum) -- [Tatsuya Nishiyama](https://github.com/nishi-t) -- [Pariksheet Pinjari](https://github.com/PariksheetPinjari909) -- [Jared Roesch](https://github.com/jroesch) -- [Siva](https://github.com/srkreddy1238) -- [Siju Samuel](https://github.com/siju-samuel) -- [Alex Weaver](https://github.com/alex-weaver) -- [Yao Wang](https://github.com/kevinthesun) -- [Jian Weng](https://github.com/were) -- [Eddie Yan](https://github.com/eqy) -- [Joshua Z. Zhang](https://github.com/zhreshold) - -## List of Contributors -- [Full List of Contributors](https://github.com/dmlc/tvm/graphs/contributors) - - To contributors: please add your name to the list. -- [Qiao Zhang](https://github.com/zhangqiaorjc) -- [Haolong Zhang](https://github.com/haolongzhangm) -- [Cody Hao Yu](https://github.com/comaniac) -- [Chris Nuernberger](https://github.com/cnuernber) - -_____ - -FreeBSD: getopt.c file - -Copyright (c) 1987, 1993, 1994 -The Regents of the University of California. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - -3. Neither the name of the University nor the names of its contributors -may be used to endorse or promote products derived from this software -without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. -_____ - - -google/googletest - -Copyright 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -G3log : Asynchronous logger with Dynamic Sinks - -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or -distribute this software, either in source code form or as a compiled -binary, for any purpose, commercial or non-commercial, and by any -means. - -In jurisdictions that recognize copyright laws, the author or authors -of this software dedicate any and all copyright interest in the -software to the public domain. We make this dedication for the benefit -of the public at large and to the detriment of our heirs and -successors. We intend this dedication to be an overt act of -relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to -_____ - -Scikit-learn - -Copyright (c) 2007–2018 The scikit-learn developers. -All rights reserved. - - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - a. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - b. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - c. Neither the name of the Scikit-learn Developers nor the names of - its contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. - -_____ - -google/nsync - -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -google/re2 - -Copyright (c) 2009 The RE2 Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -_____ -onnx/onnx-tensorrt - -MIT License - -Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -Copyright (c) 2018 Open Neural Network Exchange - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ -nvidia/cutlass - -Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: BSD-3-Clause - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ -Boost - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - - -_____ - -JDAI-CV/DNNLibrary - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [2019] [JD.com Inc. JD AI] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -google/flatbuffers - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2014 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -google/glog - -Copyright (c) 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -A function gettimeofday in utilities.cc is based on - -http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd - -The license of this code is: - -Copyright (c) 2003-2008, Jouni Malinen and contributors -All Rights Reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name(s) of the above-listed copyright holder(s) nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -abseil-cpp -https://github.com/abseil/abseil-cpp - - Apache License - Version 2.0, January 2004 - https://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -microsoft/wil - -MIT License - -Copyright (c) Microsoft Corporation. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE - -_____ - -nlohmann/json - -MIT License - -Copyright (c) 2013-2019 Niels Lohmann - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -dcleblanc/SafeInt - -MIT License - -Copyright (c) 2018 Microsoft - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ -Open MPI - -3-Clause BSD License - -Most files in this release are marked with the copyrights of the -organizations who have edited them. The copyrights below are in no -particular order and generally reflect members of the Open MPI core -team who have contributed code to this release. The copyrights for -code used under license from other parties are included in the -corresponding files. - -Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana - University Research and Technology - Corporation. All rights reserved. -Copyright (c) 2004-2017 The University of Tennessee and The University - of Tennessee Research Foundation. All rights - reserved. -Copyright (c) 2004-2010 High Performance Computing Center Stuttgart, - University of Stuttgart. All rights reserved. -Copyright (c) 2004-2008 The Regents of the University of California. - All rights reserved. -Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights - reserved. -Copyright (c) 2006-2017 Cisco Systems, Inc. All rights reserved. -Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved. -Copyright (c) 2006-2017 Sandia National Laboratories. All rights reserved. -Copyright (c) 2006-2010 Sun Microsystems, Inc. All rights reserved. - Use is subject to license terms. -Copyright (c) 2006-2017 The University of Houston. All rights reserved. -Copyright (c) 2006-2009 Myricom, Inc. All rights reserved. -Copyright (c) 2007-2017 UT-Battelle, LLC. All rights reserved. -Copyright (c) 2007-2017 IBM Corporation. All rights reserved. -Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing - Centre, Federal Republic of Germany -Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany -Copyright (c) 2007 Evergrid, Inc. All rights reserved. -Copyright (c) 2008 Chelsio, Inc. All rights reserved. -Copyright (c) 2008-2009 Institut National de Recherche en - Informatique. All rights reserved. -Copyright (c) 2007 Lawrence Livermore National Security, LLC. - All rights reserved. -Copyright (c) 2007-2017 Mellanox Technologies. All rights reserved. -Copyright (c) 2006-2010 QLogic Corporation. All rights reserved. -Copyright (c) 2008-2017 Oak Ridge National Labs. All rights reserved. -Copyright (c) 2006-2012 Oracle and/or its affiliates. All rights reserved. -Copyright (c) 2009-2015 Bull SAS. All rights reserved. -Copyright (c) 2010 ARM ltd. All rights reserved. -Copyright (c) 2016 ARM, Inc. All rights reserved. -Copyright (c) 2010-2011 Alex Brick . All rights reserved. -Copyright (c) 2012 The University of Wisconsin-La Crosse. All rights - reserved. -Copyright (c) 2013-2016 Intel, Inc. All rights reserved. -Copyright (c) 2011-2017 NVIDIA Corporation. All rights reserved. -Copyright (c) 2016 Broadcom Limited. All rights reserved. -Copyright (c) 2011-2017 Fujitsu Limited. All rights reserved. -Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP. All - rights reserved. -Copyright (c) 2013-2017 Research Organization for Information Science (RIST). - All rights reserved. -Copyright (c) 2017-2018 Amazon.com, Inc. or its affiliates. All Rights - reserved. -Copyright (c) 2018 DataDirect Networks. All rights reserved. -Copyright (c) 2018-2019 Triad National Security, LLC. All rights reserved. - -$COPYRIGHT$ - -Additional copyrights may follow - -$HEADER$ - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -- Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -- Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer listed - in this license in the documentation and/or other materials - provided with the distribution. - -- Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -The copyright holders provide no reassurances that the source code -provided does not infringe any patent, copyright, or any other -intellectual property rights of third parties. The copyright holders -disclaim any liability to any recipient for claims brought against -recipient by any third party for infringement of that parties -intellectual property rights. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -The Android Open Source Project - -Copyright (C) 2017 The Android Open Source Project -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ------- - -libprotobuf-mutator - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- - - openucx/ucx - https://github.com/openucx/ucx - - Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. - Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. - Copyright (C) 2014-2015 The University of Houston System. All rights reserved. - Copyright (C) 2015 The University of Tennessee and The University - of Tennessee Research Foundation. All rights reserved. - Copyright (C) 2016-2020 ARM Ltd. All rights reserved. - Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. - Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. - Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. - Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. - Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. - Copyright (C) 2016-2020 Stony Brook University. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - ----- - - From PyTorch: - - Copyright (c) 2016- Facebook, Inc (Adam Paszke) - Copyright (c) 2014- Facebook, Inc (Soumith Chintala) - Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) - Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) - Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) - Copyright (c) 2011-2013 NYU (Clement Farabet) - Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) - Copyright (c) 2006 Idiap Research Institute (Samy Bengio) - Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) - - From Caffe2: - - Copyright (c) 2016-present, Facebook Inc. All rights reserved. - - All contributions by Facebook: - Copyright (c) 2016 Facebook Inc. - - All contributions by Google: - Copyright (c) 2015 Google Inc. - All rights reserved. - - All contributions by Yangqing Jia: - Copyright (c) 2015 Yangqing Jia - All rights reserved. - - All contributions from Caffe: - Copyright(c) 2013, 2014, 2015, the respective contributors - All rights reserved. - - All other contributions: - Copyright(c) 2015, 2016 the respective contributors - All rights reserved. - - Caffe2 uses a copyright model similar to Caffe: each contributor holds - copyright over their contributions to Caffe2. The project versioning records - all such contribution and copyright details. If a contributor wants to further - mark their specific copyright on a particular contribution, they should - indicate their copyright solely in the commit message of the change when it is - committed. - - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America - and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -_____ - - mpi4py - https://github.com/mpi4py/mpi4py/ - - ======================= - LICENSE: MPI for Python - ======================= - - :Author: Lisandro Dalcin - :Contact: dalcinl@gmail.com - - - Copyright (c) 2019, Lisandro Dalcin. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -_____ -huggingface/transformers - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -_____ -msgpack/msgpack-python - -Copyright (C) 2008-2011 INADA Naoki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -_____ -lanpa/tensorboardX - -MIT License - -Copyright (c) 2017 Tzu-Wei Huang - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -_____ -tensorflow/tensorboard - -Copyright 2017 The TensorFlow Authors. All rights reserved. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017, The TensorFlow Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -cerberus - -Cerberus is a lightweight and extensible data validation library for Python. - -ISC License - -Copyright (c) 2012-2016 Nicola Iarocci. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. - -_____ - -MurmurHash3 - -MIT license - -https://github.com/aappleby/smhasher - -SMHasher is a test suite designed to test the distribution, collision, and -performance properties of non-cryptographic hash functions. -This is the home for the MurmurHash family of hash functions along with the -SMHasher test suite used to verify them. -SMHasher is released under the MIT license. -All MurmurHash versions are public domain software, and the author disclaims all copyright to their code. - -_____ - -gtest-ios-framework - -https://github.com/mestevens/gtest-ios-framework - -Copyright (c) 2013 Matthew Stevens - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -DLPack - -https://github.com/dmlc/dlpack - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 by Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -emsdk - -MIT/Expat license - -https://github.com/emscripten-core/emsdk - -Copyright (c) 2018 Emscripten authors (see AUTHORS in Emscripten) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ----------------------------------------------------------------------------- - -This is the MIT/Expat Licence. For more information see: - -1. http://www.opensource.org/licenses/mit-license.php - -2. http://en.wikipedia.org/wiki/MIT_License - -_____ - -coremltools - -BSD-3-Clause License - -https://github.com/apple/coremltools - -Copyright (c) 2020, Apple Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -© 2021 GitHub, Inc. - -_____ - -react-native - -MIT License - -https://github.com/facebook/react-native - -Copyright (c) Facebook, Inc. and its affiliates. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -pytorch/cpuinfo - -BSD 2-Clause "Simplified" License - -https://github.com/pytorch/cpuinfo - -Copyright (c) 2019 Google LLC -Copyright (c) 2017-2018 Facebook Inc. -Copyright (C) 2012-2017 Georgia Institute of Technology -Copyright (C) 2010-2012 Marat Dukhan - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -SQLite Is Public Domain - -All of the code and documentation in SQLite has been dedicated to the public -domain by the authors. All code authors, and representatives of the companies -they work for, have signed affidavits dedicating their contributions to the -public domain and originals of those signed affidavits are stored in a firesafe -at the main offices of Hwaci. Anyone is free to copy, modify, publish, use, -compile, sell, or distribute the original SQLite code, either in source code -form or as a compiled binary, for any purpose, commercial or non-commercial, -and by any means. - -The previous paragraph applies to the deliverable code and documentation in -SQLite - those parts of the SQLite library that you actually bundle and ship -with a larger application. Some scripts used as part of the build process (for -example the "configure" scripts generated by autoconf) might fall under other -open-source licenses. Nothing from these build scripts ever reaches the final -deliverable SQLite library, however, and so the licenses associated with those -scripts should not be a factor in assessing your rights to copy and use the -SQLite library. - -All of the deliverable code in SQLite has been written from scratch. No code -has been taken from other projects or from the open internet. Every line of -code can be traced back to its original author, and all of those authors have -public domain dedications on file. So the SQLite code base is clean and is -uncontaminated with licensed code from other projects. - -_____ - -google/XNNPACK - -BSD License - -For XNNPACK software - -Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. -Copyright 2019 Google LLC - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -google/sentencepiece, https://github.com/google/sentencepiece -(included when statically linked with onnxruntime-extensions) - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -dlfcn-win32/dlfcn-win32 is licensed under the MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -_____ - -The Python Imaging Library (PIL) is - - Copyright © 1997-2011 by Secret Labs AB - Copyright © 1995-2011 by Fredrik Lundh - -Pillow is the friendly PIL fork. It is - - Copyright © 2010-2023 by Alex Clark and contributors - -Like PIL, Pillow is licensed under the open source HPND License: - -By obtaining, using, and/or copying this software and/or its associated -documentation, you agree that you have read, understood, and will comply -with the following terms and conditions: - -Permission to use, copy, modify, and distribute this software and its -associated documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appears in all copies, and that -both that copyright notice and this permission notice appear in supporting -documentation, and that the name of Secret Labs AB or the author not be -used in advertising or publicity pertaining to distribution of the software -without specific, written prior permission. - -SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS -SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. -IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, -INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. - -_____ - -openssl/openssl, https://github.com/openssl/openssl - - Apache License - Version 2.0, January 2004 - https://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -_____ - -Tencent/rapidjson, https://github.com/Tencent/rapidjson - -Tencent is pleased to support the open source community by making RapidJSON available. - -Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. - -If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. -A copy of the MIT License is included in this file. - -Other dependencies and licenses: - -Open Source Software Licensed Under the BSD License: --------------------------------------------------------------------- - -The msinttypes r29 -Copyright (c) 2006-2013 Alexander Chemeris -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* Neither the name of copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Open Source Software Licensed Under the JSON License: --------------------------------------------------------------------- - -json.org -Copyright (c) 2002 JSON.org -All Rights Reserved. - -JSON_checker -Copyright (c) 2002 JSON.org -All Rights Reserved. - - -Terms of the JSON License: ---------------------------------------------------- - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -The Software shall be used for Good, not Evil. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -Terms of the MIT License: --------------------------------------------------------------------- - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -boostorg/boost, https://github.com/boostorg/boost - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -_____ - -libb64/libb64, https://github.com/libb64/libb64 - -Copyright-Only Dedication (based on United States law) or Public Domain Certification - -The person or persons who have associated work with this document (the "Dedicator" or "Certifier") hereby either (a) certifies that, to the best of his knowledge, the work of authorship identified is in the public domain of the country from which the work is published, or (b) hereby dedicates whatever copyright the dedicators holds in the work of authorship identified below (the "Work") to the public domain. A certifier, moreover, dedicates any copyright interest he may have in the associated work, and for these purposes, is described as a "dedicator" below. - -A certifier has taken reasonable steps to verify the copyright status of this work. Certifier recognizes that his good faith efforts may not shield him from liability if in fact the work certified is not in the public domain. - -Dedicator makes this dedication for the benefit of the public at large and to the detriment of the Dedicator's heirs and successors. Dedicator intends this dedication to be an overt act of relinquishment in perpetuity of all present and future rights under copyright law, whether vested or contingent, in the Work. Dedicator understands that such relinquishment of all rights includes the relinquishment of all rights to enforce (by lawsuit or otherwise) those copyrights in the Work. - -Dedicator recognizes that, once placed in the public domain, the Work may be freely reproduced, distributed, transmitted, used, modified, built upon, or otherwise exploited by anyone for any purpose, commercial or non-commercial, and in any way, including by methods that have not yet been invented or conceived. - -_____ - -posix pthread library, https://sourceforge.net/projects/pthreads4w - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -Triton Inference Server & Client, https://github.com/triton-inference-server - -Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of NVIDIA CORPORATION nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -microsoft/mimalloc, https://github.com/microsoft/mimalloc - -MIT License - -Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -TensorFlow.js - -https://github.com/tensorflow/tfjs - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -—— - -curl/curl - -https://github.com/curl - -COPYRIGHT AND PERMISSION NOTICE - -Copyright (C) Daniel Stenberg, , and many -contributors, see the THANKS file. - -All rights reserved. - -Permission to use, copy, modify, and distribute this software for any purpose -with or without fee is hereby granted, provided that the above copyright -notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN -NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. - -Except as contained in this notice, the name of a copyright holder shall not -be used in advertising or otherwise to promote the sale, use or other dealings -in this Software without prior written authorization of the copyright holder. - -_____ - -Intel neural-compressor - -https://github.com/intel/neural-compressor - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - ============================================================================ - - Copyright 2016-2019 Intel Corporation - Copyright 2018 YANDEX LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - This distribution includes third party software ("third party programs"). - This third party software, even if included with the distribution of - the Intel software, may be governed by separate license terms, including - without limitation, third party license terms, other Intel software license - terms, and open source software license terms. These separate license terms - govern your use of the third party programs as set forth in the - "THIRD-PARTY-PROGRAMS" file. - -_____ - -FlashAttention, https://github.com/Dao-AILab/flash-attention - -BSD 3-Clause License - -Copyright (c) 2022, the respective contributors, as shown by the AUTHORS file. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -composable_kernel - -https://github.com/ROCmSoftwarePlatform/composable_kernel - -Copyright (c) 2018- , Advanced Micro Devices, Inc. (Chao Liu, Jing Zhang) -Copyright (c) 2019- , Advanced Micro Devices, Inc. (Letao Qin, Qianfeng Zhang, Liang Huang, Shaojie Wang) -Copyright (c) 2022- , Advanced Micro Devices, Inc. (Anthony Chang, Chunyu Lai, Illia Silin, Adam Osewski, Poyen Chen, Jehandad Khan) -Copyright (c) 2019-2021, Advanced Micro Devices, Inc. (Hanwen Chang) -Copyright (c) 2019-2020, Advanced Micro Devices, Inc. (Tejash Shah) -Copyright (c) 2020 , Advanced Micro Devices, Inc. (Xiaoyan Zhou) -Copyright (c) 2021-2022, Advanced Micro Devices, Inc. (Jianfeng Yan) - -SPDX-License-Identifier: MIT -Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -neural-speed - -https://github.com/intel/neural-speed - - Apache License - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - ============================================================================ - - Copyright 2016-2019 Intel Corporation - Copyright 2018 YANDEX LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - This distribution includes third party software ("third party programs"). - This third party software, even if included with the distribution of - the Intel software, may be governed by separate license terms, including - without limitation, third party license terms, other Intel software license - terms, and open source software license terms. These separate license terms - govern your use of the third party programs as set forth in the - "THIRD-PARTY-PROGRAMS" file. diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/VERSION_NUMBER b/inference_core/ort_core/onnxruntime-pkg-aarch64/VERSION_NUMBER deleted file mode 100644 index ec6d649..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/VERSION_NUMBER +++ /dev/null @@ -1 +0,0 @@ -1.18.1 diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/cpu_provider_factory.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/cpu_provider_factory.h deleted file mode 100644 index 2926786..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/cpu_provider_factory.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "onnxruntime_c_api.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \param use_arena zero: false. non-zero: true. - */ -ORT_EXPORT -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena) -ORT_ALL_ARGS_NONNULL; - -#ifdef __cplusplus -} -#endif diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_c_api.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_c_api.h deleted file mode 100644 index 52f496c..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_c_api.h +++ /dev/null @@ -1,4822 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// See docs\c_cxx\README.md on generating the Doxygen documentation from this file - -/** \mainpage ONNX Runtime - * - * ONNX Runtime is a high-performance inference and training graph execution engine for deep learning models. - * - * ONNX Runtime's C, C++ APIs offer an easy to use interface to onboard and execute onnx models. - * - \subpage c_cpp_api "Core C, C++ APIs" - * - \subpage training_c_cpp_api "Training C, C++ APIs for on-device training" - * - * \page c_cpp_api Core C, C++ APIs - *

C

- * - * ::OrtApi - Click here to go to the structure with all C API functions. - * - *

C++

- * - * ::Ort - Click here to go to the namespace holding all of the C++ wrapper classes - * - * It is a set of header only wrapper classes around the C API. The goal is to turn the C style return value error codes into C++ exceptions, and to - * automate memory management through standard C++ RAII principles. - * - * \addtogroup Global - * ONNX Runtime C API - * @{ - */ - -#pragma once -#include -#include -#include -#include - -/** \brief The API version defined in this header - * - * This value is used by some API functions to behave as this version of the header expects. - */ -#define ORT_API_VERSION 18 - -#ifdef __cplusplus -extern "C" { -#endif - -//! @} -// SAL2 Definitions -#ifndef _WIN32 -#define _In_ -#define _In_z_ -#define _In_opt_ -#define _In_opt_z_ -#define _Out_ -#define _Outptr_ -#define _Out_opt_ -#define _Inout_ -#define _Inout_opt_ -#define _Frees_ptr_opt_ -#define _Ret_maybenull_ -#define _Ret_notnull_ -#define _Check_return_ -#define _Outptr_result_maybenull_ -#define _In_reads_(X) -#define _Inout_updates_(X) -#define _Out_writes_(X) -#define _Inout_updates_all_(X) -#define _Out_writes_bytes_all_(X) -#define _Out_writes_all_(X) -#define _Success_(X) -#define _Outptr_result_buffer_maybenull_(X) -#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull)) -#else -#include -#define ORT_ALL_ARGS_NONNULL -#endif - -#ifdef _WIN32 -// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort. -// dllexport is not used, we use a .def file. -#ifdef ORT_DLL_IMPORT -#define ORT_EXPORT __declspec(dllimport) -#else -#define ORT_EXPORT -#endif -#define ORT_API_CALL _stdcall -#define ORT_MUST_USE_RESULT -#define ORTCHAR_T wchar_t -#else -// To make symbols visible on macOS/iOS -#ifdef __APPLE__ -#define ORT_EXPORT __attribute__((visibility("default"))) -#else -#define ORT_EXPORT -#endif -#define ORT_API_CALL -#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) -#define ORTCHAR_T char -#endif - -/// ORTCHAR_T, ORT_TSTR are reserved specifically for path handling. -/// All other strings are UTF-8 encoded, use char and std::string -#ifndef ORT_TSTR -#ifdef _WIN32 -#define ORT_TSTR(X) L##X -// When X is a macro, L##X is not defined. In this case, we need to use ORT_TSTR_ON_MACRO. -#define ORT_TSTR_ON_MACRO(X) L"" X -#else -#define ORT_TSTR(X) X -#define ORT_TSTR_ON_MACRO(X) X -#endif -#endif - -// On Windows, ORT_FILE is a wchar_t version of the __FILE__ macro. -// Otherwise, ORT_FILE is equivalent to __FILE__. -#ifndef ORT_FILE -#define ORT_FILE_INTERNAL(x) ORT_TSTR(x) -#define ORT_FILE ORT_FILE_INTERNAL(__FILE__) -#endif - -// Any pointer marked with _In_ or _Out_, cannot be NULL. - -// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation -// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. -// for ReleaseXXX(...) functions, they can accept NULL pointer. - -#ifdef __cplusplus -// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept. -// Such complex condition is needed because compilers set __cplusplus value differently. -#ifndef __has_feature -#define __has_feature(x) 0 -#endif -#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept))) -#define NO_EXCEPTION noexcept -#else -#define NO_EXCEPTION throw() -#endif -#else -#define NO_EXCEPTION -#endif - -// __VA_ARGS__ on Windows and Linux are different -#define ORT_API(RETURN_TYPE, NAME, ...) RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION - -#define ORT_API_STATUS(NAME, ...) \ - _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) \ - NO_EXCEPTION ORT_MUST_USE_RESULT - -// XXX: Unfortunately, SAL annotations are known to not work with function pointers -#define ORT_API2_STATUS(NAME, ...) \ - _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT - -// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT -#define ORT_API_STATUS_IMPL(NAME, ...) \ - _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION - -#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input) - -#ifdef __DOXYGEN__ -#undef ORT_API_STATUS -#define ORT_API_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) -#undef ORT_API2_STATUS -#define ORT_API2_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) -#undef ORT_CLASS_RELEASE -#define ORT_CLASS_RELEASE(X) void Release##X(Ort##X* input) -#undef NO_EXCEPTION -#define NO_EXCEPTION -#endif -/** \addtogroup Global - * ONNX Runtime C API - * @{ - */ - -/** Copied from TensorProto::DataType - * Currently, Ort doesn't support complex64, complex128 - */ -typedef enum ONNXTensorElementDataType { - ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string - ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, - ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16, // Non-IEEE floating-point format based on IEEE754 single-precision - // float 8 types were introduced in onnx 1.14, see https://onnx.ai/onnx/technical/float8.html - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ // Non-IEEE floating-point format based on IEEE754 single-precision -} ONNXTensorElementDataType; - -// Synced with onnx TypeProto oneof -typedef enum ONNXType { - ONNX_TYPE_UNKNOWN, - ONNX_TYPE_TENSOR, - ONNX_TYPE_SEQUENCE, - ONNX_TYPE_MAP, - ONNX_TYPE_OPAQUE, - ONNX_TYPE_SPARSETENSOR, - ONNX_TYPE_OPTIONAL -} ONNXType; - -// These types are synced with internal -// SparseFormatFlags -typedef enum OrtSparseFormat { - ORT_SPARSE_UNDEFINED = 0, - ORT_SPARSE_COO = 0x1, - ORT_SPARSE_CSRC = 0x2, - ORT_SPARSE_BLOCK_SPARSE = 0x4 -} OrtSparseFormat; - -// Enum allows to query sparse tensor indices -enum OrtSparseIndicesFormat { - ORT_SPARSE_COO_INDICES, - ORT_SPARSE_CSR_INNER_INDICES, - ORT_SPARSE_CSR_OUTER_INDICES, - ORT_SPARSE_BLOCK_SPARSE_INDICES -}; - -/** \brief Logging severity levels - * - * In typical API usage, specifying a logging severity level specifies the minimum severity of log messages to show. - */ -typedef enum OrtLoggingLevel { - ORT_LOGGING_LEVEL_VERBOSE, ///< Verbose informational messages (least severe). - ORT_LOGGING_LEVEL_INFO, ///< Informational messages. - ORT_LOGGING_LEVEL_WARNING, ///< Warning messages. - ORT_LOGGING_LEVEL_ERROR, ///< Error messages. - ORT_LOGGING_LEVEL_FATAL, ///< Fatal error messages (most severe). -} OrtLoggingLevel; - -typedef enum OrtErrorCode { - ORT_OK, - ORT_FAIL, - ORT_INVALID_ARGUMENT, - ORT_NO_SUCHFILE, - ORT_NO_MODEL, - ORT_ENGINE_ERROR, - ORT_RUNTIME_EXCEPTION, - ORT_INVALID_PROTOBUF, - ORT_MODEL_LOADED, - ORT_NOT_IMPLEMENTED, - ORT_INVALID_GRAPH, - ORT_EP_FAIL, -} OrtErrorCode; - -typedef enum OrtOpAttrType { - ORT_OP_ATTR_UNDEFINED = 0, - ORT_OP_ATTR_INT, - ORT_OP_ATTR_INTS, - ORT_OP_ATTR_FLOAT, - ORT_OP_ATTR_FLOATS, - ORT_OP_ATTR_STRING, - ORT_OP_ATTR_STRINGS, -} OrtOpAttrType; - -//! @} -#define ORT_RUNTIME_CLASS(X) \ - struct Ort##X; \ - typedef struct Ort##X Ort##X - -/** \addtogroup Global - * ONNX Runtime C API - * @{ - */ -// The actual types defined have an Ort prefix -ORT_RUNTIME_CLASS(Env); -ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success -ORT_RUNTIME_CLASS(MemoryInfo); -ORT_RUNTIME_CLASS(IoBinding); -ORT_RUNTIME_CLASS(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) -ORT_RUNTIME_CLASS(Value); -ORT_RUNTIME_CLASS(RunOptions); -ORT_RUNTIME_CLASS(TypeInfo); -ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); -ORT_RUNTIME_CLASS(MapTypeInfo); -ORT_RUNTIME_CLASS(SequenceTypeInfo); -ORT_RUNTIME_CLASS(OptionalTypeInfo); -ORT_RUNTIME_CLASS(SessionOptions); -ORT_RUNTIME_CLASS(CustomOpDomain); -ORT_RUNTIME_CLASS(ModelMetadata); -ORT_RUNTIME_CLASS(ThreadPoolParams); -ORT_RUNTIME_CLASS(ThreadingOptions); -ORT_RUNTIME_CLASS(ArenaCfg); -ORT_RUNTIME_CLASS(PrepackedWeightsContainer); -ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2); -ORT_RUNTIME_CLASS(CUDAProviderOptionsV2); -ORT_RUNTIME_CLASS(CANNProviderOptions); -ORT_RUNTIME_CLASS(DnnlProviderOptions); -ORT_RUNTIME_CLASS(Op); -ORT_RUNTIME_CLASS(OpAttr); -ORT_RUNTIME_CLASS(Logger); -ORT_RUNTIME_CLASS(ShapeInferContext); - -#ifdef _WIN32 -typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; -#else -typedef OrtStatus* OrtStatusPtr; -#endif - -/** \brief Memory allocation interface - * - * Structure of function pointers that defines a memory allocator. This can be created and filled in by the user for custom allocators. - * - * When an allocator is passed to any function, be sure that the allocator object is not destroyed until the last allocated object using it is freed. - */ -typedef struct OrtAllocator { - uint32_t version; ///< Must be initialized to ORT_API_VERSION - void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); ///< Returns a pointer to an allocated block of `size` bytes - void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); ///< Free a block of memory previously allocated with OrtAllocator::Alloc - const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); ///< Return a pointer to an ::OrtMemoryInfo that describes this allocator - /** - * @brief Optional allocation function to use for memory allocations made during session initialization. - * Use this function if you want to separate allocations made by ORT during Run() calls from - * those made during session initialization. This allows for separate memory management strategies for these allocations. - */ - void*(ORT_API_CALL* Reserve)(struct OrtAllocator* this_, size_t size); ///< Returns a pointer to an allocated block of `size` bytes -} OrtAllocator; - -typedef void(ORT_API_CALL* OrtLoggingFunction)( - void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, - const char* message); - -/** \brief Graph optimization level - * - * Refer to https://www.onnxruntime.ai/docs/performance/graph-optimizations.html#graph-optimization-levels - * for an in-depth understanding of the Graph Optimization Levels. - */ -typedef enum GraphOptimizationLevel { - ORT_DISABLE_ALL = 0, - ORT_ENABLE_BASIC = 1, - ORT_ENABLE_EXTENDED = 2, - ORT_ENABLE_ALL = 99 -} GraphOptimizationLevel; - -typedef enum ExecutionMode { - ORT_SEQUENTIAL = 0, - ORT_PARALLEL = 1, -} ExecutionMode; - -/** \brief Language projection identifiers - * /see OrtApi::SetLanguageProjection - */ -typedef enum OrtLanguageProjection { - ORT_PROJECTION_C = 0, - ORT_PROJECTION_CPLUSPLUS = 1, - ORT_PROJECTION_CSHARP = 2, - ORT_PROJECTION_PYTHON = 3, - ORT_PROJECTION_JAVA = 4, - ORT_PROJECTION_WINML = 5, - ORT_PROJECTION_NODEJS = 6, -} OrtLanguageProjection; - -struct OrtKernelInfo; -typedef struct OrtKernelInfo OrtKernelInfo; -struct OrtKernelContext; -typedef struct OrtKernelContext OrtKernelContext; -struct OrtCustomOp; -typedef struct OrtCustomOp OrtCustomOp; - -typedef enum OrtAllocatorType { - OrtInvalidAllocator = -1, - OrtDeviceAllocator = 0, - OrtArenaAllocator = 1 -} OrtAllocatorType; - -/** \brief Memory types for allocated memory, execution provider specific types should be extended in each provider. - */ -// Whenever this struct is updated, please also update the MakeKey function in onnxruntime / core / framework / execution_provider.cc -typedef enum OrtMemType { - OrtMemTypeCPUInput = -2, ///< Any CPU memory used by non-CPU execution provider - OrtMemTypeCPUOutput = -1, ///< CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeCPU = OrtMemTypeCPUOutput, ///< Temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeDefault = 0, ///< The default allocator for execution provider -} OrtMemType; - -/** \brief This mimics OrtDevice type constants so they can be returned in the API - */ -typedef enum OrtMemoryInfoDeviceType { - OrtMemoryInfoDeviceType_CPU = 0, - OrtMemoryInfoDeviceType_GPU = 1, - OrtMemoryInfoDeviceType_FPGA = 2 -} OrtMemoryInfoDeviceType; - -/** \brief Algorithm to use for cuDNN Convolution Op - */ -typedef enum OrtCudnnConvAlgoSearch { - OrtCudnnConvAlgoSearchExhaustive, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx - OrtCudnnConvAlgoSearchHeuristic, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7 - OrtCudnnConvAlgoSearchDefault, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM -} OrtCudnnConvAlgoSearch; - -/** \brief CUDA Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_CUDA - */ -typedef struct OrtCUDAProviderOptions { -#ifdef __cplusplus - OrtCUDAProviderOptions() - : device_id{}, - cudnn_conv_algo_search{OrtCudnnConvAlgoSearchExhaustive}, - gpu_mem_limit{SIZE_MAX}, - arena_extend_strategy{}, - do_copy_in_default_stream{1}, - has_user_compute_stream{}, - user_compute_stream{}, - default_memory_arena_cfg{}, - tunable_op_enable{false}, - tunable_op_tuning_enable{false}, - tunable_op_max_tuning_duration_ms{} {} -#endif - - /** \brief CUDA device Id - * Defaults to 0. - */ - int device_id; - - /** \brief CUDA Convolution algorithm search configuration. - * See enum OrtCudnnConvAlgoSearch for more details. - * Defaults to OrtCudnnConvAlgoSearchExhaustive. - */ - OrtCudnnConvAlgoSearch cudnn_conv_algo_search; - - /** \brief CUDA memory limit (To use all possible memory pass in maximum size_t) - * Defaults to SIZE_MAX. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - size_t gpu_mem_limit; - - /** \brief Strategy used to grow the memory arena - * 0 = kNextPowerOfTwo
- * 1 = kSameAsRequested
- * Defaults to 0. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - int arena_extend_strategy; - - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP - * 0 = Use separate streams for copying and compute. - * 1 = Use the same stream for copying and compute. - * Defaults to 1. - * WARNING: Setting this to 0 may result in data races for some models. - * Please see issue #4829 for more details. - */ - int do_copy_in_default_stream; - - /** \brief Flag indicating if there is a user provided compute stream - * Defaults to 0. - */ - int has_user_compute_stream; - - /** \brief User provided compute stream. - * If provided, please set `has_user_compute_stream` to 1. - */ - void* user_compute_stream; - - /** \brief CUDA memory arena configuration parameters - */ - OrtArenaCfg* default_memory_arena_cfg; - - /** \brief Enable TunableOp for using. - * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_CUDA_TUNABLE_OP_ENABLE. - */ - int tunable_op_enable; - - /** \brief Enable TunableOp for tuning. - * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_CUDA_TUNABLE_OP_TUNING_ENABLE. - */ - int tunable_op_tuning_enable; - - /** \brief Max tuning duration time limit for each instance of TunableOp. - * Defaults to 0 to disable the limit. - */ - int tunable_op_max_tuning_duration_ms; - -} OrtCUDAProviderOptions; - -/** \brief ROCM Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_ROCM - */ -typedef struct OrtROCMProviderOptions { -#ifdef __cplusplus - OrtROCMProviderOptions() - : device_id{}, - miopen_conv_exhaustive_search{0}, - gpu_mem_limit{SIZE_MAX}, - arena_extend_strategy{}, - do_copy_in_default_stream{1}, - has_user_compute_stream{}, - user_compute_stream{}, - default_memory_arena_cfg{}, - enable_hip_graph{false}, - tunable_op_enable{false}, - tunable_op_tuning_enable{false}, - tunable_op_max_tuning_duration_ms{} {} -#endif - - /** \brief ROCM device Id - * Defaults to 0. - */ - int device_id; - - /** \brief ROCM MIOpen Convolution algorithm exaustive search option. - * Defaults to 0 (false). - */ - int miopen_conv_exhaustive_search; - - /** \brief ROCM memory limit (To use all possible memory pass in maximum size_t) - * Defaults to SIZE_MAX. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - size_t gpu_mem_limit; - - /** \brief Strategy used to grow the memory arena - * 0 = kNextPowerOfTwo
- * 1 = kSameAsRequested
- * Defaults to 0. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - int arena_extend_strategy; - - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the ROCM EP - * 0 = Use separate streams for copying and compute. - * 1 = Use the same stream for copying and compute. - * Defaults to 1. - * WARNING: Setting this to 0 may result in data races for some models. - * Please see issue #4829 for more details. - */ - int do_copy_in_default_stream; - - /** \brief Flag indicating if there is a user provided compute stream - * Defaults to 0. - */ - int has_user_compute_stream; - - /** \brief User provided compute stream. - * If provided, please set `has_user_compute_stream` to 1. - */ - void* user_compute_stream; - - /** \brief ROCM memory arena configuration parameters - */ - OrtArenaCfg* default_memory_arena_cfg; - - int enable_hip_graph; - - /** \brief Enable TunableOp for using. - * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_ROCM_TUNABLE_OP_ENABLE. - */ - int tunable_op_enable; - - /** \brief Enable TunableOp for tuning. - * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_ROCM_TUNABLE_OP_TUNING_ENABLE. - */ - int tunable_op_tuning_enable; - - /** \brief Max tuning duration time limit for each instance of TunableOp. - * Defaults to 0 to disable the limit. - */ - int tunable_op_max_tuning_duration_ms; - -} OrtROCMProviderOptions; - -/** \brief TensorRT Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - */ -typedef struct OrtTensorRTProviderOptions { - int device_id; ///< CUDA device id (0 = default device) - int has_user_compute_stream; // indicator of user specified CUDA compute stream. - void* user_compute_stream; // user specified CUDA compute stream. - int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability - int trt_min_subgraph_size; // minimum size of TensorRT subgraphs - size_t trt_max_workspace_size; // maximum workspace size for TensorRT. - int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true - int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true - const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name. - int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true - int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true - int trt_dla_core; // DLA core number. Default 0 - int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true - int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true - const char* trt_engine_cache_path; // specify engine cache path - int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true - const char* trt_engine_decryption_lib_path; // specify engine decryption library path - int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true - // This is the legacy struct and don't add new fields here. - // For new field that can be represented by string, please add it in include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h - // For non-string field, need to create a new separate api to handle it. -} OrtTensorRTProviderOptions; - -/** \brief MIGraphX Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX - */ -typedef struct OrtMIGraphXProviderOptions { - int device_id; // hip device id. - int migraphx_fp16_enable; // MIGraphX FP16 precision. Default 0 = false, nonzero = true - int migraphx_int8_enable; // MIGraphX INT8 precision. Default 0 = false, nonzero = true - int migraphx_use_native_calibration_table; // MIGraphx INT8 cal table. Default 0 = false, noznero = true - const char* migraphx_int8_calibration_table_name; // MIGraphx INT8 calibration table name -} OrtMIGraphXProviderOptions; - -/** \brief OpenVINO Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO - */ -typedef struct OrtOpenVINOProviderOptions { -#ifdef __cplusplus - OrtOpenVINOProviderOptions() : device_type{}, - enable_npu_fast_compile{}, - device_id{}, - num_of_threads{}, - cache_dir{}, - context{}, - enable_opencl_throttling{}, - enable_dynamic_shapes{} {} -#endif - /** \brief Device type string - * - * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" - */ - const char* device_type; - unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled - const char* device_id; - size_t num_of_threads; ///< 0 = Use default number of threads - const char* cache_dir; // path is set to empty by default - void* context; - unsigned char enable_opencl_throttling; ///< 0 = disabled, nonzero = enabled - unsigned char enable_dynamic_shapes; ///< 0 = disabled, nonzero = enabled -} OrtOpenVINOProviderOptions; - -struct OrtApi; -typedef struct OrtApi OrtApi; - -struct OrtTrainingApi; -typedef struct OrtTrainingApi OrtTrainingApi; - -/** \brief The helper interface to get the right version of OrtApi - * - * Get a pointer to this structure through ::OrtGetApiBase - */ -struct OrtApiBase { - /** \brief Get a pointer to the requested version of the ::OrtApi - * - * \param[in] version Must be ::ORT_API_VERSION - * \return The ::OrtApi for the version requested, nullptr will be returned if this version is unsupported, for example when using a runtime - * older than the version created with this header file. - * - * One can call GetVersionString() to get the version of the Onnxruntime library for logging - * and error reporting purposes. - */ - const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; - - /** \brief Returns a null terminated string of the version of the Onnxruntime library (eg: "1.8.1") - * - * \return UTF-8 encoded version string. Do not deallocate the returned buffer. - */ - const char*(ORT_API_CALL* GetVersionString)(void)NO_EXCEPTION; -}; - -typedef struct OrtApiBase OrtApiBase; - -/** \brief The Onnxruntime library's entry point to access the C API - * - * Call this to get the a pointer to an ::OrtApiBase - */ -ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION; - -/** \brief Thread work loop function - * - * Onnxruntime will provide the working loop on custom thread creation - * Argument is an onnxruntime built-in type which will be provided when thread pool calls OrtCustomCreateThreadFn - */ -typedef void (*OrtThreadWorkerFn)(void* ort_worker_fn_param); - -typedef const struct OrtCustomHandleType { - char __place_holder; -}* OrtCustomThreadHandle; - -/** \brief Ort custom thread creation function - * - * The function should return a thread handle to be used in onnxruntime thread pools - * Onnxruntime will throw exception on return value of nullptr or 0, indicating that the function failed to create a thread - */ -typedef OrtCustomThreadHandle (*OrtCustomCreateThreadFn)(void* ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void* ort_worker_fn_param); - -/** \brief Custom thread join function - * - * Onnxruntime thread pool destructor will call the function to join a custom thread. - * Argument ort_custom_thread_handle is the value returned by OrtCustomCreateThreadFn - */ -typedef void (*OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle); - -typedef OrtStatus*(ORT_API_CALL* RegisterCustomOpsFn)(OrtSessionOptions* options, const OrtApiBase* api); - -/** \brief Callback function for RunAsync - * - * \param[in] user_data User specific data that passed back to the callback - * \param[out] outputs On succeed, outputs host inference results, on error, the value will be nullptr - * \param[out] num_outputs Number of outputs, on error, the value will be zero - * \param[out] status On error, status will provide details - */ -typedef void (*RunAsyncCallbackFn)(void* user_data, OrtValue** outputs, size_t num_outputs, OrtStatusPtr status); - -/** \brief The C API - * - * All C API functions are defined inside this structure as pointers to functions. - * Call OrtApiBase::GetApi to get a pointer to it - * - * \nosubgrouping - */ -struct OrtApi { - /// \name OrtStatus - /// @{ - - /** - * \brief Create an OrtStatus from a null terminated string - * - * \param[in] code - * \param[in] msg A null-terminated string. Its contents will be copied. - * \return A new OrtStatus object, must be destroyed with OrtApi::ReleaseStatus - */ - OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Get OrtErrorCode from OrtStatus - * - * \param[in] status - * \return OrtErrorCode that \p status was created with - */ - OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Get error string from OrtStatus - * - * \param[in] status - * \return The error message inside the `status`. Do not free the returned value. - */ - const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an OrtEnv - * - * \note Invoking this function will return the same instance of the environment as that returned by a previous call - * to another env creation function; all arguments to this function will be ignored. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnv, OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); - - /** \brief Create an OrtEnv - * - * \note Invoking this function will return the same instance of the environment as that returned by a previous call - * to another env creation function; all arguments to this function will be ignored. If you want to provide your - * own logging function, consider setting it using the SetUserLoggingFunction API instead. - * \param[in] logging_function A pointer to a logging function. - * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `logging_function`. This parameter is optional. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithCustomLogger, _In_ OrtLoggingFunction logging_function, _In_opt_ void* logger_param, - _In_ OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); - - /** \brief Enable Telemetry - * - * \note Telemetry events are on by default since they are lightweight - * \param[in] env - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env); - /** \brief Disable Telemetry - * - * \see OrtApi::EnableTelemetryEvents - * \param[in] env - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Create an OrtSession from a model file - * - * \param[in] env - * \param[in] model_path - * \param[in] options - * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - // TODO: document the path separator convention? '/' vs '\' - // TODO: should specify the access characteristics of model_path. Is this read only during the - // execution of CreateSession, or does the OrtSession retain a handle to the file/directory - // and continue to access throughout the OrtSession lifetime? - // What sort of access is needed to model_path : read or read/write? - ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); - - /** \brief Create an OrtSession from memory - * - * \param[in] env - * \param[in] model_data - * \param[in] model_data_length - * \param[in] options - * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); - - /** \brief Run the model in an ::OrtSession - * - * Will not return until the model run has completed. Multiple threads might be used to run the model based on - * the options in the ::OrtSession and settings used when creating the ::OrtEnv - * - * \param[in] session - * \param[in] run_options If nullptr, will use a default ::OrtRunOptions - * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names - * \param[in] inputs Array of ::OrtValue%s of the input values - * \param[in] input_len Number of elements in the input_names and inputs arrays - * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names - * \param[in] output_names_len Number of elements in the output_names and outputs array - * \param[out] outputs Array of ::OrtValue%s that the outputs are stored in. This can also be - * an array of nullptr values, in this case ::OrtValue objects will be allocated and pointers - * to them will be set into the `outputs` array. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(Run, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, - _In_reads_(input_len) const char* const* input_names, - _In_reads_(input_len) const OrtValue* const* inputs, size_t input_len, - _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, - _Inout_updates_all_(output_names_len) OrtValue** outputs); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Create an ::OrtSessionOptions object - * - * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these - * functions to enable them in the session:
- * OrtSessionOptionsAppendExecutionProvider_CPU
- * OrtSessionOptionsAppendExecutionProvider_CUDA
- * OrtSessionOptionsAppendExecutionProvider_(remaining providers...)
- * The order they are called indicates the preference order as well. In other words call this method - * on your most preferred execution provider first followed by the less preferred ones. - * If none are called Ort will use its internal CPU execution provider. - * - * \param[out] options The newly created OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options); - - /** \brief Set filepath to save optimized model after graph level transformations - * - * \param[in] options - * \param[in] optimized_model_filepath - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, - _In_ const ORTCHAR_T* optimized_model_filepath); - - /** \brief Create a copy of an existing ::OrtSessionOptions - * - * \param[in] in_options OrtSessionOptions to copy - * \param[out] out_options Returned newly created ::OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options, - _Outptr_ OrtSessionOptions** out_options); - - /** \brief Set execution mode - * - * Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model - * has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance. - * See [docs/ONNX_Runtime_Perf_Tuning.md] for more details. - * - * \param[in] options - * \param[in] execution_mode - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode); - - /** \brief Enable profiling for a session - * - * \param[in] options - * \param[in] profile_file_prefix - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix); - - /** \brief Disable profiling for a session - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options); - - /** \brief Enable the memory pattern optimization - * - * The idea is if the input shapes are the same, we could trace the internal memory allocation - * and generate a memory pattern for future request. So next time we could just do one allocation - * with a big chunk for all the internal memory allocation. - * \note Memory pattern optimization is only available when Sequential Execution mode is enabled (see OrtApi::SetSessionExecutionMode) - * - * \see OrtApi::DisableMemPattern - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options); - - /** \brief Disable the memory pattern optimization - * - * \see OrtApi::EnableMemPattern - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options); - - /** \brief Enable the memory arena on CPU - * - * Arena may pre-allocate memory for future usage. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options); - - /** \brief Disable the memory arena on CPU - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options); - - /** \brief Set session log id - * - * \param[in] options - * \param[in] logid The log identifier. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid); - - /** \brief Set session log verbosity level - * - * Applies to session load, initialization, etc - * - * \param[in] options - * \param[in] session_log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level); - - /** \brief Set session log severity level - * - * \param[in] options - * \param[in] session_log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level); - - /** \brief Set the optimization level to apply when loading a graph - * - * Please see https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html for an in-depth explanation - * \param[in,out] options The session options object - * \param[in] graph_optimization_level The optimization level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, - GraphOptimizationLevel graph_optimization_level); - - /** \brief Sets the number of threads used to parallelize the execution within nodes - * - * When running a single node operation, ex. add, this sets the maximum number of threads to use. - * - * \note If built with OpenMP, this has no effect on the number of threads used. In this case - * use the OpenMP env variables to configure the number of intra op num threads. - * - * \param[in] options - * \param[in] intra_op_num_threads Number of threads to use
- * A value of 0 will use the default number of threads
- * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads); - - /** \brief Sets the number of threads used to parallelize the execution of the graph - * - * If nodes can be run in parallel, this sets the maximum number of threads to use to run them in parallel. - * - * \note If sequential execution is enabled this value is ignored, it acts as if it was set to 1. - * - * \param[in] options - * \param[in] inter_op_num_threads Number of threads to use
- * A value of 0 will use the default number of threads
- * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads); - - /// @} - /// \name OrtCustomOpDomain - /// @{ - - /** \brief Create a custom op domain - * - * \param[in] domain - * \param[out] out Newly created domain. Must be freed with OrtApi::ReleaseCustomOpDomain - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out); - - /** \brief Add a custom op to a custom op domain - * - * \note The OrtCustomOp* pointer must remain valid until the ::OrtCustomOpDomain using it is released - * - * \param[in] custom_op_domain - * \param[in] op - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Add custom op domain to a session options - * - * \note The OrtCustomOpDomain* must not be deleted until all sessions using it are released - * - * \param[in] options - * \param[in] custom_op_domain - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain); - - /** \deprecated Use OrtApi::RegisterCustomOpsLibrary_V2. - * - * Registers custom ops from a shared library. - * - * Loads a shared library (dll on windows, so on linux, etc) named 'library_path' and looks for this entry point: - * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); - * It then passes in the provided session options to this function along with the api base. - * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in - * session options are destroyed, or if an error occurs and it is non null. - * - * \param[in] options - * \param[in] library_path - * \param[out] library_handle OS specific handle to the loaded library (Use FreeLibrary on Windows, dlclose on Linux, etc.. to unload) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path, _Outptr_ void** library_handle); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Get input count for a session - * - * This number must also match the number of inputs passed to OrtApi::Run - * - * \see OrtApi::SessionGetInputTypeInfo, OrtApi::SessionGetInputName, OrtApi::Session - * - * \param[in] session - * \param[out] out Number of inputs - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get output count for a session - * - * This number must also match the number of outputs returned by OrtApi::Run - * - * \see OrtApi::SessionGetOutputTypeInfo, OrtApi::SessionGetOutputName, OrtApi::Session - * - * \param[in] session - * \param[out] out Number of outputs - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get overridable initializer count - * - * \see OrtApi::SessionGetOverridableInitializerTypeInfo, OrtApi::SessionGetOverridableInitializerName - * - * \param[in] session - * \param[in] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get input type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get output type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get overridable initializer type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get input name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get output name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get overridable initializer name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* session, size_t index, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /// @} - /// \name OrtRunOptions - /// @{ - - /** \brief Create an OrtRunOptions - * - * \param[out] out Returned newly created ::OrtRunOptions. Must be freed with OrtApi::ReleaseRunOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out); - - /** \brief Set per-run log verbosity level - * - * \see OrtApi::RunOptionsGetRunLogVerbosityLevel - * - * \param[in] options - * \param[in] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int log_verbosity_level); - - /** \brief Set per-run log severity level - * - * \see OrtApi::RunOptionsGetRunLogSeverityLevel - * - * \param[in] options - * \param[in] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - */ - ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int log_severity_level); - - /** \brief Set per-run tag - * - * This is used in a per-run log identifier. - * - * \see OrtApi::RunOptionsGetRunTag - * - * \param[in] options - * \param[in] run_tag The run tag. - */ - ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions* options, _In_ const char* run_tag); - - /** \brief Get per-run log verbosity level - * - * \see OrtApi::RunOptionsSetRunLogVerbosityLevel - * - * \param[in] options - * \param[out] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, - _Out_ int* log_verbosity_level); - - /** \brief Get per-run log severity level - * - * \see OrtApi::RunOptionsSetRunLogSeverityLevel - * - * \param[in] options - * \param[out] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - */ - ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* log_severity_level); - - /** \brief Get per-run tag - * - * This is used in a per-run log identifier. - * - * \see OrtApi::RunOptionsSetRunTag - * - * \param[in] options - * \param[out] run_tag The run tag. - * Do not free this value, it is owned by `options`. It will be invalidated if the run tag - * changes (i.e., with OrtApi::RunOptionsSetRunTag) or `options` is freed. - */ - ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions* options, _Out_ const char** run_tag); - - /** \brief Set terminate flag - * - * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options); - - /** \brief Clears the terminate flag - * - * Used so the OrtRunOptions instance can be used in a new OrtApi::Run call without it instantly terminating - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Create a tensor - * - * Create a tensor using a supplied ::OrtAllocator - * - * \param[in] allocator - * \param[in] shape Pointer to the tensor shape dimensions. - * \param[in] shape_len The number of tensor shape dimensions. - * \param[in] type - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** \brief Create a tensor backed by a user supplied buffer - * - * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. - * p_data is owned by caller. ReleaseValue won't release p_data. - * - * \param[in] info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param[in] p_data Pointer to the data buffer. - * \param[in] p_data_len The number of bytes in the data buffer. - * \param[in] shape Pointer to the tensor shape dimensions. - * \param[in] shape_len The number of tensor shape dimensions. - * \param[in] type The data type. - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, - size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type, - _Outptr_ OrtValue** out); - - /** \brief Return if an ::OrtValue is a tensor type - * - * \param[in] value A tensor type (string tensors are not supported) - * \param[out] out Set to 1 iff ::OrtValue is a tensor, 0 otherwise - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out); - - /** \brief Get a pointer to the raw data inside a tensor - * - * Used to read/write/modify the internal tensor data directly. - * \note The returned pointer is valid until the \p value is destroyed. - * - * \param[in] value A tensor type (string tensors are not supported) - * \param[out] out Filled in with a pointer to the internal storage - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorMutableData, _In_ OrtValue* value, _Outptr_ void** out); - - /** \brief Set all strings at once in a string tensor - * - * \param[in,out] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[in] s An array of strings. Each string in this array must be null terminated. - * \param[in] s_len Count of strings in s (Must match the size of \p value's tensor shape) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len); - - /** \brief Get total byte length for all strings in a string tensor - * - * Typically used with OrtApi::GetStringTensorContent - * - * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[out] len Total byte length of all strings (does not include trailing nulls) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); - - /** \brief Get all strings from a string tensor - * - * An example of the results:
- * Given \p value is a string tensor with the strings { "This" "is" "a" "test" }
- * \p s must have a size of 11 bytes
- * \p offsets must have 4 elements
- * After the call, these values will be filled in:
- * \p s will contain "Thisisatest"
- * \p offsets will contain { 0, 4, 6, 7 }
- * The length of the last string is just s_len - offsets[last] - * - * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[in] s Buffer to sequentially write all tensor strings to. Each string is NOT null-terminated. - * \param[in] s_len Number of bytes of buffer pointed to by \p s (Get it from OrtApi::GetStringTensorDataLength) - * \param[out] offsets Array of start offsets into the strings written to \p s - * \param[in] offsets_len Number of elements in offsets - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s, - size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len); - - /// @} - /// \name OrtTypeInfo - /// @{ - - /** \brief Get ::OrtTensorTypeAndShapeInfo from an ::OrtTypeInfo - * - * \param[in] type_info - * \param[out] out Do not free this value, it will be valid until type_info is freed. - * If type_info does not represent tensor, this value will be set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out); - - /** \brief Get ::ONNXType from ::OrtTypeInfo - * - * \param[in] type_info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ enum ONNXType* out); - - /// @} - /// \name OrtTensorTypeAndShapeInfo - /// @{ - - /** \brief Create an ::OrtTensorTypeAndShapeInfo object - * - * \param[out] out Returns newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Set element type in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] type - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo* info, enum ONNXTensorElementDataType type); - - /** \brief Set shape information in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] dim_values Array with `dim_count` elements. Can contain negative values. - * \param[in] dim_count Number of elements in `dim_values` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); - - /** \brief Get element type in ::OrtTensorTypeAndShapeInfo - * - * \see OrtApi::SetTensorElementType - * - * \param[in] info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo* info, - _Out_ enum ONNXTensorElementDataType* out); - - /** \brief Get dimension count in ::OrtTensorTypeAndShapeInfo - * - * \see OrtApi::GetDimensions - * - * \param[in] info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); - - /** \brief Get dimensions in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[out] dim_values Array with `dim_values_length` elements. On return, filled with the dimensions stored in the ::OrtTensorTypeAndShapeInfo - * \param[in] dim_values_length Number of elements in `dim_values`. Use OrtApi::GetDimensionsCount to get this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, - size_t dim_values_length); - - /** \brief Get symbolic dimension names in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] dim_params Array with `dim_params_length` elements. On return filled with pointers to null terminated strings of the dimension names - * \param[in] dim_params_length Number of elements in `dim_params`. Use OrtApi::GetDimensionsCount to get this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, - _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length); - - /** \brief Get total number of elements in a tensor shape from an ::OrtTensorTypeAndShapeInfo - * - * Return the number of elements specified by the tensor shape (all dimensions multiplied by each other). - * For 0 dimensions, 1 is returned. If any dimension is less than 0, the result is always -1. - * - * Examples:
- * [] = 1
- * [1,3,4] = 12
- * [2,0,4] = 0
- * [-1,3,4] = -1
- * - * \param[in] info - * \param[out] out Number of elements - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Get type and shape information from a tensor ::OrtValue - * - * \param[in] value Must be a tensor (not a map/sequence/etc) or will return failure - * \param[out] out Newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Get type information of an OrtValue - * - * \param[in] value - * \param[out] out Newly created ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out); - - /** \brief Get ONNXType of an ::OrtValue - * - * \param[in] value - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out); - - /// @} - /// \name OrtMemoryInfo - /// @{ - - /** \brief Create an ::OrtMemoryInfo - * - * \param[in] name - * \param[in] type - * \param[in] id - * \param[in] mem_type - * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtAPi::ReleaseMemoryInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name, enum OrtAllocatorType type, int id, - enum OrtMemType mem_type, _Outptr_ OrtMemoryInfo** out); - - /** \brief Create an ::OrtMemoryInfo for CPU memory - * - * Special case version of OrtApi::CreateMemoryInfo for CPU based memory. Same as using OrtApi::CreateMemoryInfo with name = "Cpu" and id = 0. - * - * \param[in] type - * \param[in] mem_type - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type, - _Outptr_ OrtMemoryInfo** out); - - /** \brief Compare ::OrtMemoryInfo objects for equality - * - * Compares all settings of each ::OrtMemoryInfo for equality - * - * \param[in] info1 - * \param[in] info2 - * \param[out] out Set to 0 if equal, -1 if not equal - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out); - - /** \brief Get name from ::OrtMemoryInfo - * - * \param[in] ptr - * \param[out] out Writes null terminated string to this pointer. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtMemoryInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out); - - /** \brief Get the id from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out); - - /** \brief Get the ::OrtMemType from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out); - - /** \brief Get the ::OrtAllocatorType from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out); - - /// @} - /// \name OrtAllocator - /// @{ - - /// \brief Calls OrtAllocator::Alloc function - ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ort_allocator, size_t size, _Outptr_ void** out); - /// \brief Calls OrtAllocator::Free function - ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ort_allocator, void* p); - /// \brief Calls OrtAllocator::Info function - ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ort_allocator, _Outptr_ const struct OrtMemoryInfo** out); - - /** \brief Get the default allocator - * - * The default allocator is a CPU based, non-arena. Always returns the same pointer to the same default allocator. - * - * \param[out] out Returned value should NOT be freed - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Override session symbolic dimensions - * - * Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable - * optimizations that can take advantage of fixed values (such as memory planning, etc) - * - * \param[in] options - * \param[in] dim_denotation - * \param[in] dim_value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation, - _In_ int64_t dim_value); - - /// @} - /// \name OrtValue - /// @{ - - /* Internal information (not seen in Doxygen) - * - * APIs to support non-tensor types - map and sequence. - * Currently only the following types are supported - * Note: the following types should be kept in sync with data_types.h - * Map types - * ========= - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * - * Sequence types - * ============== - * std::vector - * std::vector - * std::vector - * std::vector - * std::vector> - * std::vector - */ - - /** \brief Get non tensor data from an ::OrtValue - * - * If `value` is of type ONNX_TYPE_MAP, you need to retrieve the keys and values - * separately. Use index=0 to retrieve keys and index=1 to retrieve values. - * If `value` is of type ONNX_TYPE_SEQUENCE, use index to retrieve the index'th element - * of the sequence. - * - * \param[in] value - * \param[in] index See above for usage based on `value` type - * \param[in] allocator Allocator used to allocate ::OrtValue - * \param[out] out Created ::OrtValue that holds the element requested. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator, - _Outptr_ OrtValue** out); - - /** \brief Get non tensor value count from an ::OrtValue - * - * If `value` is of type ONNX_TYPE_MAP 2 will always be returned. For ONNX_TYPE_SEQUENCE - * the number of elements in the sequence will be returned - * - * \param[in] value - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out); - - /** \brief Create a map or sequence ::OrtValue - * - * To construct a map (ONNX_TYPE_MAP), use num_values = 2 and `in` should be an array of 2 ::OrtValue%s - * representing keys and values.
- * - * To construct a sequence (ONNX_TYPE_SEQUENCE), use num_values = N where N is the number of the elements in the - * sequence. 'in' should be an array of N ::OrtValue%s. - * - * \param[in] in See above for details - * \param[in] num_values - * \param[in] value_type Must be either ONNX_TYPE_MAP or ONNX_TYPE_SEQUENCE - * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values, - enum ONNXType value_type, _Outptr_ OrtValue** out); - - /** \brief Create an opaque (custom user defined type) ::OrtValue - * - * Constructs an ::OrtValue that contains a value of non-standard type created for - * experiments or while awaiting standardization. ::OrtValue in this case would contain - * an internal representation of the Opaque type. Opaque types are distinguished from - * each other by two strings 1) domain and 2) type name. The combination of the two - * must be unique, so the type representation is properly identified internally. The combination - * must be properly registered from within ORT at both compile/run time or by another API. - * - * To construct the ::OrtValue pass domain and type names, also a pointer to a data container - * the type of which must be known to both ORT and the client program. That data container may or may - * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for - * verification purposes. - * - * \param[in] domain_name Null terminated string of the domain name - * \param[in] type_name Null terminated string of the type name - * \param[in] data_container User pointer Data to populate ::OrtValue - * \param[in] data_container_size Size in bytes of what `data_container` points to - * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name, - _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out); - - /** \brief Get internal data from an opaque (custom user defined type) ::OrtValue - * - * Copies internal data from an opaque value into a user provided buffer - * - * \see OrtApi::CreateOpaqueValue - * - * \param[in] domain_name Null terminated string of the domain name - * \param[in] type_name Null terminated string of the type name - * \param[in] in The opaque ::OrtValue - * \param[out] data_container Buffer to copy data into - * \param[out] data_container_size Size in bytes of the buffer pointed to by data_container. Must match the size of the internal buffer. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in, - _Out_ void* data_container, size_t data_container_size); - - /// @} - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Get a float stored as an attribute in the graph node - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ float* out); - - /** \brief Fetch a 64-bit int stored as an attribute in the graph node - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ int64_t* out); - - /** \brief Fetch a string stored as an attribute in the graph node - * - * If `out` is nullptr, the value of `size` is set to the true size of the string - * attribute, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual string attribute's size, - * the value of `size` is set to the true size of the string attribute, the provided memory - * is filled with the attribute's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string attribute's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string attribute - * and a failure status is returned.) - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * \param[in,out] size See above comments for details - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out, - _Inout_ size_t* size); - - /// @} - /// \name OrtKernelContext - /// Custom operator APIs. - /// @{ - - /** \brief Used for custom operators, get the input count of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); - - /** \brief Used for custom operators, get the output count of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); - - /** \brief Used for custom operators, get an input of a kernel - * - * The function attempts fetches the input of the kernel. If the input is optional - * and not present, the function returns success and out is set to nullptr. - * - * \param[in] context ::OrtKernelContext instance - * \param[in] input index. See KernelContext_GetInputCount for boundaries check. - * \param[in, out] returns a ptr to OrtValue if the input is present - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index, - _Out_ const OrtValue** out); - - /** \brief Used for custom operators, get an output of a kernel - * - * The function attempts fetches the output of the kernel. If the output is optional - * and not present, the function returns success and out is set to nullptr. - * - * \param[in] context ::OrtKernelContext instance - * \param[in] output index. See KernelContext_GetOutputCount for boundaries check. - * \param[in, out] returns a ptr to OrtValue if the output is present - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index, - _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out); - - /// @} - /// \name OrtEnv - /// @{ - ORT_CLASS_RELEASE(Env); - /// @} - /// \name OrtStatus - /// @{ - ORT_CLASS_RELEASE(Status); - /// @} - /// \name OrtMemoryInfo - /// @{ - ORT_CLASS_RELEASE(MemoryInfo); - /// @} - /// \name OrtSession - /// @{ - ORT_CLASS_RELEASE(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) - /// @} - /// \name OrtValue - /// @{ - ORT_CLASS_RELEASE(Value); - /// @} - /// \name OrtRunOptions - /// @{ - ORT_CLASS_RELEASE(RunOptions); - /// @} - /// \name OrtTypeInfo - /// @{ - ORT_CLASS_RELEASE(TypeInfo); - /// @} - /// \name OrtTensorTypeAndShapeInfo - /// @{ - ORT_CLASS_RELEASE(TensorTypeAndShapeInfo); - /// @} - /// \name OrtSessionOptions - /// @{ - ORT_CLASS_RELEASE(SessionOptions); - /// @} - /// \name OrtCustomOpDomain - /// @{ - ORT_CLASS_RELEASE(CustomOpDomain); - - /// @} - /// \name OrtTypeInfo - /// @{ - - /** \brief Get denotation from type information - * - * Augments ::OrtTypeInfo to return denotations on the type. - * - * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor. - * - * \param[in] type_info - * \param[out] denotation Pointer to the null terminated denotation string is written to this pointer. This pointer is valid until the object is destroyed or the name is changed, do not free. - * \param[out] len Length in bytes of the string returned in `denotation` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ const char** const denotation, - _Out_ size_t* len); - - /** \brief Get detailed map information from an ::OrtTypeInfo - * - * This augments ::OrtTypeInfo to return an ::OrtMapTypeInfo when the type is a map. - * The OrtMapTypeInfo has additional information about the map's key type and value type. - * - * This is used by WinML to support model reflection APIs. - * - * \param[out] type_info - * \param[out] out A pointer to the ::OrtMapTypeInfo. Do not free this value. If type_info - * does not contain a map, this value will be set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtMapTypeInfo** out); - - /** \brief Cast ::OrtTypeInfo to an ::OrtSequenceTypeInfo - * - * This api augments ::OrtTypeInfo to return an ::OrtSequenceTypeInfo when the type is a sequence. - * The ::OrtSequenceTypeInfo has additional information about the sequence's element type. - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] type_info - * \param[out] out A pointer to the OrtSequenceTypeInfo. Do not free this value. If type_info - * doesn not contain a sequence, this value will be set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out); - - /// @} - /// \name OrtMapTypeInfo - /// @{ - - /** \brief Get key type from an ::OrtMapTypeInfo - * - * Key types are restricted to being scalar types. - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] map_type_info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out); - - /** \brief Get the value type from an ::OrtMapTypeInfo - * - * \param[in] map_type_info - * \param[out] type_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info); - - /// @} - /// \name OrtSequenceTypeInfo - /// @{ - - /** \brief Get element type from an ::OrtSequenceTypeInfo - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] sequence_type_info - * \param[out] type_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info, - _Outptr_ OrtTypeInfo** type_info); - - /// @} - /// \name OrtMapTypeInfo - /// @{ - ORT_CLASS_RELEASE(MapTypeInfo); - /// @} - /// \name OrtSequenceTypeInfo - /// @{ - ORT_CLASS_RELEASE(SequenceTypeInfo); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief End profiling and return filename of the profile data - * - * Profiling is turned on through OrtApi::EnableProfiling - * - * \param[in] session - * \param[in] allocator - * \param[out] out Null terminated string of the filename, allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* session, _Inout_ OrtAllocator* allocator, _Outptr_ char** out); - - /** \brief Get ::OrtModelMetadata from an ::OrtSession - * - * \param[in] session - * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* session, _Outptr_ OrtModelMetadata** out); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** \brief Get `producer name` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get `graph name` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get `domain` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator, - _Outptr_ char** value); - - /** \brief Get `description` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Return data for a key in the custom metadata map in an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[in] key Null terminated string - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * `value` will be set to nullptr if the given key is not found in the custom metadata map. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value); - - /** \brief Get version number from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[out] value Set to the version number - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value); - - ORT_CLASS_RELEASE(ModelMetadata); - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an OrtEnv - * - * Create an environment with global threadpools that will be shared across sessions. - * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use - * its own thread pools. - * - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[in] tp_options - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel log_severity_level, _In_ const char* logid, - _In_ const OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Use global thread pool on a session - * - * Disable using per session thread pool and use the shared global threadpool. - * This should be used in conjunction with OrtApi::CreateEnvWithGlobalThreadPools. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Create an ::OrtThreadingOptions - * - * \param[out] out Newly created ::OrtThreadingOptions. Must be freed with OrtApi::ReleaseThreadingOptions - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out); - - ORT_CLASS_RELEASE(ThreadingOptions); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] keys Array of null terminated strings (array count = num_keys) allocated using `allocator`. - * The strings and the pointer array must be freed using `allocator` - * `keys` will be set to nullptr if the custom metadata map is empty. - * \param[out] num_keys Set to the number of elements in the `keys` array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** - * - * Override symbolic dimensions (by specific name strings) with actual values - * if known at session initialization time to enable optimizations that can - * take advantage of fixed values (such as memory planning, etc) - * - */ - ORT_API2_STATUS(AddFreeDimensionOverrideByName, - _Inout_ OrtSessionOptions* options, _In_ const char* dim_name, - _In_ int64_t dim_value); - - /// @} - /// \name Misc - /// @{ - - /** \brief Get the names of all available providers - * - * \note The providers in the list are not guaranteed to be usable. They may fail to load due to missing system dependencies. - * For example, if the CUDA/cuDNN libraries are not installed, the CUDA provider will report an error when it is added to the session options. - * - * \param[out] out_ptr Set to a pointer to an array of null terminated strings of the available providers. The entries and the - * array itself must be freed using OrtApi::ReleaseAvailableProviders - * \param[out] provider_length Set to the number of entries in the `out_ptr` array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr, _Out_ int* provider_length); - - /** \brief Release data from OrtApi::GetAvailableProviders. This API will never fail - * so you can rely on it in a noexcept code. - * - * \param[in] ptr The `out_ptr` result from OrtApi::GetAvailableProviders. - * \param[in] providers_length The `provider_length` result from OrtApi::GetAvailableProviders - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr, - _In_ int providers_length); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Get the length of a single string in a string tensor - * - * \param[in] value A string tensor - * \param[in] index Index of the string in the tensor - * \param[out] out Set to number of bytes of the string element - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out); - - /** \brief Get a single string from a string tensor - * - * \param[in] value A string tensor - * \param[in] s_len Number of bytes in the `s` buffer. Must match the value returned by OrtApi::GetStringTensorElementLength. - * \param[in] index Index of the string in the tensor - * \param[out] s The string element contents in UTF-8 encoding. The string is NOT null-terminated. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s); - - /** \brief Set a single string in a string tensor - * - * \param[in] value A string tensor - * \param[in] s A null terminated UTF-8 encoded string - * \param[in] index Index of the string in the tensor to set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Set a session configuration entry as a pair of strings - * - * If a configuration with same key exists, this will overwrite the configuration with the given config_value. - * - * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h - * - * \param[in] options - * \param[in] config_key A null terminated string representation of the config key - * \param[in] config_value A null terminated string representation of the config value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options, - _In_z_ const char* config_key, _In_z_ const char* config_value); - - /// @} - /// \name OrtAllocator - /// @{ - - /** \brief Create an allocator for an ::OrtSession following an ::OrtMemoryInfo - * - * \param[in] session - * \param[in] mem_info valid ::OrtMemoryInfo instance - * \param[out] out Newly created ::OrtAllocator. Must be freed with OrtApi::ReleaseAllocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* session, _In_ const OrtMemoryInfo* mem_info, - _Outptr_ OrtAllocator** out); - - /** \brief Release an ::OrtAllocator obtained from OrtApi::CreateAllocator - */ - ORT_CLASS_RELEASE(Allocator); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Run a model using Io Bindings for the inputs & outputs - * - * \see OrtApi::Run - * - * \param[in] session - * \param[in] run_options - * \param[in] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* session, _In_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr); - - /** \brief Create an ::OrtIoBinding instance - * - * An IoBinding object allows one to bind pre-allocated ::OrtValue%s to input names. - * Thus if you want to use a raw on device buffer as input or output you can avoid - * extra copy during runtime. - * - * \param[in] session - * \param[out] out Newly created ::OrtIoBinding. Must be freed with OrtApi::ReleaseIoBinding - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* session, _Outptr_ OrtIoBinding** out); - - /// @} - /// \name OrtIoBinding - /// @{ - - /** \brief Release an ::OrtIoBinding obtained from OrtApi::CreateIoBinding - */ - ORT_CLASS_RELEASE(IoBinding); - - /** \brief Bind an ::OrtValue to an ::OrtIoBinding input - * - * When using OrtApi::RunWithBinding this value is used for the named input - * - * \param[in] binding_ptr - * \param[in] name Name for the model input - * \param[in] val_ptr ::OrtValue of Tensor type. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); - - /** \brief Bind an ::OrtValue to an ::OrtIoBinding output - * - * When using OrtApi::RunWithBinding this value is used for the named output - * - * \param[in] binding_ptr - * \param[in] name Null terminated string of the model output name - * \param[in] val_ptr ::OrtValue of Tensor type. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); - - /** \brief Bind an ::OrtIoBinding output to a device - * - * Binds the ::OrtValue to a device which is specified by ::OrtMemoryInfo. - * You can either create an instance of ::OrtMemoryInfo with a device id or obtain one from the allocator that you have created/are using - * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocate and bind a chunk of - * memory within ::OrtValue ahead of time. - * - * \see OrtApi::RunWithBinding - * - * \param[in] binding_ptr - * \param[in] name Null terminated string of the device name - * \param[in] mem_info_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* mem_info_ptr); - - /** \brief Get the names of an ::OrtIoBinding's outputs - * - * Returns the names of the outputs in the order they were bound. This is useful after running the model - * with bound outputs because the returned names are in order in which output ::OrtValue are returned. This is useful if - * the order of outputs and their names is not known. - * - * \param[in] binding_ptr - * \param[in] allocator Allocator used to allocate continuous buffers for output strings and lengths. - * \param[out] buffer Returns an array of non-null terminated UTF-8 strings. The number of strings stored is returned in the count parameter. - * This buffer is allocated using `allocator` and must be freed using it. - * \param[out] lengths Returns an array of `count` lengths of the strings returned in `buffer` - * This buffer is allocated using `allocator` and must be freed using it. - * \param[out] count Number of strings returned. If `binding_ptr` has no bound outputs, zero is returned, - * no memory allocation is performed and buffer and lengths are set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, - _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count); - - /** \brief Get the output ::OrtValue objects from an ::OrtIoBinding - * - * Returns an array of pointers to individually allocated ::OrtValue%s that contain results of a model execution with OrtApi::RunWithBinding - * The array contains the same number of ::OrtValue%s and they are in the same order as they were bound with OrtApi::BindOutput - * or OrtApi::BindOutputToDevice. - * - * The returned ::OrtValue%s must be released using OrtApi::ReleaseValue after they are no longer needed. - * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after - * all the ::OrtValue%s contained therein are individually released. - * - * \param[in] binding_ptr - * \param[in] allocator Allocator used to allocate output array - * \param[out] output Set to the allocated array of allocated ::OrtValue outputs. Set to nullptr if there are 0 outputs. - * \param[out] output_count Set to number of ::OrtValue%s returned - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, - _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count); - - /** \brief Clears any previously set Inputs for an ::OrtIoBinding - */ - void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Clears any previously set Outputs for an ::OrtIoBinding - */ - void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Direct memory access to a specified tensor element - * - * For example, given a tensor with shape of [3,224,224], a pointer to the element at location [2,150,128] can be retrieved - * - * This function only works for numeric type tensors (No strings, etc). - * This is a no-copy method whose returned pointer is valid until the passed in ::OrtValue is free'd. - * - * \param[in] value - * \param[in] location_values Pointer to an array of index values that specify an element's location relative to its shape - * \param[in] location_values_count Number of elements in location_values. Must match the number of elements in the tensor's shape. - * \param[out] out Set to a pointer to the element specified - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out); - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an allocator and register it with the ::OrtEnv - * - * Enables sharing the allocator between multiple sessions that use the same env instance. - * Lifetime of the created allocator will be valid for the duration of the environment. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * - * See https://onnxruntime.ai/docs/get-started/with-c.html for details. - * - * \param[in] env ::OrtEnv instance - * \param[in] mem_info - * \param[in] arena_cfg Pass nullptr for defaults - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, - _In_ const OrtArenaCfg* arena_cfg); - - /** \brief Set language projection - * - * Set the language projection for collecting telemetry data when Env is created. - * - * The default is ORT_PROJECTION_C, which means it will classify the language not in the list to C also. - * - * \param[in] ort_env - * \param[in] projection - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Return the time that profiling was started - * - * \note The timer precision varies per platform. On Windows and MacOS, the precision will be ~100ns - * - * \param[in] session - * \param[out] out nanoseconds of profiling's start time - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* session, _Outptr_ uint64_t* out); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Set global intra-op thread count - * - * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools - * - * \param[in] tp_options - * \param[in] intra_op_num_threads Number of threads, special values:
- * 0 = Use default thread count
- * 1 = The invoking thread will be used; no threads will be created in the thread pool. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads); - - /** \brief Set global inter-op thread count - * - * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools - * - * \param[in] tp_options - * \param[in] inter_op_num_threads Number of threads, special values:
- * 0 = Use default thread count
- * 1 = The invoking thread will be used; no threads will be created in the thread pool. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads); - - /** \brief Set global spin control options - * - * This will configure the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. - * Allow spinning of thread pools when their queues are empty. This will set the value for both - * inter_op and intra_op threadpools. - * - * \param[in] tp_options - * \param[in] allow_spinning Valid values are 0 or 1.
- * 0 = It won't spin (recommended if CPU usage is high)
- * 1 = Threadpool will spin to wait for queue to become non-empty - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Add a pre-allocated initializer to a session - * - * If a model contains an initializer with a name that is same as the name passed to this call, - * ORT will use this initializer instance instead of deserializing one from the model file. This - * is useful when you want to share the same initializer across sessions. - * - * \param[in] options - * \param[in] name Null terminated string of the initializer name - * \param[in] val ::OrtValue containing the initializer. Its lifetime and the underlying initializer buffer must be - * managed by the user (created using the OrtApi::CreateTensorWithDataAsOrtValue) and it must outlive the session object - * to which it is added. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name, - _In_ const OrtValue* val); - - /// @} - /// \name OrtEnv - /// @{ - - /** - * Create a custom environment with global threadpools and logger that will be shared across sessions. - * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use - * its own thread pools. - * - * \param[in] logging_function A pointer to a logging function. - * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `logging_function`. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[in] tp_options - * \param[out] out Newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel log_severity_level, - _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append CUDA provider to session options - * - * If CUDA is not available (due to a non CUDA enabled build, or if CUDA is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] cuda_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA, - _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options); - - /** \brief Append ROCM execution provider to the session options - * - * If ROCM is not available (due to a non ROCM enabled build, or if ROCM is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] rocm_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_ROCM, - _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* rocm_options); - - /** \brief Append OpenVINO execution provider to the session options - * - * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. - * - * \param[in] options - * \param[in] provider_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO, - _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Set threading flush-to-zero and denormal-as-zero - * - * Sets global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. - * Flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool. - * \note This option is not needed if the models used have no denormals. Having no denormals is recommended as this option may hurt model accuracy. - * - * \param[in] tp_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options); - - /// @} - /// \name OrtArenaCfg - /// @{ - - /** \deprecated Use OrtApi::CreateArenaCfgV2 - * - * This will create the configuration of an arena that can eventually be used to define an arena based allocator's behavior - * - * \param[in] max_mem Use 0 to allow ORT to choose the default - * \param[in] arena_extend_strategy Use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested - * \param[in] initial_chunk_size_bytes Use -1 to allow ORT to choose the default - * \param[in] max_dead_bytes_per_chunk Use -1 to allow ORT to choose the default - * \param[in] out A pointer to an OrtArenaCfg instance - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, - int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out); - - ORT_CLASS_RELEASE(ArenaCfg); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** - * Use this to obtain the description of the graph present in the model - * (doc_string field of the GraphProto message within the ModelProto message). - * If it doesn't exist, an empty string will be returned. - * - * \param[in] model_metadata An instance of ::OrtModelMetadata - * \param[in] allocator Allocator used to allocate the string that will be returned back - * \param[out] value Set to a null terminated string allocated using `allocator`. The caller is responsible for freeing it using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append TensorRT provider to session options - * - * If TensorRT is not available (due to a non TensorRT enabled build, or if TensorRT is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] tensorrt_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT, - _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptions* tensorrt_options); - - /// @} - /// \name Misc - /// @{ - - /** \brief Set current GPU device ID - * - * Set the current device id of the GPU execution provider (CUDA/tensorrt/rocm). The device id should be less - * than the total number of devices available. This is only useful when multiple-GPUs are installed and it is - * required to restrict execution to a single GPU. - * - * \param[in] device_id - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetCurrentGpuDeviceId, _In_ int device_id); - - /** \brief Get current GPU device ID - * - * Get the current device id of the GPU execution provider (CUDA/tensorrt/rocm). - * - * \see OrtApi::SetCurrentGpuDeviceId - * - * \param[out] device_id - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetCurrentGpuDeviceId, _In_ int* device_id); - - /// @} - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Fetch an array of int64_t values stored as an attribute in the graph node - * - * - * If `out` is nullptr, the value of `size` is set to the true size of the attribute - * array's size, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual attribute array's size, - * the value of `size` is set to the true size of the attribute array's size, - * the provided memory is filled with the attribute's contents, - * and a success status is returned. - * - * If the `size` parameter is less than the actual attribute array's size and `out` - * is not nullptr, the value of `size` is set to the true size of the attribute array's size - * and a failure status is returned.) - * - * \param[in] info instance - * \param[in] name name of the attribute to be parsed - * \param[out] out pointer to memory where the attribute's contents are to be stored - * \param[in, out] size actual size of attribute array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttributeArray_float, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ float* out, _Inout_ size_t* size); - - /** \brief Fetch an array of int64_t values stored as an attribute in the graph node - * - * If `out` is nullptr, the value of `size` is set to the true size of the attribute - * array's size, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual attribute array's size, - * the value of `size` is set to the true size of the attribute array's size, - * the provided memory is filled with the attribute's contents, - * and a success status is returned. - * - * If the `size` parameter is less than the actual attribute array's size and `out` - * is not nullptr, the value of `size` is set to the true size of the attribute array's size - * and a failure status is returned.) - * - * \param[in] info instance - * \param[in] name name of the attribute to be parsed - * \param[out] out pointer to memory where the attribute's contents are to be stored - * \param[in, out] size actual size of attribute array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttributeArray_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ int64_t* out, _Inout_ size_t* size); - - /// @} - /// \name OrtArenaCfg - /// @{ - - /** \brief Create an ::OrtArenaCfg - * - * Create the configuration of an arena that can eventually be used to define an arena based allocator's behavior. - * - * Supported keys are (See https://onnxruntime.ai/docs/get-started/with-c.html for details on what the - * following parameters mean and how to choose these values.): - * "max_mem": Maximum memory that can be allocated by the arena based allocator. - * Use 0 for ORT to pick the best value. Default is 0. - * "arena_extend_strategy": 0 = kNextPowerOfTwo, 1 = kSameAsRequested. - * Use -1 to allow ORT to choose the default. - * "initial_chunk_size_bytes": (Possible) Size of the first allocation in the arena. - * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. - * Ultimately, the first allocation size is determined by the allocation memory request. - * "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after - * crossing which the current chunk is chunked into 2. - * "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. - * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. - * "max_power_of_two_extend_bytes": The maximum enxtend size if arena strategy is `kNextPowerOfTwo`. - * It is not an allocation limit, it is only a limit for extention when requested byte is less than the limit. - * When requested bytes is more than the limit, allocator will still return as requested. - * Use -1 to allow ORT to choose the default 1GB for max_power_of_two_extend_bytes. - * Ultimately, the allocation size is determined by the allocation memory request. - * Further allocation sizes are governed by the arena extend strategy. - * - * \param[in] arena_config_keys Keys to configure the arena - * \param[in] arena_config_values Values to configure the arena - * \param[in] num_keys Number of keys in `arena_config_keys` and `arena_config_values` - * \param[out] out Newly created ::OrtArenaCfg. Must be freed with OrtApi::ReleaseArenaCfg - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateArenaCfgV2, _In_reads_(num_keys) const char* const* arena_config_keys, - _In_reads_(num_keys) const size_t* arena_config_values, _In_ size_t num_keys, - _Outptr_ OrtArenaCfg** out); - - /// @} - /// \name OrtRunOptions - /// @{ - - /** \brief Set a single run configuration entry as a pair of strings - * - * If a configuration with same key exists, this will overwrite the configuration with the given config_value - * - * The config_key and the format of config_value are defined in onnxruntime_run_options_config_keys.h - * - * \param[in] options - * \param[in] config_key A null terminated string representation of the config key - * \param[in] config_value A null terminated string representation of the config value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddRunConfigEntry, _Inout_ OrtRunOptions* options, - _In_z_ const char* config_key, _In_z_ const char* config_value); - - /// @} - /// \name OrtPrepackedWeightsContainer - /// @{ - - /** \brief Create an ::OrtPrepackedWeightsContainer - * - * This container will hold pre-packed buffers of shared initializers for sharing between sessions - * (i.e.) if there are shared initializers that can be shared between sessions, the pre-packed buffers - * of these (if any) may possibly be shared to provide memory footprint savings. Pass this container - * to sessions that you would like to share pre-packed buffers of shared initializers at session - * creation time. - * - * \param[out] out Newly created ::OrtPrepackedWeightsContainer. Must be freed with OrtApi::ReleasePrepackedWeightsContainer - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreatePrepackedWeightsContainer, _Outptr_ OrtPrepackedWeightsContainer** out); - - /** \brief Release OrtPrepackedWeightsContainer instance - * - * \note instance must not be released until the sessions using it are released - */ - ORT_CLASS_RELEASE(PrepackedWeightsContainer); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Create session with prepacked weights container - * - * Same functionality offered by OrtApi::CreateSession except that a container that contains - * pre-packed weights' buffers is written into/read from by the created session. - * This is useful when used in conjunction with OrtApi::AddInitializer which injects - * shared initializer info into sessions. Wherever possible, the pre-packed versions of these - * shared initializers are cached in this container so that multiple sessions can just re-use - * these instead of duplicating these in memory. - * - * \param[in] env OrtEnv instance instance - * \param[in] model_path Null terminated string of the path (wchar on Windows, char otherwise) - * \param[in] options - * \param[in] prepacked_weights_container - * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, - _Outptr_ OrtSession** out); - - /** \brief Create session from memory with prepacked weights container - * - * Same functionality offered by OrtApi::CreateSessionFromArray except that a container that contains - * pre-packed weights' buffers is written into/read from by the created session. - * This is useful when used in conjunction with OrtApi::AddInitializer which injects - * shared initializer info into sessions. Wherever possible, the pre-packed versions of these - * shared initializers are cached in this container so that multiple sessions can just re-use - * these instead of duplicating these in memory. - * - * \param[in] env - * \param[in] model_data Array of bytes holding the model - * \param[in] model_data_length Number of bytes in `model_data_model` - * \param[in] options - * \param[in] prepacked_weights_container - * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ const OrtEnv* env, - _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, - _Outptr_ OrtSession** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append TensorRT execution provider to the session options - * - * If TensorRT is not available (due to a non TensorRT enabled build), this function will return failure. - * - * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, it takes an - * ::OrtTensorRTProviderOptions which is publicly defined. This takes an opaque ::OrtTensorRTProviderOptionsV2 - * which must be created with OrtApi::CreateTensorRTProviderOptions. - * - * For OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, the user needs to instantiate ::OrtTensorRTProviderOptions - * as well as allocate/release buffers for some members of ::OrtTensorRTProviderOptions. - * Here, OrtApi::CreateTensorRTProviderOptions and Ortapi::ReleaseTensorRTProviderOptions will do the memory management for you. - * - * \param[in] options - * \param[in] tensorrt_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT_V2, - _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options); - - /// @} - /// \name OrtTensorRTProviderOptionsV2 - /// @{ - - /** \brief Create an OrtTensorRTProviderOptionsV2 - * - * \param[out] out Newly created ::OrtTensorRTProviderOptionsV2. Must be released with OrtApi::ReleaseTensorRTProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out); - - /** \brief Set options in a TensorRT Execution Provider. - * - * Please refer to https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#cc - * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtTensorRTProviderOptionsV2 - * and value should be its related range. Recreates the options and only sets the supplied values. - * - * For example, key="trt_max_workspace_size" and value="2147483648" - * - * \param[in] tensorrt_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UpdateTensorRTProviderOptions, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Get serialized TensorRT provider options string. - * - * For example, "trt_max_workspace_size=2147483648;trt_max_partition_iterations=10;trt_int8_enable=1;......" - * - * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with OrtApi::CreateAllocator or OrtApi::GetAllocatorWithDefaultOptions - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtTensorRTProviderOptionsV2 - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - */ - void(ORT_API_CALL* ReleaseTensorRTProviderOptions)(_Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* input); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Enable custom operators - * - * See onnxruntime-extensions: https://github.com/microsoft/onnxruntime-extensions.git - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableOrtCustomOps, _Inout_ OrtSessionOptions* options); - - /// @} - /// \name OrtAllocator - /// @{ - - /** \brief Register a custom allocator - * - * Enables sharing between multiple sessions that use the same env instance. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * - * The behavior of this is exactly the same as OrtApi::CreateAndRegisterAllocator except - * instead of ORT creating an allocator based on provided info, in this case - * ORT uses the user-provided custom allocator. - * See https://onnxruntime.ai/docs/get-started/with-c.html for details. - * - * \param[in] env - * \param[in] allocator User provided allocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RegisterAllocator, _Inout_ OrtEnv* env, _In_ OrtAllocator* allocator); - - /** \brief Unregister a custom allocator - * - * It is an error if you provide an ::OrtMemoryInfo not corresponding to any - * registered allocators for sharing. - * - * \param[in] env - * \param[in] mem_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UnregisterAllocator, _Inout_ OrtEnv* env, - _In_ const OrtMemoryInfo* mem_info); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Sets *out to 1 iff an ::OrtValue is a SparseTensor, and 0 otherwise - * - * \param[in] value existing ::OrtValue - * \param[out] out unless an error occurs, contains 1 iff the value contains an instance - * of sparse tensor or 0 otherwise. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(IsSparseTensor, _In_ const OrtValue* value, _Out_ int* out); - - /** \brief Create an ::OrtValue with a sparse tensor that is empty. - * - * Use FillSparseTensor() functions to populate sparse tensor with non-zero values and - * format specific indices data. - * Use ReleaseValue to destroy the sparse tensor, this will also release the buffer inside the output value - * if any was allocated. - * \param[in,out] allocator allocator to use when performing an allocation. Allocation will be performed - * by FillSparseTensor() APIs. The lifespan of the allocator instance must eclipse the lifespan - * this sparse tensor instance as the same allocator will be used to free memory. - * \param[in] dense_shape shape of the original dense tensor - * \param[in] dense_shape_len number of shape dimensions being passed - * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx - * \param[out] out Should be freed by calling ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* dense_shape, - size_t dense_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and COO indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape pointer to values shape array - * \param[in] values_shape_len length of the values_shape - * \param[in] values pointer to an array of values. For strings, pass const char**. - * \param[in] indices_data pointer to a location of COO indices - * \param[in] indices_num number of COO indices - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* indices_data, size_t indices_num); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and CSR indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape pointer to values shape array - * \param[in] values_shape_len length of the values_shape - * \param[in] values - pointer to an array of values. For strings, pass const char**. - * \param[in] inner_indices_data pointer to a location of CSR inner indices - * \param[in] inner_indices_num number of CSR inner indices - * \param[in] outer_indices_data pointer to a location of CSR outer indices - * \param[in] outer_indices_num number of CSR outer indices - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* inner_indices_data, size_t inner_indices_num, - _In_ const int64_t* outer_indices_data, size_t outer_indices_num); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and BlockSparse indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape - * \param[in] values_shape_len - * \param[in] values structure with values information - * \param[in] indices_shape_data pointer to a location of indices shape - * \param[in] indices_shape_len length of the block sparse indices shape - * \param[in] indices_data pointer to a location of indices data. Shape will determine the length of the indices data. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* indices_shape_data, size_t indices_shape_len, - _In_ const int32_t* indices_data); - - /** - * Create an ::OrtValue with a sparse tensor. This is the first step. - * Next, use UseIndices() functions to supply sparse tensor with - * format specific indices data and set its sparse format to a specific enum value. - * This will not perform memory allocations. It will - * use supplied user buffer which should outlive the created sparse tensor. - * Use OrtApi::ReleaseValue to destroy the sparse tensor. It would not release the supplied values buffer. - * This function can not be used to map strings from the user allocated memory. Strings must always be copied - * and have UTF-8 encoding. Therefore, use OrtApi::CreateSparseTensorAsOrtValue above and then fill it with data - * using appropriate Make*() function. - * - * \param[in] info memory info where sparse values reside. - * \param[in,out] p_data pointer to a user allocated buffer with values. To create a full sparse tensor with no non-zero - * values, pass nullptr - * \param[in] dense_shape shape of the original dense tensor - * \param[in] dense_shape_len number of shape dimensions being passed - * \param[in] values_shape shape of the values data. To create a fully sparse tensor with no non-zero values, - * pass {0} shape. - * \param[in] values_shape_len number of values shape dimensions - * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx - * \param[out] out Should be freed by calling ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSparseTensorWithValuesAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, - _In_ const int64_t* dense_shape, size_t dense_shape_len, - _In_ const int64_t* values_shape, size_t values_shape_len, - ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** - * This assigns Coo format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_COO. This will not allocate any additional memory for data. The life span of - * indices_data buffer should eclipse the life span of this ::OrtValue. - * - * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in,out] indices_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] indices_num number of COO indices. Should either be 0 for fully sparse tensors, be equal - * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue for 1-D {nnz} indices or - * be twice as number of nnz values for a 2-D indices {nnz, 2} - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* indices_data, size_t indices_num); - - /** - * The assigns CSR format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_CSRC. This will not allocate any additional memory for data. The life spans of - * inner_data and outer_data buffers should eclipse the life span of this ::OrtValue. - * - * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in,out] inner_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] inner_num number of inner CSR indices. Should either be 0 for fully sparse tensors or be equal - * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue. - * \param[in,out] outer_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] outer_num number of CSR outer indices. Should either be 0 for fully sparse tensors or - * equal to rows + 1 of the dense shape. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseCsrIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* inner_data, size_t inner_num, - _Inout_ int64_t* outer_data, size_t outer_num); - - /** - * The assigns BlockSparse format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_BLOCK_SPARSE. This will not allocate any additional memory for data. The life span of - * indices_data buffer must eclipse the lifespan of this ::OrtValue. - * - * \param[in,out] ort_value OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in] indices_shape pointer to indices shape. Use {0} for fully sparse tensors - * \param[in] indices_shape_len length of the indices shape - * \param[in,out] indices_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseBlockSparseIndices, _Inout_ OrtValue* ort_value, const int64_t* indices_shape, size_t indices_shape_len, _Inout_ int32_t* indices_data); - - /** \brief Returns sparse tensor format enum iff a given ort value contains an instance of sparse tensor. - * - * \param[in] ort_value ::OrtValue that contains an instance of sparse tensor - * \param[out] out pointer to out parameter - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorFormat, _In_ const OrtValue* ort_value, _Out_ enum OrtSparseFormat* out); - - /** \brief Returns data type and shape of sparse tensor values (nnz) iff ::OrtValue contains a SparseTensor. - * - * \param[in] ort_value An ::OrtValue that contains a fully constructed sparse tensor - * \param[out] out Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorValuesTypeAndShape, _In_ const OrtValue* ort_value, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Returns numeric data for sparse tensor values (nnz). For string values use GetStringTensor*(). - * - * \param[in] ort_value an instance of ::OrtValue containing sparse tensor - * \param[out] out returns a pointer to values data. Do not attempt to free this ptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorValues, _In_ const OrtValue* ort_value, _Outptr_ const void** out); - - /** \brief Returns data type, shape for the type of indices specified by indices_format. - * - * \param[in] ort_value ::OrtValue containing sparse tensor. - * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse - * tensor does not contain. - * \param[out] out an instance of ::OrtTensorTypeAndShapeInfo. Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorIndicesTypeShape, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Returns indices data for the type of the indices specified by indices_format - * - * \param[in] ort_value ::OrtValue containing sparse tensor. - * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse tensor does not contain. - * \param[out] num_indices Pointer to where the number of indices entries is returned - * \param[out] indices Returned pointer to the indices data. Do not free the returned pointer as it refers to internal data owned by the ::OrtValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorIndices, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Out_ size_t* num_indices, _Outptr_ const void** indices); - /// @} - /// \name OrtSessionOptions - /// @{ - - /** - * \brief Sets out to 1 iff an optional type OrtValue has an element, 0 otherwise (OrtValue is None) - * Use this API to find if the optional type OrtValue is None or not. - * If the optional type OrtValue is not None, use the OrtValue just like any other OrtValue. - * For example, if you get an OrtValue that corresponds to Optional(tensor) and - * if HasValue() returns true, use it as tensor and so on. - - * \param[in] value Input OrtValue. - * \param[out] out indicating if the input OrtValue contains data (1) or if it is a None (0) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(HasValue, _In_ const OrtValue* value, _Out_ int* out); - - /// @} - /// \name OrtKernelContext - /// Custom operator APIs. - /// @{ - - /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel - * \see ::OrtCustomOp - * \param[in] context OrtKernelContext instance - * \param[out] out Returns pointer to a GPU compute stream that can be used to launch the custom GPU kernel. - * If retrieving the GPU compute stream is not relevant (GPU not enabled in the build, kernel partitioned to - * some other EP), then a nullptr is returned as the output param. - * Do not free or mutate the returned pointer as it refers to internal data owned by the underlying session. - * Only use it for custom kernel launching. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetGPUComputeStream, _In_ const OrtKernelContext* context, _Outptr_ void** out); - - /// @} - /// \name GetTensorMemoryInfo - /// @{ - /** \brief Returns a pointer to the ::OrtMemoryInfo of a Tensor - * \param[in] value ::OrtValue containing tensor. - * \param[out] mem_info ::OrtMemoryInfo of the tensor. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorMemoryInfo, _In_ const OrtValue* value, _Out_ const OrtMemoryInfo** mem_info); - - /// @} - /// \name GetExecutionProviderApi - /// @{ - /** \brief Get a pointer to the requested version of the Execution Provider specific - * API extensions to the OrtApi - * \param[in] provider_name The name of the execution provider name. Currently only the following - * values are supported: "DML". - * \param[in] version Must be ::ORT_API_VERSION. - * \param[out] provider_api A void pointer containing a reference to the execution provider versioned api structure. - * For example, the provider_api pointer can be cast to the OrtDmlApi* when the provider_name is "DML". - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetExecutionProviderApi, _In_ const char* provider_name, _In_ uint32_t version, _Outptr_ const void** provider_api); - - /// @} - - /// \name SessionOptions - /// @{ - /** \brief Set custom thread creation function - * - * \param[in] options Session options - * \param[in] ort_custom_create_thread_fn Custom thread creation function - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomCreateThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /** \brief Set creation options for custom thread - * - * \param[in] options Session options - * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomThreadCreationOptions, _Inout_ OrtSessionOptions* options, _In_ void* ort_custom_thread_creation_options); - - /** \brief Set custom thread join function - * - * \param[in] options Session options - * \param[in] ort_custom_join_thread_fn Custom join thread function, must not be nullptr when ort_custom_create_thread_fn is set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomJoinThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); - /// @} - - /// \name OrtThreadingOptions - /// @{ - /** \brief Set custom thread creation function for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_create_thread_fn Custom thread creation function - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomCreateThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /** \brief Set custom thread creation options for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomThreadCreationOptions, _Inout_ OrtThreadingOptions* tp_options, _In_ void* ort_custom_thread_creation_options); - - /** \brief Set custom thread join function for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_join_thread_fn Custom thread join function, must not be nullptr when global ort_custom_create_thread_fn is set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomJoinThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); - /// @} - - /** \brief Synchronize bound inputs. The call may be necessary for some providers, such as cuda, - * in case the system that allocated bound memory operated on a different stream. However, the - * operation is provider specific and could be a no-op. - * - * \param[inout] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SynchronizeBoundInputs, _Inout_ OrtIoBinding* binding_ptr); - - /** \brief Synchronize bound outputs. The call may be necessary for some providers, such as cuda, - * in case the system that allocated bound memory operated on a different stream. However, the - * operation is provider specific and could be a no-op. - * - * \param[inout] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr); - - /// \name OrtSessionOptions - /// @{ - - /** \brief Append CUDA execution provider to the session options - * - * If CUDA is not available (due to a non CUDA enabled build), this function will return failure. - * - * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_CUDA, it takes an - * ::OrtCUDAProviderOptions which is publicly defined. This takes an opaque ::OrtCUDAProviderOptionsV2 - * which must be created with OrtApi::CreateCUDAProviderOptions. - * - * For OrtApi::SessionOptionsAppendExecutionProvider_CUDA, the user needs to instantiate ::OrtCUDAProviderOptions - * as well as allocate/release buffers for some members of ::OrtCUDAProviderOptions. - * Here, OrtApi::CreateCUDAProviderOptions and Ortapi::ReleaseCUDAProviderOptions will do the memory management for you. - * - * \param[in] options - * \param[in] cuda_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA_V2, - _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options); - - /// @} - /// \name OrtCUDAProviderOptionsV2 - /// @{ - - /** \brief Create an OrtCUDAProviderOptionsV2 - * - * \param[out] out Newly created ::OrtCUDAProviderOptionsV2. Must be released with OrtApi::ReleaseCudaProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out); - - /** \brief Set options in a CUDA Execution Provider. - * - * Please refer to https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#configuration-options - * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtCUDAProviderOptionsV2 - * and value should be its related range. Recreates the options and only sets the supplied values. - * - * For example, key="device_id" and value="0" - * - * \param[in] cuda_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** - * Get serialized CUDA provider options string. - * - * For example, "device_id=0;arena_extend_strategy=0;......" - * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtCUDAProviderOptionsV2 - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - * - * \since Version 1.11. - */ - void(ORT_API_CALL* ReleaseCUDAProviderOptions)(_Frees_ptr_opt_ OrtCUDAProviderOptionsV2* input); - - /// @} - - /** \brief Append MIGraphX provider to session options - * - * If MIGraphX is not available (due to a non MIGraphX enabled build, or if MIGraphX is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] migraphx_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_MIGraphX, - _In_ OrtSessionOptions* options, _In_ const OrtMIGraphXProviderOptions* migraphx_options); - - /** \brief Replace initialized Tensors with external data with the data provided in initializers. - * - * The function will find the initialized TensorProtos with external data in the graph with the provided names and - * replace them with the provided tensors. The API verifies that the TensorProto being replaced - * has an external data reference and has the same name, dimensions and data type as its replacement. The replacement - * will occur before any of the optimizations take place. The data will be copied into the graph - * since TensorProto can't refer to the user provided buffers. - * - * Once the model has been loaded, the OrtValue(s) added to SessionOptions instance will be removed - * from the internal SessionOptions copy to save memory, the user provided buffers can then be deallocated - * and the SessionOptions instance that refers to them can be destroyed. - * - * \param[in] options - * \param[in] initializer_names Array of null terminated UTF-8 encoded strings of the initializers names. - * \param[in] initializers Array of ::OrtValue type - * \param[in] num_initializers Number of elements in the initializer_names and initializers - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.12. - */ - ORT_API2_STATUS(AddExternalInitializers, _In_ OrtSessionOptions* options, - _In_reads_(num_initializers) const char* const* initializer_names, - _In_reads_(num_initializers) const OrtValue* const* initializers, size_t num_initializers); - - /** \brief: Create attribute of onnxruntime operator - * - * \param[in] name Name of the attribute - * \param[in] data Data content of the attribute - * \param[in] len Number of bytes stored in data - * \param[in] type Data type - * \param[out] op_attr Attribute that has been created, which must be released by OrtApi::ReleaseOpAttr - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CreateOpAttr, - _In_ const char* name, - _In_ const void* data, - _In_ int len, - _In_ OrtOpAttrType type, - _Outptr_ OrtOpAttr** op_attr); - - /* \brief: Release op attribute - * - * \param[in] opAttr Attribute created by OrtApi::CreateOpAttr - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(OpAttr); - - /** \brief: Create onnxruntime native operator - * - * \param[in] info Kernel info - * \param[in] op_name Operator name - * \param[in] domain Operator domain - * \param[in] version Operator opset version - * \param[in] type_constraint_names Name of the type contraints, such as "T" or "T1" - * \param[in] type_constraint_values Type of each contraints - * \param[in] type_constraint_count Number of contraints - * \param[in] attr_values Attributes used to initialize the operator - * \param[in] attr_count Number of the attributes - * \param[in] input_count Number of inputs - * \param[in] output_count Number of outputs - * \param[out] ort_op Operator that has been created - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CreateOp, - _In_ const OrtKernelInfo* info, - _In_z_ const char* op_name, - _In_z_ const char* domain, - int version, - _In_reads_(type_constraint_count) const char** type_constraint_names, - _In_reads_(type_constraint_count) const ONNXTensorElementDataType* type_constraint_values, - int type_constraint_count, - _In_reads_(attr_count) const OrtOpAttr* const* attr_values, - int attr_count, - int input_count, - int output_count, - _Outptr_ OrtOp** ort_op); - - /** \brief: Invoke the operator created by OrtApi::CreateOp - * The inputs must follow the order as specified in onnx specification - * - * \param[in] context Kernel context - * \param[in] ort_op Operator that has been created - * \param[in] input_values Array of inputs - * \param[in] input_count Number of inputs - * \param[in] output_values Array of outputs - * \param[in] output_count Number of outputs - * - * \since Version 1.12. - */ - ORT_API2_STATUS(InvokeOp, - _In_ const OrtKernelContext* context, - _In_ const OrtOp* ort_op, - _In_ const OrtValue* const* input_values, - _In_ int input_count, - _Inout_ OrtValue* const* output_values, - _In_ int output_count); - - /* \brief: Release an onnxruntime operator - * - * \param[in] Op Operator created by OrtApi::CreateOp - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(Op); - - /** \brief: Append execution provider to the session options. - * \param[in] options - * \param[in] provider_name - provider to add. - * \param[in] provider_options_keys - keys to configure the provider options - * \param[in] provider_options_values - values to configure the provider options - * \param[in] num_keys - number of keys passed in - * - * Currently supported providers: - * QNN - * SNPE - * XNNPACK - * - * Note: If an execution provider has a dedicated SessionOptionsAppendExecutionProvider_ function - * that should be used to add it. - * - * QNN supported keys: - * "backend_path": file path to QNN backend library. - * "profiling_level": QNN profiling level, options: "off", "basic", "detailed". Default to off. - * "profiling_file_path": QNN profiling file path if ETW not enabled. - * "rpc_control_latency": QNN RPC control latency. - * "vtcm_mb": QNN VTCM size in MB. default to 0(not set). - * "htp_performance_mode": QNN performance mode, options: "burst", "balanced", "default", "high_performance", - * "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", "sustained_high_performance". Default to "default". - * "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will - * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and - * may alter model/EP partitioning. Use only for debugging. - * "qnn_context_priority": QNN context priority, options: "low", "normal", "normal_high", "high". Default to "normal". - * "htp_graph_finalization_optimization_mode": Set the optimization mode for graph finalization on the HTP backend. Available options: - * - "0": Default. - * - "1": Faster preparation time, less optimal graph. - * - "2": Longer preparation time, more optimal graph. - * - "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific details. - * "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values. Defaults to "0" (unknown). - * "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators. Available options: - * - "0": Default (none). - * - "68" - * - "69" - * - "73" - * - "75" - * "device_id": The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device). - "enable_htp_fp16_precision": Only used for float32 model. - Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision. - - "0": Default. With fp32 precision. - - "1": With fp16 precision. - * - * SNPE supported keys: - * "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16", - * "DSP", "DSP_FIXED8_TF", "AIP_FIXED_TF", "AIP_FIXED8_TF". - * Mapping to SNPE Runtime_t definition: CPU, CPU_FLOAT32 => zdl::DlSystem::Runtime_t::CPU; - * GPU, GPU_FLOAT32_16_HYBRID => zdl::DlSystem::Runtime_t::GPU; - * GPU_FLOAT16 => zdl::DlSystem::Runtime_t::GPU_FLOAT16; - * DSP, DSP_FIXED8_TF => zdl::DlSystem::Runtime_t::DSP. - * AIP_FIXED_TF, AIP_FIXED8_TF => zdl::DlSystem::Runtime_t::AIP_FIXED_TF. - * "priority": execution priority, options: "low", "normal". - * "buffer_type": ITensor or user buffers, options: "ITENSOR", user buffer with different types - "TF8", "TF16", "UINT8", "FLOAT". - * "ITENSOR" -- default, ITensor which is float only. - * "TF8" -- quantized model required, "FLOAT" -- for both quantized or non-quantized model - * "enable_init_cache": enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. - * If SNPE is not available (due to a non Snpe enabled build or its dependencies not being installed), this function will fail. - * - * XNNPACK supported keys: - * "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider. - * default value is 0, which means to use the session thread-pool size. - * - * \since Version 1.12. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider, _In_ OrtSessionOptions* options, - _In_ const char* provider_name, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /* \brief: Get a copy of kernel info - * - * \param[in] info Kernel info - * \param[out] info_copy Copy of kernel info - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CopyKernelInfo, - _In_ const OrtKernelInfo* info, - _Outptr_ OrtKernelInfo** info_copy); - - /* \brief: Release kernel info - * - * \param[in] KernelInfo A copy of kernel info returned by CopyKernelInfo - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(KernelInfo); - - /// \name Ort Training - /// @{ - /** \brief Gets the Training C Api struct - * - * Call this function to access the ::OrtTrainingApi structure that holds pointers to functions that enable - * training with onnxruntime. - * \note A NULL pointer will be returned and no error message will be printed if the training api - * is not supported with this build. A NULL pointer will be returned and an error message will be - * printed if the provided version is unsupported, for example when using a runtime older than the - * version created with this header file. - * - * \param[in] version Must be ::ORT_API_VERSION - * \return The ::OrtTrainingApi struct for the version requested. - * - * \since Version 1.13 - */ - const OrtTrainingApi*(ORT_API_CALL* GetTrainingApi)(uint32_t version)NO_EXCEPTION; - - /// @} - - /** \brief Append CANN provider to session options - * - * If CANN is not available (due to a non CANN enabled build, or if CANN is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] cann_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CANN, - _In_ OrtSessionOptions* options, _In_ const OrtCANNProviderOptions* cann_options); - - /** \brief Create an OrtCANNProviderOptions - * - * \param[out] out created ::OrtCANNProviderOptions. Must be released with OrtApi::ReleaseCANNProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(CreateCANNProviderOptions, _Outptr_ OrtCANNProviderOptions** out); - - /** \brief Set options in a CANN Execution Provider. - * - * \param[in] cann_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(UpdateCANNProviderOptions, _Inout_ OrtCANNProviderOptions* cann_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Get serialized CANN provider options string. - * - * \param[in] cann_options OrtCANNProviderOptions instance - * \param[in] allocator a ptr to an instance of OrtAllocator obtained with CreateAllocator() - * or GetAllocatorWithDefaultOptions(), the specified allocator will be used to allocate - * continuous buffers for output strings and lengths. - * \param[out] ptr is a UTF-8 null terminated string allocated using 'allocator'. - * The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(GetCANNProviderOptionsAsString, _In_ const OrtCANNProviderOptions* cann_options, - _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an OrtCANNProviderOptions - * - * \param[in] the pointer of OrtCANNProviderOptions which will been deleted - * - * \since Version 1.13. - */ - void(ORT_API_CALL* ReleaseCANNProviderOptions)(_Frees_ptr_opt_ OrtCANNProviderOptions* input); - - /* \brief Get OrtDevice type from MemoryInfo - * - * \since Version 1.14 - */ - void(ORT_API_CALL* MemoryInfoGetDeviceType)(_In_ const OrtMemoryInfo* ptr, _Out_ OrtMemoryInfoDeviceType* out); - - /* \brief Update the OrtEnv instance with custom log severity level - * - * \param[in] ort_env The OrtEnv instance being used - * \param[in] log_severity_level The log severity level. - * - * \since Version 1.14. - */ - ORT_API2_STATUS(UpdateEnvWithCustomLogLevel, _In_ OrtEnv* ort_env, OrtLoggingLevel log_severity_level); - - /* \brief Set affinities for intra op threads - * - * Affinity string follows format: - * logical_processor_id,logical_processor_id;logical_processor_id,logical_processor_id - * Semicolon isolates configurations among threads, while comma split processors where ith thread expected to attach to. - * e.g. 1,2,3;4,5 - * specifies affinities for two threads, with the 1st thread attach to the 1st, 2nd, and 3rd processor, and 2nd thread to the 4th and 5th. - * To ease the configuration, an "interval" is also allowed: - * e.g. 1-8;8-16;17-24 - * orders that the 1st thread runs on first eight processors, 2nd thread runs on next eight processors, and so forth. - * Note: - * 1. Once set, the number of thread affinities must equal to intra_op_num_threads - 1, - * ort does not set affinity on the main thread which is started and managed by the calling app; - * 2. For windows, ort will infer the group id from a logical processor id, for example, assuming there are two groups with each has 64 logical processors, - * an id of 64 will be inferred as the last processor of the 1st group, while 65 will be interpreted as the 1st processor of the second group. - * Hence 64-65 is an invalid configuration, because a windows thread cannot be attached to processors across group boundary. - * - * \since Version 1.14 - */ - ORT_API2_STATUS(SetGlobalIntraOpThreadAffinity, _Inout_ OrtThreadingOptions* tp_options, const char* affinity_string); - - /** \brief Register custom ops from a shared library. - * - * Loads a shared library (.dll on windows, .so on linux, etc) named 'library_name' and looks for this entry point: - * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); - * It then passes in the provided session options to this function along with the api base. - * - * The handle to the loaded library is automatically released by ORT when the last OrtSession that references the - * library handle is released. If no OrtSession is created, then the library handle is released when the provided - * OrtSessionOptions is released. - * - * \param[in] options The session options. - * \param[in] library_name The name of the shared library to load and register. Refer to OS-specific dynamic library - * loading utilities (e.g., LoadLibraryEx on Windows or dlopen on Linux/MacOS) for information - * on the format of library names and search paths. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(RegisterCustomOpsLibrary_V2, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* library_name); - - /** \brief Register custom ops by calling a RegisterCustomOpsFn function. - * - * Searches for registration_func_name and if found calls it. - * - * The library containing the function must either be linked against or previously loaded by the executable. - * - * If you want ONNX Runtime to load the library and manage its lifetime, use RegisterCustomOpsLibrary_V2. - * - * RegisterCustomOpsUsingFunction can be used in scenarios where it may not be possible for ONNX Runtime to load - * the library from a path. e.g. mobile platforms where the library must be linked into the app. - * - * The registration function must have the signature of RegisterCustomOpsFn: - * OrtStatus* (*fn)(OrtSessionOptions* options, const OrtApiBase* api); - * - * See https://onnxruntime.ai/docs/reference/operators/add-custom-op.html for details on how the registration - * function should be implemented. - * - * \param[in] options OrtSessionOptions that is passed through as the first argument in the call to the - * registration function. - * \param[in] registration_func_name Name of registration function to use. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(RegisterCustomOpsUsingFunction, _Inout_ OrtSessionOptions* options, - _In_ const char* registration_func_name); - - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Get the number of inputs from ::OrtKernelInfo. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the number of inputs - * during kernel/session creation. - * - * \param[in] info Instance of ::OrtKernelInfo. - * \param[out] out Pointer to variable assigned with the result on success. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetInputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); - - /** \brief Get the number of outputs from ::OrtKernelInfo. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the number of outputs - * during kernel/session creation. - * - * \param[in] info Instance of ::OrtKernelInfo. - * \param[out] out Pointer to variable assigned with the result on success. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetOutputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); - - /** \brief Get the name of a ::OrtKernelInfo's input. - * - * Used in the CreateKernel callback of an OrtCustomOp to query an input's name - * during kernel/session creation. - * - * If `out` is nullptr, the value of `size` is set to the size of the name - * string (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the name string's size, - * the value of `size` is set to the true size of the string (including null-terminator), - * the provided memory is filled with the string's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string - * and a failure status is returned. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index The index of the input name to get. Returns a failure status if out-of-bounds. - * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the input's name. - * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetInputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, - _Inout_ size_t* size); - - /** \brief Get the name of a ::OrtKernelInfo's output. - * - * Used in the CreateKernel callback of an OrtCustomOp to query an output's name - * during kernel/session creation. - * - * If `out` is nullptr, the value of `size` is set to the size of the name - * string (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the name string's size, - * the value of `size` is set to the true size of the string (including null-terminator), - * the provided memory is filled with the string's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string - * and a failure status is returned. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index The index of the output name to get. Returns a failure status if out-of-bounds. - * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the output's - * name. - * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetOutputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, - _Inout_ size_t* size); - - /** \brief Get the type information for a ::OrtKernelInfo's input. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information - * of an input during kernel/session creation. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index Which input to get the type information for - * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetInputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, - _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get the type information for a ::OrtKernelInfo's output. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information - * of an output during kernel/session creation. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index Which input to get the type information for - * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetOutputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, - _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get a ::OrtValue tensor stored as an attribute in the graph node. - * - * Used in the CreateKernel callback of an OrtCustomOp to get a tensor attribute. - * - * \param[in] info ::OrtKernelInfo instance. - * \param[in] name UTF-8 null-terminated string representing the attribute's name. - * \param[in] allocator Allocator used to allocate the internal tensor state. - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue, - * which will also free internal tensor state allocated with the provided allocator. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_tensor, _In_ const OrtKernelInfo* info, _In_z_ const char* name, - _Inout_ OrtAllocator* allocator, _Outptr_ OrtValue** out); - - /// @} - /// \name OrtSessionOptions - /// Custom operator APIs - /// @{ - - /** \brief Checks if the given session configuration entry exists. - * - * The config_key formats are defined in onnxruntime_session_options_config_keys.h - * - * Can be used in a custom operator library to check for session configuration entries - * that target one or more custom operators in the library. Example: The config entry - * custom_op.myop.some_key targets a custom op named "myop". - * - * \param[in] options The ::OrtSessionOptions instance. - * \param[in] config_key A null-terminated UTF-8 string representation of the configuration key. - * \param[out] out Pointer set to 1 if the entry exists and 0 otherwise. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(HasSessionConfigEntry, _In_ const OrtSessionOptions* options, - _In_z_ const char* config_key, _Out_ int* out); - - /** \brief Get a session configuration value. - * - * Returns a failure status if the configuration key does not exist. - * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h - * - * If `config_value` is nullptr, the value of `size` is set to the true size of the string - * value (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual string value's size, - * the value of `size` is set to the true size of the string value, the provided memory - * is filled with the value's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string value's size and `config_value` - * is not nullptr, the value of `size` is set to the true size of the string value - * and a failure status is returned. - * - * Can be used in a custom operator library to get session configuration entries - * that target one or more custom operators in the library. Example: The config entry - * custom_op.myop.some_key targets a custom op named "myop". - * - * \param[in] options The session options. - * \param[in] config_key A null-terminated UTF-8 string representation of the config key. - * \param[in] config_value Pointer to memory where the null-terminated UTF-8 string value will be stored. - * \param[in,out] size Pointer to the size of the `config_value` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(GetSessionConfigEntry, _In_ const OrtSessionOptions* options, - _In_z_ const char* config_key, _Out_ char* config_value, _Inout_ size_t* size); - - /// @} - - /** \brief Append dnnl provider to session options - * - * If oneDNN is not available, this function will return failure. - * - * \param[in] options - * \param[in] dnnl_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_Dnnl, - _In_ OrtSessionOptions* options, _In_ const OrtDnnlProviderOptions* dnnl_options); - - /** \brief Create an OrtDnnlProviderOptions - * - * \param[out] out Newly created ::OrtDnnlProviderOptions. Must be released with OrtApi::ReleaseDnnlProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(CreateDnnlProviderOptions, _Outptr_ OrtDnnlProviderOptions** out); - - /** \brief Set options in a oneDNN Execution Provider. - * - * Key should be in null terminated string format of the member of ::OrtDnnlProviderOptions - * and value should be its related range. - * - * For example, key="use_arena" and value="1" - * - * \param[in] dnnl_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(UpdateDnnlProviderOptions, _Inout_ OrtDnnlProviderOptions* dnnl_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** - * Get serialized oneDNN provider options string. - * - * For example, "use_arena=1;......" - * - * \param dnnl_options - OrtDnnlProviderOptions instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(GetDnnlProviderOptionsAsString, _In_ const OrtDnnlProviderOptions* dnnl_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtDnnlProviderOptions - * - * \since Version 1.15. - */ - void(ORT_API_CALL* ReleaseDnnlProviderOptions)(_Frees_ptr_opt_ OrtDnnlProviderOptions* input); - - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Get the graph node name from ::OrtKernelInfo. - * - * If `out` is nullptr, the value of `size` is set to the size of the name - * string (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the name string's size, - * the value of `size` is set to the true size of the string (including null-terminator), - * the provided memory is filled with the string's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string - * and a failure status is returned. - * - * Can be used in a custom operator's CreateKernel callback to get the name of the operator's node name in the graph. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the name. - * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(KernelInfo_GetNodeName, _In_ const OrtKernelInfo* info, _Out_ char* out, _Inout_ size_t* size); - - /** \brief Get the session logger from ::OrtKernelInfo. - * - * Used in the CreateKernel callback of an OrtCustomOp to get a logger that can be used to log - * messages. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[out] logger Pointer set to the session's ::OrtLogger. Owned by ONNX Runtime, so do not free. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(KernelInfo_GetLogger, _In_ const OrtKernelInfo* info, _Outptr_ const OrtLogger** logger); - - /// @} - /// \name OrtKernelContext - /// Custom operator APIs. - /// @{ - - /** \brief Get the runtime logger from ::OrtKernelContext. - * - * Used in the KernelCompute callback of an OrtCustomOp to get a logger that can be used to log - * messages during inference. - * - * \param[in] context An instance of ::OrtKernelContext. - * \param[out] logger Pointer set to the kernel context's ::OrtLogger. Owned by ONNX Runtime, so do not free. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(KernelContext_GetLogger, _In_ const OrtKernelContext* context, _Outptr_ const OrtLogger** logger); - - /// @} - /// \name OrtLogger - /// Custom operator APIs. - /// @{ - - /** \brief Logs a message at the given severity level using the provided ::OrtLogger. - * - * Only messages with a severity level equal or greater than the ::OrtLogger's logging severity level - * are logged. Use OrtApi::Logger_GetLoggingSeverityLevel to get the ::OrtLogger's logging severity - * level. - * - * Can be used in custom operators to log messages with the logger retrieved via OrtApi::KernelInfo_GetLogger. - * - * \param[in] logger The ::OrtLogger instance. - * \param[in] log_severity_level The message's severity level. - * \param[in] message The message to log. - * \param[in] file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. - * \param[in] line_number The file line number in which the message is logged. Usually the value of __LINE__. - * \param[in] func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(Logger_LogMessage, _In_ const OrtLogger* logger, OrtLoggingLevel log_severity_level, - _In_z_ const char* message, _In_z_ const ORTCHAR_T* file_path, int line_number, - _In_z_ const char* func_name); - - /** \brief Get the logging severity level of the ::OrtLogger. - * - * Can be used in a custom operator to get the logging serverity level of the ::OrtLogger associated with - * the ::OrtKernelInfo. - * - * \param[in] logger The ::OrtLogger instance. - * \param[out] out Pointer to variable assigned with the logging severity level on success. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(Logger_GetLoggingSeverityLevel, _In_ const OrtLogger* logger, _Out_ OrtLoggingLevel* out); - - /// @} - - /** \brief Get a ::OrtValue tensor stored as a constant initializer in the graph node. - * - * Used in the CreateKernel callback of an OrtCustomOp to get a tensor value. - * - * \param[in] info ::OrtKernelInfo instance. - * \param[in] index The node index. - * \param[out] is_constant Is it a constant node input or not. - * \param[out] out The OrtValue tensor value. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(KernelInfoGetConstantInput_tensor, _In_ const OrtKernelInfo* info, size_t index, _Out_ int* is_constant, _Outptr_ const OrtValue** out); - - /** \brief Get Optional Type information from an ::OrtTypeInfo - * - * This augments ::OrtTypeInfo to return an ::OrtOptionalTypeInfo when the type is optional. - * The OrtOptionalTypeInfo also has a nested ::OrtTypeInfo that describes the type of the optional value. - * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. - * The actual OrtValues that are supplied in place of optional type inputs should contain - * specific type that is described by ::OrtOptionalTypeInfo. - * - * So the picture: ::OrtTypeInfo -> ::OrtOptionalTypeInfo -> ::OrtTypeInfo (describes the type that can be supplied - * in place of the optional type when creating the actual ::OrtValue). - * - * \param[in] type_info - * \param[out] out A pointer to the ::OrtOptionalTypeInfo. Do not free this value, - * it is owned by OrtTypeInfo instance. When the type_info does not represent - * optional type, nullptr is returned in out. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(CastTypeInfoToOptionalTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtOptionalTypeInfo** out); - - /** \brief Get OrtTypeInfo for the allowed contained type from an ::OrtOptionalTypeInfo. - * - * This augments ::OrtOptionalTypeInfo to return an ::OrtTypeInfo for the contained type. - * The OrtOptionalTypeInfo has a nested ::OrtTypeInfo that describes the type of the optional value. - * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. - * The actual OrtValues that are supplied in place of optional type inputs should contain - * specific type that is described by the returned ::OrtTypeInfo. - * - * \param[in] optional_type_info - * \param[out] out A pointer to the ::OrtTypeInfo for what the optional value could be. - * it is owned by OrtOptionalTypeInfo instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(GetOptionalContainedTypeInfo, _In_ const OrtOptionalTypeInfo* optional_type_info, - _Outptr_ OrtTypeInfo** out); - - /** \brief Set a single string in a string tensor - * Do not zero terminate the string data. - * - * \param[in] value A string tensor - * \param[in] index - flat index of the element - * \param[in] length_in_bytes length of the buffer in utf-8 bytes (without the null terminator) - * \param[inout] buffer - address of return value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetResizedStringTensorElementBuffer, _Inout_ OrtValue* value, _In_ size_t index, _In_ size_t length_in_bytes, _Inout_ char** buffer); - - /** \brief Get Allocator from KernelContext for a specific memoryInfo. Please use C API ReleaseAllocator to release out object - * - * \param[in] context OrtKernelContext instance - * \param[in] mem_info OrtMemoryInfo instance - * \param[out] out A pointer to OrtAllocator. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(KernelContext_GetAllocator, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _Outptr_ OrtAllocator** out); - - /** \brief Returns a null terminated string of the build info including git info and cxx flags - * - * \return UTF-8 encoded version string. Do not deallocate the returned buffer. - * - * \since Version 1.15. - */ - const char*(ORT_API_CALL* GetBuildInfoString)(void); - - /// \name OrtROCMProviderOptions - /// @{ - - /** \brief Create an OrtROCMProviderOptions - * - * \param[out] out Newly created ::OrtROCMProviderOptions. Must be released with OrtApi::ReleaseROCMProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.16. - */ - ORT_API2_STATUS(CreateROCMProviderOptions, _Outptr_ OrtROCMProviderOptions** out); - - /** \brief Set options in a ROCm Execution Provider. - * - * Please refer to https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html - * to know the available keys and values. Key should be in null terminated string format of the member of - * ::OrtROCMProviderOptions and value should be its related range. - * - * For example, key="device_id" and value="0" - * - * \param[in] rocm_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.16. - */ - ORT_API2_STATUS(UpdateROCMProviderOptions, _Inout_ OrtROCMProviderOptions* rocm_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** - * Get serialized ROCm provider options string. - * - * For example, "device_id=0;arena_extend_strategy=0;......" - * - * \param rocm_options - OrtROCMProviderOptions instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.16. - */ - ORT_API2_STATUS(GetROCMProviderOptionsAsString, _In_ const OrtROCMProviderOptions* rocm_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtROCMProviderOptions - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - * - * \since Version 1.16. - */ - void(ORT_API_CALL* ReleaseROCMProviderOptions)(_Frees_ptr_opt_ OrtROCMProviderOptions* input); - - /** \brief Create an allocator with specific type and register it with the ::OrtEnv - * This API enhance CreateAndRegisterAllocator that it can create an allocator with specific type, not just CPU allocator - * Enables sharing the allocator between multiple sessions that use the same env instance. - * Lifetime of the created allocator will be valid for the duration of the environment. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * \param[in] env OrtEnv instance - * \param[in] provider_type ExecutionProvider type - * \param[in] mem_info OrtMemoryInfo instance - * \param[in] arena_cfg Arena configuration - * \param[in] provider_options_keys key of the provider options map - * \param[in] provider_options_values value of the provider options map - * \param[in] num_keys Length of the provider options map - */ - ORT_API2_STATUS(CreateAndRegisterAllocatorV2, _Inout_ OrtEnv* env, _In_ const char* provider_type, _In_ const OrtMemoryInfo* mem_info, _In_ const OrtArenaCfg* arena_cfg, - _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys); - - /** \brief Run the model asynchronously in a thread owned by intra op thread pool - * - * \param[in] session - * \param[in] run_options If nullptr, will use a default ::OrtRunOptions - * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names - * \param[in] input Array of ::OrtValue%s of the input values - * \param[in] input_len Number of elements in the input_names and inputs arrays - * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names - * \param[in] output_names_len Number of elements in the output_names and outputs array - * \param[out] output OrtValue* array of size output_names_len. - * On calling RunAsync, output[i] could either be a null or a pointer to a preallocated OrtValue. - * Later, the output array will be passed to run_async_callback with all null(s) filled with valid - * OrtValue pointer(s) allocated by onnxruntime. - * NOTE: it is customer's duty to finally release the output array and each of its member, - * regardless of whether the member (OrtValue*) is allocated by onnxruntime or preallocated by the customer. - * \param[in] run_async_callback Callback function on model run completion - * \param[in] user_data User data that pass back to run_async_callback - */ - ORT_API2_STATUS(RunAsync, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, - _In_reads_(input_len) const char* const* input_names, - _In_reads_(input_len) const OrtValue* const* input, size_t input_len, - _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, - _Inout_updates_all_(output_names_len) OrtValue** output, - _In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data); - - /** - * Update TensorRT EP provider option where its data type is pointer, for example 'user_compute_stream'. - * If the data type of the provider option can be represented by string please use UpdateTensorRTProviderOptions. - * - * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. - * - * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance - * \param key - Name of the provider option - * \param value - A pointer to the instance that will be assigned to this provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value); - - /** - * Get TensorRT EP provider option where its data type is pointer. - * If the data type of the provider option can be represented by string please use GetTensorRTProviderOptionsAsString. - * - * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance - * \param key - Name of the provider option - * \param ptr - A pointer to the instance that is kept by the provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr); - - /** - * Update CUDA EP provider option where its data type is pointer, for example 'user_compute_stream'. - * If the data type of the provider option can be represented by string please use UpdateCUDAProviderOptions. - * - * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. - * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance - * \param key - Name of the provider option - * \param value - A pointer to the instance that will be assigned to this provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(UpdateCUDAProviderOptionsWithValue, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _In_ void* value); - - /** - * Get CUDA EP provider option where its data type is pointer. - * If the data type of the provider option can be represented by string please use GetCUDAProviderOptionsAsString. - * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance - * \param key - Name of the provider option - * \param ptr - A pointer to the instance that is kept by the provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(GetCUDAProviderOptionsByName, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _Outptr_ void** ptr); - - /** - * Get a EP resource. - * E.g. a cuda stream or a cublas handle - * - * \param context - Kernel context - * \param resouce_version - Version of the resource - * \param resource_id - Type of resource - * \param resource - A pointer to returned resource - * - * \since Version 1.16. - */ - ORT_API2_STATUS(KernelContext_GetResource, _In_ const OrtKernelContext* context, _In_ int resouce_version, _In_ int resource_id, _Outptr_ void** resource); - - /** \brief Set user logging function - * - * By default the logger created by the CreateEnv* functions is used to create the session logger as well. - * This function allows a user to override this default session logger with a logger of their own choosing. This way - * the user doesn't have to create a separate environment with a custom logger. This addresses the problem when - * the user already created an env but now wants to use a different logger for a specific session (for debugging or - * other reasons). - * - * \param[in] options - * \param[in] user_logging_function A pointer to a logging function. - * \param[in] user_logging_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `user_logging_function`. This parameter is optional. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.17. - */ - ORT_API2_STATUS(SetUserLoggingFunction, _Inout_ OrtSessionOptions* options, - _In_ OrtLoggingFunction user_logging_function, _In_opt_ void* user_logging_param); - - /** - * Get number of input from OrtShapeInferContext - * - * \param[in] context - * \param[out] out The number of inputs - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_GetInputCount, _In_ const OrtShapeInferContext* context, _Out_ size_t* out); - - /** - * Get type and shape info of an input - * - * \param[in] context - * \param[in] index The index of the input - * \param[out] info Type shape info of the input - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_GetInputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _Outptr_ OrtTensorTypeAndShapeInfo** info); - - /** - * Get attribute from OrtShapeInferContext. Note that OrtShapeInferContext is a per-node context, one could only read attribute from current node. - * - * \param[in] context - * \param[in] attr_name Name of the attribute - * \param[out] attr Handle of the attribute fetched - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_GetAttribute, _In_ const OrtShapeInferContext* context, _In_ const char* attr_name, _Outptr_ const OrtOpAttr** attr); - - /** - * Set type and shape info of an ouput - * - * \param[in] context - * \param[in] index The index of the ouput - * \param[out] info Type shape info of the output - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_SetOutputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _In_ const OrtTensorTypeAndShapeInfo* info); - - /** - * Set symbolic shape to type shape info - * - * \param[in] info Type shape info - * \param[in] dim_params Symbolic strings - * \param[in] dim_params_length Number of strings - * - * \since Version 1.17. - */ - ORT_API2_STATUS(SetSymbolicDimensions, _In_ OrtTensorTypeAndShapeInfo* info, _In_ const char* dim_params[], _In_ size_t dim_params_length); - - /** - * Read contents of an attribute to data - * - * \param[in] op_attr - * \param[in] type Attribute type - * \param[out] data Memory address to save raw content of the attribute - * \param[in] len Number of bytes allowed to store in data - * \param[out] out Number of bytes required to save the data when the call failed, or the real number of bytes saved to data on success - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ReadOpAttr, _In_ const OrtOpAttr* op_attr, _In_ OrtOpAttrType type, _Inout_ void* data, _In_ size_t len, _Out_ size_t* out); - - /** \brief Set whether to use deterministic compute. - * - * Default is false. If set to true, this will enable deterministic compute for GPU kernels where possible. - * Note that this most likely will have a performance cost. - * - * \param[in] options - * \param[in] value - * - * \since Version 1.17. - */ - ORT_API2_STATUS(SetDeterministicCompute, _Inout_ OrtSessionOptions* options, bool value); - - /** - * Run fn in parallel - * - * \param[in] context - * \param[in] fn Function accepting usr_data and an integer as iterator - * \param[in] total The number of times fn is to be invoked - * \param[in] num_batch Number of batches by which the "total" is to be divided in maximum. When zero, there is no limit - * \param[in] usr_data User data to be passed back to fn - * - * \since Version 1.17. - */ - ORT_API2_STATUS(KernelContext_ParallelFor, _In_ const OrtKernelContext* context, _In_ void (*fn)(void*, size_t), _In_ size_t total, _In_ size_t num_batch, _In_ void* usr_data); - - /** \brief Append OpenVINO execution provider to the session options - * - * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. - * - * \param[in] options - * \param[in] provider_options_keys - * \param[in] provider_options_values - * \param[in] num_keys - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO_V2, - _In_ OrtSessionOptions* options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Append VitisAI provider to session options - * - * If VitisAI is not available (due to a non VitisAI enabled build, or if VitisAI is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] provider_options_keys - * \param[in] provider_options_values - * \param[in] num_keys - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_VitisAI, - _In_ OrtSessionOptions* options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Get scratch buffer from the corresponding allocator under the sepcific OrtMemoryInfo object. - * NOTE: callers are responsible to release this scratch buffer from the corresponding allocator - * \param[in] context OrtKernelContext instance - * \param[in] mem_info OrtMemoryInfo instance - * \param[in] count_or_bytes How many bytes is this scratch buffer - * \param[out] out A pointer to the scrach buffer - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out); - - /** \brief Get allocator from KernelInfo for a specific memory type. Please use C API ReleaseAllocator to release out object - * - * \param[in] info OrtKernelInfo instance - * \param[in] mem_type OrtMemType object - * \param[out] out A pointer to OrtAllocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out); - - /** \brief Replace initialized Tensors with external data with the provided files in memory - * - * The function will find the initialized TensorProtos with external data in the graph with the provided - * external file names and the file content in memory. The API gets the external file name, offset, data length - * from TensorProto, and locate the tensor data from the file in memory buffer. - * It creates a Tensor to replace the existing Tensor in graph. The replacement - * will occur before any of the optimizations take place. The data will be copied into the graph - * since TensorProto can't refer to the user provided buffers. - * - * \param[in] session options - * \param[in] external_initializer_file_names Array of null terminated UTF-8 encoded strings of the file names - * which holds the external initializers. - * \param[in] external_initializer_file_buffer_array Array of pointers to the buffer of the file content. - * The buffer can be freed after session creation. - * \param[in] external_initializer_file_lengths Array of size_t to indicate the length of file content - * \param[in] num_external_initializer_files Number of external files - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddExternalInitializersFromFilesInMemory, _In_ OrtSessionOptions* options, - _In_reads_(num_external_initializer_files) const ORTCHAR_T* const* external_initializer_file_names, - _In_reads_(num_external_initializer_files) char* const* external_initializer_file_buffer_array, - _In_reads_(num_external_initializer_files) const size_t* external_initializer_file_lengths, - size_t num_external_initializer_files); -}; - -/* - * Steps to use a custom op: - * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops - * 2 Create an OrtCustomOp structure for each op and add them to the domain - * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options - */ - -// Specifies some characteristics of inputs/outputs of custom ops: -// Specify if the inputs/outputs are one of: -// 1) Non-optional (input/output must be present in the node) -// 2) Optional (input/output may be absent in the node) -// 3) Variadic: A variadic input or output specifies N (i.e., the minimum arity) or more operands. -// Only the last input or output of a custom op may be marked as variadic. -// The homogeneity of the variadic input or output determines whether all operands must be of the same -// tensor element type. -typedef enum OrtCustomOpInputOutputCharacteristic { - INPUT_OUTPUT_REQUIRED = 0, - INPUT_OUTPUT_OPTIONAL, - INPUT_OUTPUT_VARIADIC, -} OrtCustomOpInputOutputCharacteristic; - -/* - * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by - * the implementor of the custom op. - */ -struct OrtCustomOp { - uint32_t version; // Must be initialized to ORT_API_VERSION - - // This callback creates the kernel, which is a user defined - // parameter that is passed to the Kernel* callbacks below. It is - // recommended to use CreateKernelV2 which allows for a safe error - // propagation by returning an OrtStatusPtr. - void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, - _In_ const OrtKernelInfo* info); - - // Returns the name of the op - const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op); - - // Returns the type of the execution provider, return nullptr to use CPU execution provider - const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op); - - // Returns the count and types of the input & output tensors - ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op); - ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op); - - // Perform a computation step. It is recommended to use - // KernelComputeV2 which allows for a safe error propagation by - // returning an OrtStatusPtr. - void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context); - void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel); - - // Returns the characteristics of the input & output tensors - OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetInputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetOutputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - - // Returns the memory type of the input tensors. This API allows the custom op - // to place the inputs on specific devices. By default, it returns - // OrtMemTypeDefault, which means the input is placed on the default device for - // the execution provider. If the inputs need to be with different memory tyeps, - // this function can be overridden to return the specific memory types. - OrtMemType(ORT_API_CALL* GetInputMemoryType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - - // Returns the minimum number of input arguments expected for the variadic input. - // Applicable only for custom ops that have a variadic input. - int(ORT_API_CALL* GetVariadicInputMinArity)(_In_ const struct OrtCustomOp* op); - - // Returns true (non-zero) if all arguments of a variadic input have to be of the same type (homogeneous), - // and false (zero) otherwise. - // Applicable only for custom ops that have a variadic input. - int(ORT_API_CALL* GetVariadicInputHomogeneity)(_In_ const struct OrtCustomOp* op); - - // Returns the minimum number of output values expected for the variadic output. - // Applicable only for custom ops that have a variadic output. - int(ORT_API_CALL* GetVariadicOutputMinArity)(_In_ const struct OrtCustomOp* op); - - // Returns true (non-zero) if all outputs values of a variadic output have to be of the same type (homogeneous), - // and false (zero) otherwise. - // Applicable only for custom ops that have a variadic output. - int(ORT_API_CALL* GetVariadicOutputHomogeneity)(_In_ const struct OrtCustomOp* op); - - // Create the kernel state which is passed to each compute call. - OrtStatusPtr(ORT_API_CALL* CreateKernelV2)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, - _In_ const OrtKernelInfo* info, - _Out_ void** kernel); - - // Perform the computation step. - OrtStatusPtr(ORT_API_CALL* KernelComputeV2)(_In_ void* op_kernel, _In_ OrtKernelContext* context); - - OrtStatusPtr(ORT_API_CALL* InferOutputShapeFn)(_In_ const struct OrtCustomOp* op, _In_ OrtShapeInferContext*); - - // Get start range - int(ORT_API_CALL* GetStartVersion)(_In_ const struct OrtCustomOp* op); - int(ORT_API_CALL* GetEndVersion)(_In_ const struct OrtCustomOp* op); - - // Get the inplace_map that defines which output can reuse which input - // Callers will provide 2 raw int* and pass in their address, this function will fill these 2 arrays - // when return, output (*output_index)[i] may reuse the input (*input_index[i]). - // The return value is the size of these 2 arrays. - // Callers are responsible to delete these 2 arrays after use by calling OrtCustomOp::ReleaseMayInplace(). - size_t(ORT_API_CALL* GetMayInplace)(_Out_ int** input_index, _Out_ int** output_index); - - // Release the pointer input_index and output_index allocated from GetMayInplace() function. - // If GetMayInplace() is defined, this function MUST be defined as well. - void(ORT_API_CALL* ReleaseMayInplace)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); - - // Same as GetMayInplace() and ReleaseMayInplace() - size_t(ORT_API_CALL* GetAliasMap)(_Out_ int** input_index, _Out_ int** output_index); - void(ORT_API_CALL* ReleaseAliasMap)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); -}; - -/* - * This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists - * - * \param device_id CUDA device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id); - -/* - * This is the old way to add the ROCm provider to the session, please use - * SessionOptionsAppendExecutionProvider_ROCM above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with - * HIP support and the ROCm provider shared library exists - * - * \param device_id HIP device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id); - -/* - * This is the old way to add the MIGraphX provider to the session, please use - * SessionOptionsAppendExecutionProvider_MIGraphX above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with - * HIP support and the MIGraphX provider shared library exists - * - * \param device_id HIP device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, int device_id); - -/* - * This is the old way to add the oneDNN provider to the session, please use - * SessionOptionsAppendExecutionProvider_oneDNN above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with - * oneDNN support and the oneDNN provider shared library exists - * - * \param use_arena zero: false. non-zero: true. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena); - -/* - * This is the old way to add the TensorRT provider to the session, please use SessionOptionsAppendExecutionProvider_TensorRT_V2 above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with TensorRT support and the TensorRT provider shared library exists - * - * \param device_id CUDA device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id); - -#ifdef __cplusplus -} -#endif -/// @} diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_cxx_api.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_cxx_api.h deleted file mode 100644 index 8091fd4..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_cxx_api.h +++ /dev/null @@ -1,2387 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Summary: The Ort C++ API is a header only wrapper around the Ort C API. -// -// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors -// and automatically releasing resources in the destructors. The primary purpose of C++ API is exception safety so -// all the resources follow RAII and do not leak memory. -// -// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers. -// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). However, you can't use them -// until you assign an instance that actually holds an underlying object. -// -// For Ort objects only move assignment between objects is allowed, there are no copy constructors. -// Some objects have explicit 'Clone' methods for this purpose. -// -// ConstXXXX types are copyable since they do not own the underlying C object, so you can pass them to functions as arguments -// by value or by reference. ConstXXXX types are restricted to const only interfaces. -// -// UnownedXXXX are similar to ConstXXXX but also allow non-const interfaces. -// -// The lifetime of the corresponding owning object must eclipse the lifetimes of the ConstXXXX/UnownedXXXX types. They exists so you do not -// have to fallback to C types and the API with the usual pitfalls. In general, do not use C API from your C++ code. - -#pragma once -#include "onnxruntime_c_api.h" -#include "onnxruntime_float16.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef ORT_NO_EXCEPTIONS -#include -#endif - -/** \brief All C++ Onnxruntime APIs are defined inside this namespace - * - */ -namespace Ort { - -/** \brief All C++ methods that can fail will throw an exception of this type - * - * If ORT_NO_EXCEPTIONS is defined, then any error will result in a call to abort() - */ -struct Exception : std::exception { - Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {} - - OrtErrorCode GetOrtErrorCode() const { return code_; } - const char* what() const noexcept override { return message_.c_str(); } - - private: - std::string message_; - OrtErrorCode code_; -}; - -#ifdef ORT_NO_EXCEPTIONS -// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors. -// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW -#ifndef ORT_CXX_API_THROW -#define ORT_CXX_API_THROW(string, code) \ - do { \ - std::cerr << Ort::Exception(string, code) \ - .what() \ - << std::endl; \ - abort(); \ - } while (false) -#endif -#else -#define ORT_CXX_API_THROW(string, code) \ - throw Ort::Exception(string, code) -#endif - -// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi, -// it's in a template so that we can define a global variable in a header and make -// it transparent to the users of the API. -template -struct Global { - static const OrtApi* api_; -}; - -// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it. -template -#ifdef ORT_API_MANUAL_INIT -const OrtApi* Global::api_{}; -inline void InitApi() noexcept { Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); } - -// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is -// required by C++ APIs. -// -// Example mycustomop.cc: -// -// #define ORT_API_MANUAL_INIT -// #include -// #undef ORT_API_MANUAL_INIT -// -// OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) { -// Ort::InitApi(api_base->GetApi(ORT_API_VERSION)); -// // ... -// } -// -inline void InitApi(const OrtApi* api) noexcept { Global::api_ = api; } -#else -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(push) -// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers. -// Please define ORT_API_MANUAL_INIT if it conerns you. -#pragma warning(disable : 26426) -#endif -const OrtApi* Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(pop) -#endif -#endif - -/// This returns a reference to the OrtApi interface in use -inline const OrtApi& GetApi() noexcept { return *Global::api_; } - -/// -/// This function returns the onnxruntime version string -/// -/// version string major.minor.rev -std::string GetVersionString(); - -/// -/// This function returns the onnxruntime build information: including git branch, -/// git commit id, build type(Debug/Release/RelWithDebInfo) and cmake cpp flags. -/// -/// string -std::string GetBuildInfoString(); - -/// -/// This is a C++ wrapper for OrtApi::GetAvailableProviders() and -/// returns a vector of strings representing the available execution providers. -/// -/// vector of strings -std::vector GetAvailableProviders(); - -/** \brief IEEE 754 half-precision floating point data type - * - * \details This struct is used for converting float to float16 and back - * so the user could feed inputs and fetch outputs using these type. - * - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data. - * - * \code{.unparsed} - * // This example demonstrates converion from float to float16 - * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; - * std::vector fp16_values; - * fp16_values.reserve(std::size(values)); - * std::transform(std::begin(values), std::end(values), std::back_inserter(fp16_values), - * [](float value) { return Ort::Float16_t(value); }); - * - * \endcode - */ -struct Float16_t : onnxruntime_float16::Float16Impl { - private: - /// - /// Constructor from a 16-bit representation of a float16 value - /// No conversion is done here. - /// - /// 16-bit representation - constexpr explicit Float16_t(uint16_t v) noexcept { val = v; } - - public: - using Base = onnxruntime_float16::Float16Impl; - - /// - /// Default constructor - /// - Float16_t() = default; - - /// - /// Explicit conversion to uint16_t representation of float16. - /// - /// uint16_t bit representation of float16 - /// new instance of Float16_t - constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); } - - /// - /// __ctor from float. Float is converted into float16 16-bit representation. - /// - /// float value - explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); } - - /// - /// Converts float16 to float - /// - /// float representation of float16 value - float ToFloat() const noexcept { return Base::ToFloatImpl(); } - - /// - /// Checks if the value is negative - /// - /// true if negative - using Base::IsNegative; - - /// - /// Tests if the value is NaN - /// - /// true if NaN - using Base::IsNaN; - - /// - /// Tests if the value is finite - /// - /// true if finite - using Base::IsFinite; - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - using Base::IsPositiveInfinity; - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - using Base::IsNegativeInfinity; - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - using Base::IsInfinity; - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - using Base::IsNaNOrZero; - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - using Base::IsNormal; - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - using Base::IsSubnormal; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - using Base::Abs; - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - using Base::Negate; - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - using Base::AreZero; - - /// - /// User defined conversion operator. Converts Float16_t to float. - /// - explicit operator float() const noexcept { return ToFloat(); } - - using Base::operator==; - using Base::operator!=; - using Base::operator<; -}; - -static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match"); - -/** \brief bfloat16 (Brain Floating Point) data type - * - * \details This struct is used for converting float to bfloat16 and back - * so the user could feed inputs and fetch outputs using these type. - * - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data. - * - * \code{.unparsed} - * // This example demonstrates converion from float to float16 - * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; - * std::vector bfp16_values; - * bfp16_values.reserve(std::size(values)); - * std::transform(std::begin(values), std::end(values), std::back_inserter(bfp16_values), - * [](float value) { return Ort::BFloat16_t(value); }); - * - * \endcode - */ -struct BFloat16_t : onnxruntime_float16::BFloat16Impl { - private: - /// - /// Constructor from a uint16_t representation of bfloat16 - /// used in FromBits() to escape overload resolution issue with - /// constructor from float. - /// No conversion is done. - /// - /// 16-bit bfloat16 value - constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; } - - public: - using Base = onnxruntime_float16::BFloat16Impl; - - BFloat16_t() = default; - - /// - /// Explicit conversion to uint16_t representation of bfloat16. - /// - /// uint16_t bit representation of bfloat16 - /// new instance of BFloat16_t - static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); } - - /// - /// __ctor from float. Float is converted into bfloat16 16-bit representation. - /// - /// float value - explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); } - - /// - /// Converts bfloat16 to float - /// - /// float representation of bfloat16 value - float ToFloat() const noexcept { return Base::ToFloatImpl(); } - - /// - /// Checks if the value is negative - /// - /// true if negative - using Base::IsNegative; - - /// - /// Tests if the value is NaN - /// - /// true if NaN - using Base::IsNaN; - - /// - /// Tests if the value is finite - /// - /// true if finite - using Base::IsFinite; - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - using Base::IsPositiveInfinity; - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - using Base::IsNegativeInfinity; - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - using Base::IsInfinity; - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - using Base::IsNaNOrZero; - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - using Base::IsNormal; - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - using Base::IsSubnormal; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - using Base::Abs; - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - using Base::Negate; - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - using Base::AreZero; - - /// - /// User defined conversion operator. Converts BFloat16_t to float. - /// - explicit operator float() const noexcept { return ToFloat(); } - - // We do not have an inherited impl for the below operators - // as the internal class implements them a little differently - bool operator==(const BFloat16_t& rhs) const noexcept; - bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); } - bool operator<(const BFloat16_t& rhs) const noexcept; -}; - -static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); - -/** \brief float8e4m3fn (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E4M3FN_t { - uint8_t value; - constexpr Float8E4M3FN_t() noexcept : value(0) {} - constexpr Float8E4M3FN_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E4M3FN_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E4M3FN_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E4M3FN_t) == sizeof(uint8_t), "Sizes must match"); - -/** \brief float8e4m3fnuz (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E4M3FNUZ_t { - uint8_t value; - constexpr Float8E4M3FNUZ_t() noexcept : value(0) {} - constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E4M3FNUZ_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E4M3FNUZ_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E4M3FNUZ_t) == sizeof(uint8_t), "Sizes must match"); - -/** \brief float8e5m2 (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E5M2_t { - uint8_t value; - constexpr Float8E5M2_t() noexcept : value(0) {} - constexpr Float8E5M2_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E5M2_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E5M2_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E5M2_t) == sizeof(uint8_t), "Sizes must match"); - -/** \brief float8e5m2fnuz (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E5M2FNUZ_t { - uint8_t value; - constexpr Float8E5M2FNUZ_t() noexcept : value(0) {} - constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E5M2FNUZ_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E5M2FNUZ_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E5M2FNUZ_t) == sizeof(uint8_t), "Sizes must match"); - -namespace detail { -// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type -// This can't be done in the C API since C doesn't have function overloading. -#define ORT_DEFINE_RELEASE(NAME) \ - inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); } - -ORT_DEFINE_RELEASE(Allocator); -ORT_DEFINE_RELEASE(MemoryInfo); -ORT_DEFINE_RELEASE(CustomOpDomain); -ORT_DEFINE_RELEASE(ThreadingOptions); -ORT_DEFINE_RELEASE(Env); -ORT_DEFINE_RELEASE(RunOptions); -ORT_DEFINE_RELEASE(Session); -ORT_DEFINE_RELEASE(SessionOptions); -ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo); -ORT_DEFINE_RELEASE(SequenceTypeInfo); -ORT_DEFINE_RELEASE(MapTypeInfo); -ORT_DEFINE_RELEASE(TypeInfo); -ORT_DEFINE_RELEASE(Value); -ORT_DEFINE_RELEASE(ModelMetadata); -ORT_DEFINE_RELEASE(IoBinding); -ORT_DEFINE_RELEASE(ArenaCfg); -ORT_DEFINE_RELEASE(Status); -ORT_DEFINE_RELEASE(OpAttr); -ORT_DEFINE_RELEASE(Op); -ORT_DEFINE_RELEASE(KernelInfo); - -#undef ORT_DEFINE_RELEASE - -/** \brief This is a tagging template type. Use it with Base to indicate that the C++ interface object - * has no ownership of the underlying C object. - */ -template -struct Unowned { - using Type = T; -}; - -/** \brief Used internally by the C++ API. C++ wrapper types inherit from this. - * This is a zero cost abstraction to wrap the C API objects and delete them on destruction. - * - * All of the C++ classes - * a) serve as containers for pointers to objects that are created by the underlying C API. - * Their size is just a pointer size, no need to dynamically allocate them. Use them by value. - * b) Each of struct XXXX, XXX instances function as smart pointers to the underlying C API objects. - * they would release objects owned automatically when going out of scope, they are move-only. - * c) ConstXXXX and UnownedXXX structs function as non-owning, copyable containers for the above pointers. - * ConstXXXX allow calling const interfaces only. They give access to objects that are owned by somebody else - * such as Onnxruntime or instances of XXXX classes. - * d) serve convenient interfaces that return C++ objects and further enhance exception and type safety so they can be used - * in C++ code. - * - */ - -/// -/// This is a non-const pointer holder that is move-only. Disposes of the pointer on destruction. -/// -template -struct Base { - using contained_type = T; - - constexpr Base() = default; - constexpr explicit Base(contained_type* p) noexcept : p_{p} {} - ~Base() { OrtRelease(p_); } - - Base(const Base&) = delete; - Base& operator=(const Base&) = delete; - - Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } - Base& operator=(Base&& v) noexcept { - OrtRelease(p_); - p_ = v.release(); - return *this; - } - - constexpr operator contained_type*() const noexcept { return p_; } - - /// \brief Relinquishes ownership of the contained C object pointer - /// The underlying object is not destroyed - contained_type* release() { - T* p = p_; - p_ = nullptr; - return p; - } - - protected: - contained_type* p_{}; -}; - -// Undefined. For const types use Base> -template -struct Base; - -/// -/// Covers unowned pointers owned by either the ORT -/// or some other instance of CPP wrappers. -/// Used for ConstXXX and UnownedXXXX types that are copyable. -/// Also convenient to wrap raw OrtXX pointers . -/// -/// -template -struct Base> { - using contained_type = typename Unowned::Type; - - constexpr Base() = default; - constexpr explicit Base(contained_type* p) noexcept : p_{p} {} - - ~Base() = default; - - Base(const Base&) = default; - Base& operator=(const Base&) = default; - - Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } - Base& operator=(Base&& v) noexcept { - p_ = nullptr; - std::swap(p_, v.p_); - return *this; - } - - constexpr operator contained_type*() const noexcept { return p_; } - - protected: - contained_type* p_{}; -}; - -// Light functor to release memory with OrtAllocator -struct AllocatedFree { - OrtAllocator* allocator_; - explicit AllocatedFree(OrtAllocator* allocator) - : allocator_(allocator) {} - void operator()(void* ptr) const { - if (ptr) allocator_->Free(allocator_, ptr); - } -}; - -} // namespace detail - -struct AllocatorWithDefaultOptions; -struct Env; -struct TypeInfo; -struct Value; -struct ModelMetadata; - -/** \brief unique_ptr typedef used to own strings allocated by OrtAllocators - * and release them at the end of the scope. The lifespan of the given allocator - * must eclipse the lifespan of AllocatedStringPtr instance - */ -using AllocatedStringPtr = std::unique_ptr; - -/** \brief The Status that holds ownership of OrtStatus received from C API - * Use it to safely destroy OrtStatus* returned from the C API. Use appropriate - * constructors to construct an instance of a Status object from exceptions. - */ -struct Status : detail::Base { - explicit Status(std::nullptr_t) noexcept {} ///< Create an empty object, must be assigned a valid one to be used - explicit Status(OrtStatus* status) noexcept; ///< Takes ownership of OrtStatus instance returned from the C API. - explicit Status(const Exception&) noexcept; ///< Creates status instance out of exception - explicit Status(const std::exception&) noexcept; ///< Creates status instance out of exception - Status(const char* message, OrtErrorCode code) noexcept; ///< Creates status instance out of null-terminated string message. - std::string GetErrorMessage() const; - OrtErrorCode GetErrorCode() const; - bool IsOK() const noexcept; ///< Returns true if instance represents an OK (non-error) status. -}; - -/** \brief The ThreadingOptions - * - * The ThreadingOptions used for set global threadpools' options of The Env. - */ -struct ThreadingOptions : detail::Base { - /// \brief Wraps OrtApi::CreateThreadingOptions - ThreadingOptions(); - - /// \brief Wraps OrtApi::SetGlobalIntraOpNumThreads - ThreadingOptions& SetGlobalIntraOpNumThreads(int intra_op_num_threads); - - /// \brief Wraps OrtApi::SetGlobalInterOpNumThreads - ThreadingOptions& SetGlobalInterOpNumThreads(int inter_op_num_threads); - - /// \brief Wraps OrtApi::SetGlobalSpinControl - ThreadingOptions& SetGlobalSpinControl(int allow_spinning); - - /// \brief Wraps OrtApi::SetGlobalDenormalAsZero - ThreadingOptions& SetGlobalDenormalAsZero(); - - /// \brief Wraps OrtApi::SetGlobalCustomCreateThreadFn - ThreadingOptions& SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /// \brief Wraps OrtApi::SetGlobalCustomThreadCreationOptions - ThreadingOptions& SetGlobalCustomThreadCreationOptions(void* ort_custom_thread_creation_options); - - /// \brief Wraps OrtApi::SetGlobalCustomJoinThreadFn - ThreadingOptions& SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); -}; - -/** \brief The Env (Environment) - * - * The Env holds the logging state used by all other objects. - * Note: One Env must be created before using any other Onnxruntime functionality - */ -struct Env : detail::Base { - explicit Env(std::nullptr_t) {} ///< Create an empty Env object, must be assigned a valid one to be used - - /// \brief Wraps OrtApi::CreateEnv - Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief Wraps OrtApi::CreateEnvWithCustomLogger - Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param); - - /// \brief Wraps OrtApi::CreateEnvWithGlobalThreadPools - Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief Wraps OrtApi::CreateEnvWithCustomLoggerAndGlobalThreadPools - Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, - OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief C Interop Helper - explicit Env(OrtEnv* p) : Base{p} {} - - Env& EnableTelemetryEvents(); ///< Wraps OrtApi::EnableTelemetryEvents - Env& DisableTelemetryEvents(); ///< Wraps OrtApi::DisableTelemetryEvents - - Env& UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level); ///< Wraps OrtApi::UpdateEnvWithCustomLogLevel - - Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocator - - Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, const std::unordered_map& options, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocatorV2 -}; - -/** \brief Custom Op Domain - * - */ -struct CustomOpDomain : detail::Base { - explicit CustomOpDomain(std::nullptr_t) {} ///< Create an empty CustomOpDomain object, must be assigned a valid one to be used - - /// \brief Wraps OrtApi::CreateCustomOpDomain - explicit CustomOpDomain(const char* domain); - - // This does not take ownership of the op, simply registers it. - void Add(const OrtCustomOp* op); ///< Wraps CustomOpDomain_Add -}; - -/** \brief RunOptions - * - */ -struct RunOptions : detail::Base { - explicit RunOptions(std::nullptr_t) {} ///< Create an empty RunOptions object, must be assigned a valid one to be used - RunOptions(); ///< Wraps OrtApi::CreateRunOptions - - RunOptions& SetRunLogVerbosityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel - int GetRunLogVerbosityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel - - RunOptions& SetRunLogSeverityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogSeverityLevel - int GetRunLogSeverityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogSeverityLevel - - RunOptions& SetRunTag(const char* run_tag); ///< wraps OrtApi::RunOptionsSetRunTag - const char* GetRunTag() const; ///< Wraps OrtApi::RunOptionsGetRunTag - - RunOptions& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddRunConfigEntry - - /** \brief Terminates all currently executing Session::Run calls that were made using this RunOptions instance - * - * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error - * Wraps OrtApi::RunOptionsSetTerminate - */ - RunOptions& SetTerminate(); - - /** \brief Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without it instantly terminating - * - * Wraps OrtApi::RunOptionsUnsetTerminate - */ - RunOptions& UnsetTerminate(); -}; - -namespace detail { -// Utility function that returns a SessionOption config entry key for a specific custom operator. -// Ex: custom_op.[custom_op_name].[config] -std::string MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config); -} // namespace detail - -/// -/// Class that represents session configuration entries for one or more custom operators. -/// -/// Example: -/// Ort::CustomOpConfigs op_configs; -/// op_configs.AddConfig("my_custom_op", "device_type", "CPU"); -/// -/// Passed to Ort::SessionOptions::RegisterCustomOpsLibrary. -/// -struct CustomOpConfigs { - CustomOpConfigs() = default; - ~CustomOpConfigs() = default; - CustomOpConfigs(const CustomOpConfigs&) = default; - CustomOpConfigs& operator=(const CustomOpConfigs&) = default; - CustomOpConfigs(CustomOpConfigs&& o) = default; - CustomOpConfigs& operator=(CustomOpConfigs&& o) = default; - - /** \brief Adds a session configuration entry/value for a specific custom operator. - * - * \param custom_op_name The name of the custom operator for which to add a configuration entry. - * Must match the name returned by the CustomOp's GetName() method. - * \param config_key The name of the configuration entry. - * \param config_value The value of the configuration entry. - * \return A reference to this object to enable call chaining. - */ - CustomOpConfigs& AddConfig(const char* custom_op_name, const char* config_key, const char* config_value); - - /** \brief Returns a flattened map of custom operator configuration entries and their values. - * - * The keys has been flattened to include both the custom operator name and the configuration entry key name. - * For example, a prior call to AddConfig("my_op", "key", "value") corresponds to the flattened key/value pair - * {"my_op.key", "value"}. - * - * \return An unordered map of flattened configurations. - */ - const std::unordered_map& GetFlattenedConfigs() const; - - private: - std::unordered_map flat_configs_; -}; - -/** \brief Options object used when creating a new Session object - * - * Wraps ::OrtSessionOptions object and methods - */ - -struct SessionOptions; - -namespace detail { -// we separate const-only methods because passing const ptr to non-const methods -// is only discovered when inline methods are compiled which is counter-intuitive -template -struct ConstSessionOptionsImpl : Base { - using B = Base; - using B::B; - - SessionOptions Clone() const; ///< Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions - - std::string GetConfigEntry(const char* config_key) const; ///< Wraps OrtApi::GetSessionConfigEntry - bool HasConfigEntry(const char* config_key) const; ///< Wraps OrtApi::HasSessionConfigEntry - std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def); -}; - -template -struct SessionOptionsImpl : ConstSessionOptionsImpl { - using B = ConstSessionOptionsImpl; - using B::B; - - SessionOptionsImpl& SetIntraOpNumThreads(int intra_op_num_threads); ///< Wraps OrtApi::SetIntraOpNumThreads - SessionOptionsImpl& SetInterOpNumThreads(int inter_op_num_threads); ///< Wraps OrtApi::SetInterOpNumThreads - SessionOptionsImpl& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); ///< Wraps OrtApi::SetSessionGraphOptimizationLevel - SessionOptionsImpl& SetDeterministicCompute(bool value); ///< Wraps OrtApi::SetDeterministicCompute - - SessionOptionsImpl& EnableCpuMemArena(); ///< Wraps OrtApi::EnableCpuMemArena - SessionOptionsImpl& DisableCpuMemArena(); ///< Wraps OrtApi::DisableCpuMemArena - - SessionOptionsImpl& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file); ///< Wraps OrtApi::SetOptimizedModelFilePath - - SessionOptionsImpl& EnableProfiling(const ORTCHAR_T* profile_file_prefix); ///< Wraps OrtApi::EnableProfiling - SessionOptionsImpl& DisableProfiling(); ///< Wraps OrtApi::DisableProfiling - - SessionOptionsImpl& EnableOrtCustomOps(); ///< Wraps OrtApi::EnableOrtCustomOps - - SessionOptionsImpl& EnableMemPattern(); ///< Wraps OrtApi::EnableMemPattern - SessionOptionsImpl& DisableMemPattern(); ///< Wraps OrtApi::DisableMemPattern - - SessionOptionsImpl& SetExecutionMode(ExecutionMode execution_mode); ///< Wraps OrtApi::SetSessionExecutionMode - - SessionOptionsImpl& SetLogId(const char* logid); ///< Wraps OrtApi::SetSessionLogId - SessionOptionsImpl& SetLogSeverityLevel(int level); ///< Wraps OrtApi::SetSessionLogSeverityLevel - - SessionOptionsImpl& Add(OrtCustomOpDomain* custom_op_domain); ///< Wraps OrtApi::AddCustomOpDomain - - SessionOptionsImpl& DisablePerSessionThreads(); ///< Wraps OrtApi::DisablePerSessionThreads - - SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddSessionConfigEntry - - SessionOptionsImpl& AddInitializer(const char* name, const OrtValue* ort_val); ///< Wraps OrtApi::AddInitializer - SessionOptionsImpl& AddExternalInitializers(const std::vector& names, const std::vector& ort_values); ///< Wraps OrtApi::AddExternalInitializers - SessionOptionsImpl& AddExternalInitializersFromFilesInMemory(const std::vector>& external_initializer_file_names, - const std::vector& external_initializer_file_buffer_array, - const std::vector& external_initializer_file_lengths); ///< Wraps OrtApi::AddExternalInitializersFromFilesInMemory - - SessionOptionsImpl& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA - SessionOptionsImpl& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA_V2 - SessionOptionsImpl& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_ROCM - SessionOptionsImpl& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO_V2 - SessionOptionsImpl& AppendExecutionProvider_OpenVINO_V2(const std::unordered_map& provider_options = {}); - SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN - SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options); - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl - SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options); - /// Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports QNN, SNPE and XNNPACK. - SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name, - const std::unordered_map& provider_options = {}); - - SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn - SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options); ///< Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions - SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn - - ///< Registers the custom operator from the specified shared library via OrtApi::RegisterCustomOpsLibrary_V2. - ///< The custom operator configurations are optional. If provided, custom operator configs are set via - ///< OrtApi::AddSessionConfigEntry. - SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, const CustomOpConfigs& custom_op_configs = {}); - - SessionOptionsImpl& RegisterCustomOpsUsingFunction(const char* function_name); ///< Wraps OrtApi::RegisterCustomOpsUsingFunction - - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_VitisAI - SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map& provider_options = {}); -}; -} // namespace detail - -using UnownedSessionOptions = detail::SessionOptionsImpl>; -using ConstSessionOptions = detail::ConstSessionOptionsImpl>; - -/** \brief Wrapper around ::OrtSessionOptions - * - */ -struct SessionOptions : detail::SessionOptionsImpl { - explicit SessionOptions(std::nullptr_t) {} ///< Create an empty SessionOptions object, must be assigned a valid one to be used - SessionOptions(); ///< Wraps OrtApi::CreateSessionOptions - explicit SessionOptions(OrtSessionOptions* p) : SessionOptionsImpl{p} {} ///< Used for interop with the C API - UnownedSessionOptions GetUnowned() const { return UnownedSessionOptions{this->p_}; } - ConstSessionOptions GetConst() const { return ConstSessionOptions{this->p_}; } -}; - -/** \brief Wrapper around ::OrtModelMetadata - * - */ -struct ModelMetadata : detail::Base { - explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used - explicit ModelMetadata(OrtModelMetadata* p) : Base{p} {} ///< Used for interop with the C API - - /** \brief Returns a copy of the producer name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetProducerNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetProducerName - - /** \brief Returns a copy of the graph name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetGraphNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphName - - /** \brief Returns a copy of the domain name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetDomainAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDomain - - /** \brief Returns a copy of the description. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDescription - - /** \brief Returns a copy of the graph description. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphDescription - - /** \brief Returns a vector of copies of the custom metadata keys. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance std::vector of smart pointers that would deallocate the buffers when out of scope. - * The OrtAllocator instance must be valid at the point of memory release. - */ - std::vector GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetCustomMetadataMapKeys - - /** \brief Looks up a value by a key in the Custom Metadata map - * - * \param key zero terminated string key to lookup - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * maybe nullptr if key is not found. - * - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataLookupCustomMetadataMap - - int64_t GetVersion() const; ///< Wraps OrtApi::ModelMetadataGetVersion -}; - -struct IoBinding; - -namespace detail { - -// we separate const-only methods because passing const ptr to non-const methods -// is only discovered when inline methods are compiled which is counter-intuitive -template -struct ConstSessionImpl : Base { - using B = Base; - using B::B; - - size_t GetInputCount() const; ///< Returns the number of model inputs - size_t GetOutputCount() const; ///< Returns the number of model outputs - size_t GetOverridableInitializerCount() const; ///< Returns the number of inputs that have defaults that can be overridden - - /** \brief Returns a copy of input name at the specified index. - * - * \param index must less than the value returned by GetInputCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator* allocator) const; - - /** \brief Returns a copy of output name at then specified index. - * - * \param index must less than the value returned by GetOutputCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const; - - /** \brief Returns a copy of the overridable initializer name at then specified index. - * - * \param index must less than the value returned by GetOverridableInitializerCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionGetOverridableInitializerName - - uint64_t GetProfilingStartTimeNs() const; ///< Wraps OrtApi::SessionGetProfilingStartTimeNs - ModelMetadata GetModelMetadata() const; ///< Wraps OrtApi::SessionGetModelMetadata - - TypeInfo GetInputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetInputTypeInfo - TypeInfo GetOutputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOutputTypeInfo - TypeInfo GetOverridableInitializerTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOverridableInitializerTypeInfo -}; - -template -struct SessionImpl : ConstSessionImpl { - using B = ConstSessionImpl; - using B::B; - - /** \brief Run the model returning results in an Ort allocated vector. - * - * Wraps OrtApi::Run - * - * The caller provides a list of inputs and a list of the desired outputs to return. - * - * See the output logs for more information on warnings/errors that occur while processing the model. - * Common errors are.. (TODO) - * - * \param[in] run_options - * \param[in] input_names Array of null terminated strings of length input_count that is the list of input names - * \param[in] input_values Array of Value objects of length input_count that is the list of input values - * \param[in] input_count Number of inputs (the size of the input_names & input_values arrays) - * \param[in] output_names Array of C style strings of length output_count that is the list of output names - * \param[in] output_count Number of outputs (the size of the output_names array) - * \return A std::vector of Value objects that directly maps to the output_names array (eg. output_name[0] is the first entry of the returned vector) - */ - std::vector Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, size_t output_count); - - /** \brief Run the model returning results in user provided outputs - * Same as Run(const RunOptions&, const char* const*, const Value*, size_t,const char* const*, size_t) - */ - void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count); - - void Run(const RunOptions& run_options, const IoBinding&); ///< Wraps OrtApi::RunWithBinding - - /** \brief Run the model asynchronously in a thread owned by intra op thread pool - * - * Wraps OrtApi::RunAsync - * - * \param[in] run_options - * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names - * \param[in] input_values Array of Value objects of length input_count - * \param[in] input_count Number of elements in the input_names and inputs arrays - * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names - * \param[out] output_values Array of provided Values to be filled with outputs. - * On calling RunAsync, output_values[i] could either be initialized by a null pointer or a preallocated OrtValue*. - * Later, on invoking the callback, each output_values[i] of null will be filled with an OrtValue* allocated by onnxruntime. - * Then, an OrtValue** pointer will be casted from output_values, and pass to the callback. - * NOTE: it is customer's duty to finally release output_values and each of its member, - * regardless of whether the member (Ort::Value) is allocated by onnxruntime or preallocated by the customer. - * \param[in] output_count Number of elements in the output_names and outputs array - * \param[in] callback Callback function on model run completion - * \param[in] user_data User data that pass back to the callback - */ - void RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data); - - /** \brief End profiling and return a copy of the profiling file name. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr EndProfilingAllocated(OrtAllocator* allocator); ///< Wraps OrtApi::SessionEndProfiling -}; - -} // namespace detail - -using ConstSession = detail::ConstSessionImpl>; -using UnownedSession = detail::SessionImpl>; - -/** \brief Wrapper around ::OrtSession - * - */ -struct Session : detail::SessionImpl { - explicit Session(std::nullptr_t) {} ///< Create an empty Session object, must be assigned a valid one to be used - Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); ///< Wraps OrtApi::CreateSession - Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container); ///< Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer - Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options); ///< Wraps OrtApi::CreateSessionFromArray - Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container); ///< Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer - - ConstSession GetConst() const { return ConstSession{this->p_}; } - UnownedSession GetUnowned() const { return UnownedSession{this->p_}; } -}; - -namespace detail { -template -struct MemoryInfoImpl : Base { - using B = Base; - using B::B; - - std::string GetAllocatorName() const; - OrtAllocatorType GetAllocatorType() const; - int GetDeviceId() const; - OrtMemoryInfoDeviceType GetDeviceType() const; - OrtMemType GetMemoryType() const; - - template - bool operator==(const MemoryInfoImpl& o) const; -}; -} // namespace detail - -// Const object holder that does not own the underlying object -using ConstMemoryInfo = detail::MemoryInfoImpl>; - -/** \brief Wrapper around ::OrtMemoryInfo - * - */ -struct MemoryInfo : detail::MemoryInfoImpl { - static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1); - explicit MemoryInfo(std::nullptr_t) {} ///< No instance is created - explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl{p} {} ///< Take ownership of a pointer created by C Api - MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type); - ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; } -}; - -namespace detail { -template -struct TensorTypeAndShapeInfoImpl : Base { - using B = Base; - using B::B; - - ONNXTensorElementDataType GetElementType() const; ///< Wraps OrtApi::GetTensorElementType - size_t GetElementCount() const; ///< Wraps OrtApi::GetTensorShapeElementCount - - size_t GetDimensionsCount() const; ///< Wraps OrtApi::GetDimensionsCount - - /** \deprecated use GetShape() returning std::vector - * [[deprecated]] - * This interface is unsafe to use - */ - [[deprecated("use GetShape()")]] void GetDimensions(int64_t* values, size_t values_count) const; ///< Wraps OrtApi::GetDimensions - - void GetSymbolicDimensions(const char** values, size_t values_count) const; ///< Wraps OrtApi::GetSymbolicDimensions - - std::vector GetShape() const; ///< Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape -}; - -} // namespace detail - -using ConstTensorTypeAndShapeInfo = detail::TensorTypeAndShapeInfoImpl>; - -/** \brief Wrapper around ::OrtTensorTypeAndShapeInfo - * - */ -struct TensorTypeAndShapeInfo : detail::TensorTypeAndShapeInfoImpl { - explicit TensorTypeAndShapeInfo(std::nullptr_t) {} ///< Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used - explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {} ///< Used for interop with the C API - ConstTensorTypeAndShapeInfo GetConst() const { return ConstTensorTypeAndShapeInfo{this->p_}; } -}; - -namespace detail { -template -struct SequenceTypeInfoImpl : Base { - using B = Base; - using B::B; - TypeInfo GetSequenceElementType() const; ///< Wraps OrtApi::GetSequenceElementType -}; - -} // namespace detail - -using ConstSequenceTypeInfo = detail::SequenceTypeInfoImpl>; - -/** \brief Wrapper around ::OrtSequenceTypeInfo - * - */ -struct SequenceTypeInfo : detail::SequenceTypeInfoImpl { - explicit SequenceTypeInfo(std::nullptr_t) {} ///< Create an empty SequenceTypeInfo object, must be assigned a valid one to be used - explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl{p} {} ///< Used for interop with the C API - ConstSequenceTypeInfo GetConst() const { return ConstSequenceTypeInfo{this->p_}; } -}; - -namespace detail { -template -struct OptionalTypeInfoImpl : Base { - using B = Base; - using B::B; - TypeInfo GetOptionalElementType() const; ///< Wraps OrtApi::CastOptionalTypeToContainedTypeInfo -}; - -} // namespace detail - -// This is always owned by the TypeInfo and can only be obtained from it. -using ConstOptionalTypeInfo = detail::OptionalTypeInfoImpl>; - -namespace detail { -template -struct MapTypeInfoImpl : detail::Base { - using B = Base; - using B::B; - ONNXTensorElementDataType GetMapKeyType() const; ///< Wraps OrtApi::GetMapKeyType - TypeInfo GetMapValueType() const; ///< Wraps OrtApi::GetMapValueType -}; - -} // namespace detail - -using ConstMapTypeInfo = detail::MapTypeInfoImpl>; - -/** \brief Wrapper around ::OrtMapTypeInfo - * - */ -struct MapTypeInfo : detail::MapTypeInfoImpl { - explicit MapTypeInfo(std::nullptr_t) {} ///< Create an empty MapTypeInfo object, must be assigned a valid one to be used - explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl{p} {} ///< Used for interop with the C API - ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; } -}; - -namespace detail { -template -struct TypeInfoImpl : detail::Base { - using B = Base; - using B::B; - - ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; ///< Wraps OrtApi::CastTypeInfoToTensorInfo - ConstSequenceTypeInfo GetSequenceTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToSequenceTypeInfo - ConstMapTypeInfo GetMapTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToMapTypeInfo - ConstOptionalTypeInfo GetOptionalTypeInfo() const; ///< wraps OrtApi::CastTypeInfoToOptionalTypeInfo - - ONNXType GetONNXType() const; -}; -} // namespace detail - -/// -/// Contains a constant, unowned OrtTypeInfo that can be copied and passed around by value. -/// Provides access to const OrtTypeInfo APIs. -/// -using ConstTypeInfo = detail::TypeInfoImpl>; - -/// -/// Type information that may contain either TensorTypeAndShapeInfo or -/// the information about contained sequence or map depending on the ONNXType. -/// -struct TypeInfo : detail::TypeInfoImpl { - explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used - explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl{p} {} ///< C API Interop - - ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; } -}; - -namespace detail { -// This structure is used to feed sparse tensor values -// information for use with FillSparseTensor() API -// if the data type for the sparse tensor values is numeric -// use data.p_data, otherwise, use data.str pointer to feed -// values. data.str is an array of const char* that are zero terminated. -// number of strings in the array must match shape size. -// For fully sparse tensors use shape {0} and set p_data/str -// to nullptr. -struct OrtSparseValuesParam { - const int64_t* values_shape; - size_t values_shape_len; - union { - const void* p_data; - const char** str; - } data; -}; - -// Provides a way to pass shape in a single -// argument -struct Shape { - const int64_t* shape; - size_t shape_len; -}; - -template -struct ConstValueImpl : Base { - using B = Base; - using B::B; - - /// - /// Obtains a pointer to a user defined data for experimental purposes - /// - template - void GetOpaqueData(const char* domain, const char* type_name, R&) const; ///< Wraps OrtApi::GetOpaqueValue - - bool IsTensor() const; ///< Returns true if Value is a tensor, false for other types like map/sequence/etc - bool HasValue() const; /// < Return true if OrtValue contains data and returns false if the OrtValue is a None - - size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements - Value GetValue(int index, OrtAllocator* allocator) const; - - /// - /// This API returns a full length of string data contained within either a tensor or a sparse Tensor. - /// For sparse tensor it returns a full length of stored non-empty strings (values). The API is useful - /// for allocating necessary memory and calling GetStringTensorContent(). - /// - /// total length of UTF-8 encoded bytes contained. No zero terminators counted. - size_t GetStringTensorDataLength() const; - - /// - /// The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor - /// into a supplied buffer. Use GetStringTensorDataLength() to find out the length of the buffer to allocate. - /// The user must also allocate offsets buffer with the number of entries equal to that of the contained - /// strings. - /// - /// Strings are always assumed to be on CPU, no X-device copy. - /// - /// user allocated buffer - /// length in bytes of the allocated buffer - /// a pointer to the offsets user allocated buffer - /// count of offsets, must be equal to the number of strings contained. - /// that can be obtained from the shape of the tensor or from GetSparseTensorValuesTypeAndShapeInfo() - /// for sparse tensors - void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const; - - /// - /// Returns a const typed pointer to the tensor contained data. - /// No type checking is performed, the caller must ensure the type matches the tensor type. - /// - /// - /// const pointer to data, no copies made - template - const R* GetTensorData() const; ///< Wraps OrtApi::GetTensorMutableData /// - - /// - /// Returns a non-typed pointer to a tensor contained data. - /// - /// const pointer to data, no copies made - const void* GetTensorRawData() const; - - /// - /// The API returns type information for data contained in a tensor. For sparse - /// tensors it returns type information for contained non-zero values. - /// It returns dense shape for sparse tensors. - /// - /// TypeInfo - TypeInfo GetTypeInfo() const; - - /// - /// The API returns type information for data contained in a tensor. For sparse - /// tensors it returns type information for contained non-zero values. - /// It returns dense shape for sparse tensors. - /// - /// TensorTypeAndShapeInfo - TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; - - /// - /// This API returns information about the memory allocation used to hold data. - /// - /// Non owning instance of MemoryInfo - ConstMemoryInfo GetTensorMemoryInfo() const; - - /// - /// The API copies UTF-8 encoded bytes for the requested string element - /// contained within a tensor or a sparse tensor into a provided buffer. - /// Use GetStringTensorElementLength() to obtain the length of the buffer to allocate. - /// - /// - /// - /// - void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const; - - /// - /// Returns string tensor UTF-8 encoded string element. - /// Use of this API is recommended over GetStringTensorElement() that takes void* buffer pointer. - /// - /// - /// std::string - std::string GetStringTensorElement(size_t element_index) const; - - /// - /// The API returns a byte length of UTF-8 encoded string element - /// contained in either a tensor or a spare tensor values. - /// - /// - /// byte length for the specified string element - size_t GetStringTensorElementLength(size_t element_index) const; - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// The API returns the sparse data format this OrtValue holds in a sparse tensor. - /// If the sparse tensor was not fully constructed, i.e. Use*() or Fill*() API were not used - /// the value returned is ORT_SPARSE_UNDEFINED. - /// - /// Format enum - OrtSparseFormat GetSparseFormat() const; - - /// - /// The API returns type and shape information for stored non-zero values of the - /// sparse tensor. Use GetSparseTensorValues() to obtain values buffer pointer. - /// - /// TensorTypeAndShapeInfo values information - TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const; - - /// - /// The API returns type and shape information for the specified indices. Each supported - /// indices have their own enum values even if a give format has more than one kind of indices. - /// Use GetSparseTensorIndicesData() to obtain pointer to indices buffer. - /// - /// enum requested - /// type and shape information - TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const; - - /// - /// The API retrieves a pointer to the internal indices buffer. The API merely performs - /// a convenience data type casting on the return type pointer. Make sure you are requesting - /// the right type, use GetSparseTensorIndicesTypeShapeInfo(); - /// - /// type to cast to - /// requested indices kind - /// number of indices entries - /// Pinter to the internal sparse tensor buffer containing indices. Do not free this pointer. - template - const R* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const; - - /// - /// Returns true if the OrtValue contains a sparse tensor - /// - /// - bool IsSparseTensor() const; - - /// - /// The API returns a pointer to an internal buffer of the sparse tensor - /// containing non-zero values. The API merely does casting. Make sure you - /// are requesting the right data type by calling GetSparseTensorValuesTypeAndShapeInfo() - /// first. - /// - /// numeric data types only. Use GetStringTensor*() to retrieve strings. - /// a pointer to the internal values buffer. Do not free this pointer. - template - const R* GetSparseTensorValues() const; - -#endif -}; - -template -struct ValueImpl : ConstValueImpl { - using B = ConstValueImpl; - using B::B; - - /// - /// Returns a non-const typed pointer to an OrtValue/Tensor contained buffer - /// No type checking is performed, the caller must ensure the type matches the tensor type. - /// - /// non-const pointer to data, no copies made - template - R* GetTensorMutableData(); - - /// - /// Returns a non-typed non-const pointer to a tensor contained data. - /// - /// pointer to data, no copies made - void* GetTensorMutableRawData(); - - /// - // Obtain a reference to an element of data at the location specified - /// by the vector of dims. - /// - /// - /// [in] expressed by a vecotr of dimensions offsets - /// - template - R& At(const std::vector& location); - - /// - /// Set all strings at once in a string tensor - /// - /// [in] An array of strings. Each string in this array must be null terminated. - /// [in] Count of strings in s (Must match the size of \p value's tensor shape) - void FillStringTensor(const char* const* s, size_t s_len); - - /// - /// Set a single string in a string tensor - /// - /// [in] A null terminated UTF-8 encoded string - /// [in] Index of the string in the tensor to set - void FillStringTensorElement(const char* s, size_t index); - - /// - /// Allocate if necessary and obtain a pointer to a UTF-8 - /// encoded string element buffer indexed by the flat element index, - /// of the specified length. - /// - /// This API is for advanced usage. It avoids a need to construct - /// an auxiliary array of string pointers, and allows to write data directly - /// (do not zero terminate). - /// - /// - /// - /// a pointer to a writable buffer - char* GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length); - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// pointer to the user allocated buffer with indices. Use nullptr for fully sparse tensors. - /// number of indices entries. Use 0 for fully sparse tensors - void UseCooIndices(int64_t* indices_data, size_t indices_num); - - /// - /// Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// pointer to the user allocated buffer with inner indices or nullptr for fully sparse tensors - /// number of csr inner indices or 0 for fully sparse tensors - /// pointer to the user allocated buffer with outer indices or nullptr for fully sparse tensors - /// number of csr outer indices or 0 for fully sparse tensors - void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num); - - /// - /// Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSparse format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// indices shape or a {0} for fully sparse - /// user allocated buffer with indices or nullptr for fully spare tensors - void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and COO indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information. - /// coo indices buffer or nullptr for fully sparse data - /// number of COO indices or 0 for fully sparse data - void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param, - const int64_t* indices_data, size_t indices_num); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and CSR indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information - /// csr inner indices pointer or nullptr for fully sparse tensors - /// number of csr inner indices or 0 for fully sparse tensors - /// pointer to csr indices data or nullptr for fully sparse tensors - /// number of csr outer indices or 0 - void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const int64_t* inner_indices_data, size_t inner_indices_num, - const int64_t* outer_indices_data, size_t outer_indices_num); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and BlockSparse indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information - /// indices shape. use {0} for fully sparse tensors - /// pointer to indices data or nullptr for fully sparse tensors - void FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const Shape& indices_shape, - const int32_t* indices_data); - -#endif -}; - -} // namespace detail - -using ConstValue = detail::ConstValueImpl>; -using UnownedValue = detail::ValueImpl>; - -/** \brief Wrapper around ::OrtValue - * - */ -struct Value : detail::ValueImpl { - using Base = detail::ValueImpl; - using OrtSparseValuesParam = detail::OrtSparseValuesParam; - using Shape = detail::Shape; - - explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used - explicit Value(OrtValue* p) : Base{p} {} ///< Used for interop with the C API - Value(Value&&) = default; - Value& operator=(Value&&) = default; - - ConstValue GetConst() const { return ConstValue{this->p_}; } - UnownedValue GetUnowned() const { return UnownedValue{this->p_}; } - - /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. - * \tparam T The numeric datatype. This API is not suitable for strings. - * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param p_data Pointer to the data buffer. - * \param p_data_element_count The number of elements in the data buffer. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - */ - template - static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len); - - /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. - * - * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param p_data Pointer to the data buffer. - * \param p_data_byte_count The number of bytes in the data buffer. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - * \param type The data type. - */ - static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type); - - /** \brief Creates an OrtValue with a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue. - * This overload will allocate the buffer for the tensor according to the supplied shape and data type. - * The allocated buffer will be owned by the returned OrtValue and will be freed when the OrtValue is released. - * The input data would need to be copied into the allocated buffer. - * This API is not suitable for strings. - * - * \tparam T The numeric datatype. This API is not suitable for strings. - * \param allocator The allocator to use. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - */ - template - static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len); - - /** \brief Creates an OrtValue with a tensor using the supplied OrtAllocator. - * Wraps OrtApi::CreateTensorAsOrtValue. - * The allocated buffer will be owned by the returned OrtValue and will be freed when the OrtValue is released. - * The input data would need to be copied into the allocated buffer. - * This API is not suitable for strings. - * - * \param allocator The allocator to use. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - * \param type The data type. - */ - static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type); - - /** \brief Creates an OrtValue with a Map Onnx type representation. - * The API would ref-count the supplied OrtValues and they will be released - * when the returned OrtValue is released. The caller may release keys and values after the call - * returns. - * - * \param keys an OrtValue containing a tensor with primitive data type keys. - * \param values an OrtValue that may contain a tensor. Ort currently supports only primitive data type values. - */ - static Value CreateMap(const Value& keys, const Value& values); ///< Wraps OrtApi::CreateValue - - /** \brief Creates an OrtValue with a Sequence Onnx type representation. - * The API would ref-count the supplied OrtValues and they will be released - * when the returned OrtValue is released. The caller may release the values after the call - * returns. - * - * \param values a vector of OrtValues that must have the same Onnx value type. - */ - static Value CreateSequence(const std::vector& values); ///< Wraps OrtApi::CreateValue - - /** \brief Creates an OrtValue wrapping an Opaque type. - * This is used for experimental support of non-tensor types. - * - * \tparam T - the type of the value. - * \param domain - zero terminated utf-8 string. Domain of the type. - * \param type_name - zero terminated utf-8 string. Name of the type. - * \param value - the value to be wrapped. - */ - template - static Value CreateOpaque(const char* domain, const char* type_name, const T& value); ///< Wraps OrtApi::CreateOpaqueValue - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// This is a simple forwarding method to the other overload that helps deducing - /// data type enum value from the type of the buffer. - /// - /// numeric datatype. This API is not suitable for strings. - /// Memory description where the user buffers reside (CPU vs GPU etc) - /// pointer to the user supplied buffer, use nullptr for fully sparse tensors - /// a would be dense shape of the tensor - /// non zero values shape. Use a single 0 shape for fully sparse tensors. - /// - template - static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, - const Shape& values_shape); - - /// - /// Creates an OrtValue instance containing SparseTensor. This constructs - /// a sparse tensor that makes use of user allocated buffers. It does not make copies - /// of the user provided data and does not modify it. The lifespan of user provided buffers should - /// eclipse the life span of the resulting OrtValue. This call constructs an instance that only contain - /// a pointer to non-zero values. To fully populate the sparse tensor call UseIndices() API below - /// to supply a sparse format specific indices. - /// This API is not suitable for string data. Use CreateSparseTensor() with allocator specified so strings - /// can be properly copied into the allocated buffer. - /// - /// Memory description where the user buffers reside (CPU vs GPU etc) - /// pointer to the user supplied buffer, use nullptr for fully sparse tensors - /// a would be dense shape of the tensor - /// non zero values shape. Use a single 0 shape for fully sparse tensors. - /// data type - /// Ort::Value instance containing SparseTensor - static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, - const Shape& values_shape, ONNXTensorElementDataType type); - - /// - /// This is a simple forwarding method to the below CreateSparseTensor. - /// This helps to specify data type enum in terms of C++ data type. - /// Use CreateSparseTensor - /// - /// numeric data type only. String data enum must be specified explicitly. - /// allocator to use - /// a would be dense shape of the tensor - /// Ort::Value - template - static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape); - - /// - /// Creates an instance of OrtValue containing sparse tensor. The created instance has no data. - /// The data must be supplied by on of the FillSparseTensor() methods that take both non-zero values - /// and indices. The data will be copied into a buffer that would be allocated using the supplied allocator. - /// Use this API to create OrtValues that contain sparse tensors with all supported data types including - /// strings. - /// - /// allocator to use. The allocator lifespan must eclipse that of the resulting OrtValue - /// a would be dense shape of the tensor - /// data type - /// an instance of Ort::Value - static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type); - -#endif // !defined(DISABLE_SPARSE_TENSORS) -}; - -/// -/// Represents native memory allocation coming from one of the -/// OrtAllocators registered with OnnxRuntime. -/// Use it to wrap an allocation made by an allocator -/// so it can be automatically released when no longer needed. -/// -struct MemoryAllocation { - MemoryAllocation(OrtAllocator* allocator, void* p, size_t size); - ~MemoryAllocation(); - MemoryAllocation(const MemoryAllocation&) = delete; - MemoryAllocation& operator=(const MemoryAllocation&) = delete; - MemoryAllocation(MemoryAllocation&&) noexcept; - MemoryAllocation& operator=(MemoryAllocation&&) noexcept; - - void* get() { return p_; } - size_t size() const { return size_; } - - private: - OrtAllocator* allocator_; - void* p_; - size_t size_; -}; - -namespace detail { -template -struct AllocatorImpl : Base { - using B = Base; - using B::B; - - void* Alloc(size_t size); - MemoryAllocation GetAllocation(size_t size); - void Free(void* p); - ConstMemoryInfo GetInfo() const; -}; - -} // namespace detail - -/** \brief Wrapper around ::OrtAllocator default instance that is owned by Onnxruntime - * - */ -struct AllocatorWithDefaultOptions : detail::AllocatorImpl> { - explicit AllocatorWithDefaultOptions(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance - AllocatorWithDefaultOptions(); -}; - -/** \brief Wrapper around ::OrtAllocator - * - */ -struct Allocator : detail::AllocatorImpl { - explicit Allocator(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance - Allocator(const Session& session, const OrtMemoryInfo*); -}; - -using UnownedAllocator = detail::AllocatorImpl>; - -namespace detail { -namespace binding_utils { -// Bring these out of template -std::vector GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator*); -std::vector GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator*); -} // namespace binding_utils - -template -struct ConstIoBindingImpl : Base { - using B = Base; - using B::B; - - std::vector GetOutputNames() const; - std::vector GetOutputNames(OrtAllocator*) const; - std::vector GetOutputValues() const; - std::vector GetOutputValues(OrtAllocator*) const; -}; - -template -struct IoBindingImpl : ConstIoBindingImpl { - using B = ConstIoBindingImpl; - using B::B; - - void BindInput(const char* name, const Value&); - void BindOutput(const char* name, const Value&); - void BindOutput(const char* name, const OrtMemoryInfo*); - void ClearBoundInputs(); - void ClearBoundOutputs(); - void SynchronizeInputs(); - void SynchronizeOutputs(); -}; - -} // namespace detail - -using ConstIoBinding = detail::ConstIoBindingImpl>; -using UnownedIoBinding = detail::IoBindingImpl>; - -/** \brief Wrapper around ::OrtIoBinding - * - */ -struct IoBinding : detail::IoBindingImpl { - explicit IoBinding(std::nullptr_t) {} ///< Create an empty object for convenience. Sometimes, we want to initialize members later. - explicit IoBinding(Session& session); - ConstIoBinding GetConst() const { return ConstIoBinding{this->p_}; } - UnownedIoBinding GetUnowned() const { return UnownedIoBinding{this->p_}; } -}; - -/*! \struct Ort::ArenaCfg - * \brief it is a structure that represents the configuration of an arena based allocator - * \details Please see docs/C_API.md for details - */ -struct ArenaCfg : detail::Base { - explicit ArenaCfg(std::nullptr_t) {} ///< Create an empty ArenaCfg object, must be assigned a valid one to be used - /** - * Wraps OrtApi::CreateArenaCfg - * \param max_mem - use 0 to allow ORT to choose the default - * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested - * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default - * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default - * See docs/C_API.md for details on what the following parameters mean and how to choose these values - */ - ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk); -}; - -// -// Custom OPs (only needed to implement custom OPs) -// - -/// -/// This struct provides life time management for custom op attribute -/// -struct OpAttr : detail::Base { - OpAttr(const char* name, const void* data, int len, OrtOpAttrType type); -}; - -/** - * Macro that logs a message using the provided logger. Throws an exception if OrtApi::Logger_LogMessage fails. - * Example: ORT_CXX_LOG(logger, ORT_LOGGING_LEVEL_INFO, "Log a message"); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param message A null-terminated UTF-8 message to log. - */ -#define ORT_CXX_LOG(logger, message_severity, message) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - Ort::ThrowOnError(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), message)); \ - } \ - } while (false) - -/** - * Macro that logs a message using the provided logger. Can be used in noexcept code since errors are silently ignored. - * Example: ORT_CXX_LOG_NOEXCEPT(logger, ORT_LOGGING_LEVEL_INFO, "Log a message"); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param message A null-terminated UTF-8 message to log. - */ -#define ORT_CXX_LOG_NOEXCEPT(logger, message_severity, message) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - static_cast(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), message)); \ - } \ - } while (false) - -/** - * Macro that logs a printf-like formatted message using the provided logger. Throws an exception if - * OrtApi::Logger_LogMessage fails or if a formatting error occurs. - * Example: ORT_CXX_LOGF(logger, ORT_LOGGING_LEVEL_INFO, "Log an int: %d", 12); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. - * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. - * \param ... Zero or more variadic arguments referenced by the format string. - */ -#define ORT_CXX_LOGF(logger, message_severity, /*format,*/...) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - Ort::ThrowOnError(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), __VA_ARGS__)); \ - } \ - } while (false) - -/** - * Macro that logs a printf-like formatted message using the provided logger. Can be used in noexcept code since errors - * are silently ignored. - * Example: ORT_CXX_LOGF_NOEXCEPT(logger, ORT_LOGGING_LEVEL_INFO, "Log an int: %d", 12); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. - * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. - * \param ... Zero or more variadic arguments referenced by the format string. - */ -#define ORT_CXX_LOGF_NOEXCEPT(logger, message_severity, /*format,*/...) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - static_cast(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), __VA_ARGS__)); \ - } \ - } while (false) - -/// -/// This class represents an ONNX Runtime logger that can be used to log information with an -/// associated severity level and source code location (file path, line number, function name). -/// -/// A Logger can be obtained from within custom operators by calling Ort::KernelInfo::GetLogger(). -/// Instances of Ort::Logger are the size of two pointers and can be passed by value. -/// -/// Use the ORT_CXX_LOG macros to ensure the source code location is set properly from the callsite -/// and to take advantage of a cached logging severity level that can bypass calls to the underlying C API. -/// -struct Logger { - /** - * Creates an empty Ort::Logger. Must be initialized from a valid Ort::Logger before use. - */ - Logger() = default; - - /** - * Creates an empty Ort::Logger. Must be initialized from a valid Ort::Logger before use. - */ - explicit Logger(std::nullptr_t) {} - - /** - * Creates a logger from an ::OrtLogger instance. Caches the logger's current severity level by calling - * OrtApi::Logger_GetLoggingSeverityLevel. Throws an exception if OrtApi::Logger_GetLoggingSeverityLevel fails. - * - * \param logger The ::OrtLogger to wrap. - */ - explicit Logger(const OrtLogger* logger); - - ~Logger() = default; - - Logger(const Logger&) = default; - Logger& operator=(const Logger&) = default; - - Logger(Logger&& v) noexcept = default; - Logger& operator=(Logger&& v) noexcept = default; - - /** - * Returns the logger's current severity level from the cached member. - * - * \return The current ::OrtLoggingLevel. - */ - OrtLoggingLevel GetLoggingSeverityLevel() const noexcept; - - /** - * Logs the provided message via OrtApi::Logger_LogMessage. Use the ORT_CXX_LOG or ORT_CXX_LOG_NOEXCEPT - * macros to properly set the source code location and to use the cached severity level to potentially bypass - * calls to the underlying C API. - * - * \param log_severity_level The message's logging severity level. - * \param file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. - * \param line_number The file line number in which the message is logged. Usually the value of __LINE__. - * \param func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. - * \param message The message to log. - * \return A Ort::Status value to indicate error or success. - */ - Status LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, - const char* func_name, const char* message) const noexcept; - - /** - * Logs a printf-like formatted message via OrtApi::Logger_LogMessage. Use the ORT_CXX_LOGF or ORT_CXX_LOGF_NOEXCEPT - * macros to properly set the source code location and to use the cached severity level to potentially bypass - * calls to the underlying C API. Returns an error status if a formatting error occurs. - * - * \param log_severity_level The message's logging severity level. - * \param file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. - * \param line_number The file line number in which the message is logged. Usually the value of __LINE__. - * \param func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. - * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. - * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. - * \param args Zero or more variadic arguments referenced by the format string. - * \return A Ort::Status value to indicate error or success. - */ - template - Status LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, - const char* func_name, const char* format, Args&&... args) const noexcept; - - private: - const OrtLogger* logger_{}; - OrtLoggingLevel cached_severity_level_{}; -}; - -/// -/// This class wraps a raw pointer OrtKernelContext* that is being passed -/// to the custom kernel Compute() method. Use it to safely access context -/// attributes, input and output parameters with exception safety guarantees. -/// See usage example in onnxruntime/test/testdata/custom_op_library/custom_op_library.cc -/// -struct KernelContext { - explicit KernelContext(OrtKernelContext* context); - size_t GetInputCount() const; - size_t GetOutputCount() const; - // If input is optional and is not present, the method returns en empty ConstValue - // which can be compared to nullptr. - ConstValue GetInput(size_t index) const; - // If outout is optional and is not present, the method returns en empty UnownedValue - // which can be compared to nullptr. - UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const; - UnownedValue GetOutput(size_t index, const std::vector& dims) const; - void* GetGPUComputeStream() const; - Logger GetLogger() const; - OrtAllocator* GetAllocator(const OrtMemoryInfo& memory_info) const; - OrtKernelContext* GetOrtKernelContext() const { return ctx_; } - void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const; - - private: - OrtKernelContext* ctx_; -}; - -struct KernelInfo; - -namespace detail { -namespace attr_utils { -void GetAttr(const OrtKernelInfo* p, const char* name, float&); -void GetAttr(const OrtKernelInfo* p, const char* name, int64_t&); -void GetAttr(const OrtKernelInfo* p, const char* name, std::string&); -void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector&); -void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector&); -} // namespace attr_utils - -template -struct KernelInfoImpl : Base { - using B = Base; - using B::B; - - KernelInfo Copy() const; - - template // R is only implemented for float, int64_t, and string - R GetAttribute(const char* name) const { - R val; - attr_utils::GetAttr(this->p_, name, val); - return val; - } - - template // R is only implemented for std::vector, std::vector - std::vector GetAttributes(const char* name) const { - std::vector result; - attr_utils::GetAttrs(this->p_, name, result); - return result; - } - - Value GetTensorAttribute(const char* name, OrtAllocator* allocator) const; - - size_t GetInputCount() const; - size_t GetOutputCount() const; - - std::string GetInputName(size_t index) const; - std::string GetOutputName(size_t index) const; - - TypeInfo GetInputTypeInfo(size_t index) const; - TypeInfo GetOutputTypeInfo(size_t index) const; - - ConstValue GetTensorConstantInput(size_t index, int* is_constant) const; - - std::string GetNodeName() const; - Logger GetLogger() const; -}; - -} // namespace detail - -using ConstKernelInfo = detail::KernelInfoImpl>; - -/// -/// This struct owns the OrtKernInfo* pointer when a copy is made. -/// For convenient wrapping of OrtKernelInfo* passed to kernel constructor -/// and query attributes, warp the pointer with Ort::Unowned instance -/// so it does not destroy the pointer the kernel does not own. -/// -struct KernelInfo : detail::KernelInfoImpl { - explicit KernelInfo(std::nullptr_t) {} ///< Create an empty instance to initialize later - explicit KernelInfo(OrtKernelInfo* info); ///< Take ownership of the instance - ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; } -}; - -/// -/// Create and own custom defined operation. -/// -struct Op : detail::Base { - explicit Op(std::nullptr_t) {} ///< Create an empty Operator object, must be assigned a valid one to be used - - explicit Op(OrtOp*); ///< Take ownership of the OrtOp - - static Op Create(const OrtKernelInfo* info, const char* op_name, const char* domain, - int version, const char** type_constraint_names, - const ONNXTensorElementDataType* type_constraint_values, - size_t type_constraint_count, - const OpAttr* attr_values, - size_t attr_count, - size_t input_count, size_t output_count); - - void Invoke(const OrtKernelContext* context, - const Value* input_values, - size_t input_count, - Value* output_values, - size_t output_count); - - // For easier refactoring - void Invoke(const OrtKernelContext* context, - const OrtValue* const* input_values, - size_t input_count, - OrtValue* const* output_values, - size_t output_count); -}; - -/// -/// Provide access to per-node attributes and input shapes, so one could compute and set output shapes. -/// -struct ShapeInferContext { - struct SymbolicInteger { - SymbolicInteger(int64_t i) : i_(i), is_int_(true){}; - SymbolicInteger(const char* s) : s_(s), is_int_(false){}; - SymbolicInteger(const SymbolicInteger&) = default; - SymbolicInteger(SymbolicInteger&&) = default; - - SymbolicInteger& operator=(const SymbolicInteger&) = default; - SymbolicInteger& operator=(SymbolicInteger&&) = default; - - bool operator==(const SymbolicInteger& dim) const { - if (is_int_ == dim.is_int_) { - if (is_int_) { - return i_ == dim.i_; - } else { - return std::string{s_} == std::string{dim.s_}; - } - } - return false; - } - - bool IsInt() const { return is_int_; } - int64_t AsInt() const { return i_; } - const char* AsSym() const { return s_; } - - static constexpr int INVALID_INT_DIM = -2; - - private: - union { - int64_t i_; - const char* s_; - }; - bool is_int_; - }; - - using Shape = std::vector; - - ShapeInferContext(const OrtApi* ort_api, OrtShapeInferContext* ctx); - - const Shape& GetInputShape(size_t indice) const { return input_shapes_.at(indice); } - - size_t GetInputCount() const { return input_shapes_.size(); } - - Status SetOutputShape(size_t indice, const Shape& shape); - - int64_t GetAttrInt(const char* attr_name); - - using Ints = std::vector; - Ints GetAttrInts(const char* attr_name); - - float GetAttrFloat(const char* attr_name); - - using Floats = std::vector; - Floats GetAttrFloats(const char* attr_name); - - std::string GetAttrString(const char* attr_name); - - using Strings = std::vector; - Strings GetAttrStrings(const char* attr_name); - - private: - const OrtOpAttr* GetAttrHdl(const char* attr_name) const; - const OrtApi* ort_api_; - OrtShapeInferContext* ctx_; - std::vector input_shapes_; -}; - -using ShapeInferFn = Ort::Status (*)(Ort::ShapeInferContext&); - -#define MAX_CUSTOM_OP_END_VER (1UL << 31) - 1 - -template -struct CustomOpBase : OrtCustomOp { - CustomOpBase() { - OrtCustomOp::version = ORT_API_VERSION; - OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast(this_)->GetName(); }; - - OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast(this_)->GetExecutionProviderType(); }; - - OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetInputTypeCount(); }; - OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputType(index); }; - OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputMemoryType(index); }; - - OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetOutputTypeCount(); }; - OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputType(index); }; - -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(push) -#pragma warning(disable : 26409) -#endif - OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast(op_kernel); }; -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(pop) -#endif - OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputCharacteristic(index); }; - OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputCharacteristic(index); }; - - OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp* this_) { return static_cast(this_)->GetVariadicInputMinArity(); }; - OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp* this_) { return static_cast(static_cast(this_)->GetVariadicInputHomogeneity()); }; - OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp* this_) { return static_cast(this_)->GetVariadicOutputMinArity(); }; - OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp* this_) { return static_cast(static_cast(this_)->GetVariadicOutputHomogeneity()); }; -#ifdef __cpp_if_constexpr - if constexpr (WithStatus) { -#else - if (WithStatus) { -#endif - OrtCustomOp::CreateKernelV2 = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info, void** op_kernel) -> OrtStatusPtr { - return static_cast(this_)->CreateKernelV2(*api, info, op_kernel); - }; - OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { - return static_cast(op_kernel)->ComputeV2(context); - }; - } else { - OrtCustomOp::CreateKernelV2 = nullptr; - OrtCustomOp::KernelComputeV2 = nullptr; - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast(this_)->CreateKernel(*api, info); }; - OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { - static_cast(op_kernel)->Compute(context); - }; - } - - SetShapeInferFn(0); - - OrtCustomOp::GetStartVersion = [](const OrtCustomOp* this_) { - return static_cast(this_)->start_ver_; - }; - - OrtCustomOp::GetEndVersion = [](const OrtCustomOp* this_) { - return static_cast(this_)->end_ver_; - }; - - OrtCustomOp::GetMayInplace = nullptr; - OrtCustomOp::ReleaseMayInplace = nullptr; - OrtCustomOp::GetAliasMap = nullptr; - OrtCustomOp::ReleaseAliasMap = nullptr; - } - - // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider - const char* GetExecutionProviderType() const { return nullptr; } - - // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below - // (inputs and outputs are required by default) - OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t /*index*/) const { - return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; - } - - OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t /*index*/) const { - return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; - } - - // Default implemention of GetInputMemoryType() that returns OrtMemTypeDefault - OrtMemType GetInputMemoryType(size_t /*index*/) const { - return OrtMemTypeDefault; - } - - // Default implementation of GetVariadicInputMinArity() returns 1 to specify that a variadic input - // should expect at least 1 argument. - int GetVariadicInputMinArity() const { - return 1; - } - - // Default implementation of GetVariadicInputHomegeneity() returns true to specify that all arguments - // to a variadic input should be of the same type. - bool GetVariadicInputHomogeneity() const { - return true; - } - - // Default implementation of GetVariadicOutputMinArity() returns 1 to specify that a variadic output - // should produce at least 1 output value. - int GetVariadicOutputMinArity() const { - return 1; - } - - // Default implementation of GetVariadicOutputHomegeneity() returns true to specify that all output values - // produced by a variadic output should be of the same type. - bool GetVariadicOutputHomogeneity() const { - return true; - } - - // Declare list of session config entries used by this Custom Op. - // Implement this function in order to get configs from CustomOpBase::GetSessionConfigs(). - // This default implementation returns an empty vector of config entries. - std::vector GetSessionConfigKeys() const { - return std::vector{}; - } - - template - decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape)) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp*, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - ShapeInferContext ctx(&GetApi(), ort_ctx); - return C::InferOutputShape(ctx); - }; - return {}; - } - - template - void SetShapeInferFn(...) { - OrtCustomOp::InferOutputShapeFn = {}; - } - - protected: - // Helper function that returns a map of session config entries specified by CustomOpBase::GetSessionConfigKeys. - void GetSessionConfigs(std::unordered_map& out, ConstSessionOptions options) const; - - int start_ver_ = 1; - int end_ver_ = MAX_CUSTOM_OP_END_VER; -}; - -} // namespace Ort - -#include "onnxruntime_cxx_inline.h" diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_cxx_inline.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_cxx_inline.h deleted file mode 100644 index a732bf1..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_cxx_inline.h +++ /dev/null @@ -1,2125 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Do not include this file directly. Please include "onnxruntime_cxx_api.h" instead. -// If interested in trying out features of the new experimental C++ API, include "experimental_onnxruntime_cxx_api.h" instead. -// -// These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter -// the main C++ file with implementation details. - -#include -#include -#include -#include - -// Convert OrtStatus to Ort::Status and return -// instead of throwing -#define ORT_CXX_RETURN_ON_API_FAIL(expression) \ - { \ - auto ort_status = (expression); \ - if (ort_status) { \ - return Ort::Status(ort_status); \ - } \ - } - -#ifdef __cpp_if_constexpr -#define ORT_CXX_IF_CONSTEXPR if constexpr -#else -#define ORT_CXX_IF_CONSTEXPR if -#endif - -namespace Ort { - -namespace detail { -inline void ThrowStatus(const Status& st) { - std::string error_message = st.GetErrorMessage(); - OrtErrorCode error_code = st.GetErrorCode(); - ORT_CXX_API_THROW(std::move(error_message), error_code); -} -} // namespace detail - -inline void ThrowOnError(OrtStatus* ort_status) { - if (ort_status) { - Ort::Status st(ort_status); - detail::ThrowStatus(st); - } -} - -inline void ThrowOnError(const Status& st) { - if (st) { - detail::ThrowStatus(st); - } -} - -inline Status::Status(OrtStatus* status) noexcept : Base{status} { -} - -inline Status::Status(const std::exception& e) noexcept { - p_ = GetApi().CreateStatus(ORT_FAIL, e.what()); -} - -inline Status::Status(const Exception& e) noexcept { - p_ = GetApi().CreateStatus(e.GetOrtErrorCode(), e.what()); -} - -inline Status::Status(const char* message, OrtErrorCode code) noexcept { - p_ = GetApi().CreateStatus(code, message); -} - -inline std::string Status::GetErrorMessage() const { - std::string message(GetApi().GetErrorMessage(p_)); - return message; -} - -inline OrtErrorCode Status::GetErrorCode() const { - return GetApi().GetErrorCode(p_); -} - -inline bool Status::IsOK() const noexcept { - return (p_ == nullptr); -} - -// This template converts a C++ type into it's ONNXTensorElementDataType -template -struct TypeToTensorType; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; -}; - -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ; -}; - -inline bool BFloat16_t::operator==(const BFloat16_t& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is not equal to anything, including itself. - return false; - } - return val == rhs.val; -} - -inline bool BFloat16_t::operator<(const BFloat16_t& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is unordered with respect to everything, including itself. - return false; - } - - const bool left_is_negative = IsNegative(); - if (left_is_negative != rhs.IsNegative()) { - // When the signs of left and right differ, we know that left is less than right if it is - // the negative value. The exception to this is if both values are zero, in which case IEEE - // says they should be equal, even if the signs differ. - return left_is_negative && !AreZero(*this, rhs); - } - return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); -} - -inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size) - : allocator_(allocator), p_(p), size_(size) { -} - -inline MemoryAllocation::~MemoryAllocation() { - if (p_ != nullptr) { - // We do not throw out of destructor - auto ret = GetApi().AllocatorFree(allocator_, p_); - static_cast(ret); - } -} - -inline MemoryAllocation::MemoryAllocation(MemoryAllocation&& o) noexcept : allocator_(nullptr), p_(nullptr), size_(0) { - *this = std::move(o); -} - -inline MemoryAllocation& MemoryAllocation::operator=(MemoryAllocation&& o) noexcept { - OrtAllocator* alloc = nullptr; - void* p = nullptr; - size_t sz = 0; - - // Swap out this - std::swap(alloc, allocator_); - std::swap(p, p_); - std::swap(sz, size_); - - // Swap with incoming - std::swap(allocator_, o.allocator_); - std::swap(p_, o.p_); - std::swap(size_, o.size_); - - // Destroy this instance if needed - MemoryAllocation this_alloc(alloc, p, sz); - return *this; -} - -namespace detail { - -template -inline void* AllocatorImpl::Alloc(size_t size) { - void* out; - ThrowOnError(GetApi().AllocatorAlloc(this->p_, size, &out)); - return out; -} - -template -inline MemoryAllocation AllocatorImpl::GetAllocation(size_t size) { - void* out; - ThrowOnError(GetApi().AllocatorAlloc(this->p_, size, &out)); - MemoryAllocation result(this->p_, out, size); - return result; -} - -template -inline void AllocatorImpl::Free(void* p) { - ThrowOnError(GetApi().AllocatorFree(this->p_, p)); -} - -template -inline ConstMemoryInfo AllocatorImpl::GetInfo() const { - const OrtMemoryInfo* out; - ThrowOnError(GetApi().AllocatorGetInfo(this->p_, &out)); - return ConstMemoryInfo{out}; -} - -} // namespace detail - -inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { - ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&this->p_)); -} - -inline Allocator::Allocator(const Session& sess, const OrtMemoryInfo* mem_info) { - ThrowOnError(GetApi().CreateAllocator(sess, mem_info, &this->p_)); -} - -namespace detail { - -template -inline std::string MemoryInfoImpl::GetAllocatorName() const { - const char* name = nullptr; - ThrowOnError(GetApi().MemoryInfoGetName(this->p_, &name)); - return std::string(name); -} - -template -inline OrtAllocatorType MemoryInfoImpl::GetAllocatorType() const { - OrtAllocatorType type; - ThrowOnError(GetApi().MemoryInfoGetType(this->p_, &type)); - return type; -} - -template -inline int MemoryInfoImpl::GetDeviceId() const { - int id = 0; - ThrowOnError(GetApi().MemoryInfoGetId(this->p_, &id)); - return id; -} - -template -inline OrtMemoryInfoDeviceType MemoryInfoImpl::GetDeviceType() const { - OrtMemoryInfoDeviceType type; - GetApi().MemoryInfoGetDeviceType(this->p_, &type); - return type; -} - -template -inline OrtMemType MemoryInfoImpl::GetMemoryType() const { - OrtMemType type; - ThrowOnError(GetApi().MemoryInfoGetMemType(this->p_, &type)); - return type; -} - -template -template -inline bool MemoryInfoImpl::operator==(const MemoryInfoImpl& o) const { - int comp_result = 0; - ThrowOnError(Ort::GetApi().CompareMemoryInfo(this->p_, o, &comp_result)); - return comp_result == 0; -} - -} // namespace detail - -inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) { - OrtMemoryInfo* p; - ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p)); - return MemoryInfo(p); -} - -inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type) { - ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &this->p_)); -} - -namespace detail { -template -inline std::vector ConstIoBindingImpl::GetOutputNames() const { - AllocatorWithDefaultOptions allocator; - return binding_utils::GetOutputNamesHelper(this->p_, allocator); -} - -template -inline std::vector ConstIoBindingImpl::GetOutputNames(OrtAllocator* allocator) const { - return binding_utils::GetOutputNamesHelper(this->p_, allocator); -} - -template -inline std::vector ConstIoBindingImpl::GetOutputValues() const { - AllocatorWithDefaultOptions allocator; - return binding_utils::GetOutputValuesHelper(this->p_, allocator); -} - -template -inline std::vector ConstIoBindingImpl::GetOutputValues(OrtAllocator* allocator) const { - return binding_utils::GetOutputValuesHelper(this->p_, allocator); -} - -template -inline void IoBindingImpl::BindInput(const char* name, const Value& value) { - ThrowOnError(GetApi().BindInput(this->p_, name, value)); -} - -template -inline void IoBindingImpl::BindOutput(const char* name, const Value& value) { - ThrowOnError(GetApi().BindOutput(this->p_, name, value)); -} - -template -inline void IoBindingImpl::BindOutput(const char* name, const OrtMemoryInfo* mem_info) { - ThrowOnError(GetApi().BindOutputToDevice(this->p_, name, mem_info)); -} - -template -inline void IoBindingImpl::ClearBoundInputs() { - GetApi().ClearBoundInputs(this->p_); -} - -template -inline void IoBindingImpl::ClearBoundOutputs() { - GetApi().ClearBoundOutputs(this->p_); -} - -template -inline void IoBindingImpl::SynchronizeInputs() { - ThrowOnError(GetApi().SynchronizeBoundInputs(this->p_)); -} - -template -inline void IoBindingImpl::SynchronizeOutputs() { - ThrowOnError(GetApi().SynchronizeBoundOutputs(this->p_)); -} - -namespace binding_utils { -inline std::vector GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator* allocator) { - std::vector result; - auto free_fn = detail::AllocatedFree(allocator); - using Ptr = std::unique_ptr; - - char* buffer = nullptr; - size_t* lengths = nullptr; - size_t count = 0; - ThrowOnError(GetApi().GetBoundOutputNames(binding, allocator, &buffer, &lengths, &count)); - - if (count == 0) { - return result; - } - - Ptr buffer_g(buffer, free_fn); - Ptr lengths_g(lengths, free_fn); - - result.reserve(count); - for (size_t i = 0; i < count; ++i) { - auto sz = *lengths; - result.emplace_back(buffer, sz); - buffer += sz; - ++lengths; - } - return result; -} - -inline std::vector GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator* allocator) { - std::vector result; - size_t owned = 0; - size_t output_count = 0; - // Lambda to release the buffer when no longer needed and - // make sure that we destroy all instances on exception - auto free_fn = [&owned, &output_count, allocator](OrtValue** buffer) { - if (buffer) { - while (owned < output_count) { - auto* p = buffer + owned++; - GetApi().ReleaseValue(*p); - } - allocator->Free(allocator, buffer); - } - }; - using Ptr = std::unique_ptr; - - OrtValue** output_buffer = nullptr; - ThrowOnError(GetApi().GetBoundOutputValues(binding, allocator, &output_buffer, &output_count)); - if (output_count == 0) { - return result; - } - - Ptr buffer_g(output_buffer, free_fn); - - result.reserve(output_count); - for (size_t i = 0; i < output_count; ++i) { - result.emplace_back(output_buffer[i]); - ++owned; - } - return result; -} - -} // namespace binding_utils -} // namespace detail - -inline IoBinding::IoBinding(Session& session) { - ThrowOnError(GetApi().CreateIoBinding(session, &this->p_)); -} - -inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk) { - ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_)); -} - -inline ThreadingOptions::ThreadingOptions() { - ThrowOnError(GetApi().CreateThreadingOptions(&p_)); -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalIntraOpNumThreads(int intra_op_num_threads) { - ThrowOnError(GetApi().SetGlobalIntraOpNumThreads(p_, intra_op_num_threads)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalInterOpNumThreads(int inter_op_num_threads) { - ThrowOnError(GetApi().SetGlobalInterOpNumThreads(p_, inter_op_num_threads)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalSpinControl(int allow_spinning) { - ThrowOnError(GetApi().SetGlobalSpinControl(p_, allow_spinning)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalDenormalAsZero() { - ThrowOnError(GetApi().SetGlobalDenormalAsZero(p_)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { - ThrowOnError(GetApi().SetGlobalCustomCreateThreadFn(p_, ort_custom_create_thread_fn)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalCustomThreadCreationOptions(void* ort_custom_thread_creation_options) { - ThrowOnError(GetApi().SetGlobalCustomThreadCreationOptions(p_, ort_custom_thread_creation_options)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn) { - ThrowOnError(GetApi().SetGlobalCustomJoinThreadFn(p_, ort_custom_join_thread_fn)); - return *this; -} - -inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param) { - ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(logging_level, logid, tp_options, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, - OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env& Env::EnableTelemetryEvents() { - ThrowOnError(GetApi().EnableTelemetryEvents(p_)); - return *this; -} - -inline Env& Env::DisableTelemetryEvents() { - ThrowOnError(GetApi().DisableTelemetryEvents(p_)); - return *this; -} - -inline Env& Env::UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level) { - ThrowOnError(GetApi().UpdateEnvWithCustomLogLevel(p_, log_severity_level)); - return *this; -} - -inline Env& Env::CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg) { - ThrowOnError(GetApi().CreateAndRegisterAllocator(p_, mem_info, arena_cfg)); - return *this; -} - -inline Env& Env::CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, const std::unordered_map& options, const OrtArenaCfg* arena_cfg) { - std::vector keys, values; - auto num_entries = options.size(); - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - for (const auto& entry : options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - ThrowOnError(GetApi().CreateAndRegisterAllocatorV2(p_, provider_type.c_str(), mem_info, arena_cfg, keys.data(), values.data(), num_entries)); - return *this; -} - -inline CustomOpDomain::CustomOpDomain(const char* domain) { - ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_)); -} - -inline void CustomOpDomain::Add(const OrtCustomOp* op) { - ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); -} - -inline RunOptions::RunOptions() { - ThrowOnError(GetApi().CreateRunOptions(&p_)); -} - -inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level)); - return *this; -} - -inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level)); - return *this; -} - -inline int RunOptions::GetRunLogVerbosityLevel() const { - int out; - ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out)); - return out; -} - -inline int RunOptions::GetRunLogSeverityLevel() const { - int out; - ThrowOnError(GetApi().RunOptionsGetRunLogSeverityLevel(p_, &out)); - return out; -} - -inline RunOptions& RunOptions::SetRunTag(const char* run_tag) { - ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag)); - return *this; -} - -inline const char* RunOptions::GetRunTag() const { - const char* out; - ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out)); - return out; -} - -inline RunOptions& RunOptions::AddConfigEntry(const char* config_key, const char* config_value) { - ThrowOnError(GetApi().AddRunConfigEntry(p_, config_key, config_value)); - return *this; -} - -inline RunOptions& RunOptions::SetTerminate() { - ThrowOnError(GetApi().RunOptionsSetTerminate(p_)); - return *this; -} - -inline RunOptions& RunOptions::UnsetTerminate() { - ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_)); - return *this; -} - -namespace detail { - -template -inline Ort::SessionOptions ConstSessionOptionsImpl::Clone() const { - OrtSessionOptions* out; - ThrowOnError(GetApi().CloneSessionOptions(this->p_, &out)); - return SessionOptions{out}; -} - -template -inline std::string ConstSessionOptionsImpl::GetConfigEntry(const char* config_key) const { - size_t size = 0; - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline bool ConstSessionOptionsImpl::HasConfigEntry(const char* config_key) const { - int out = 0; - Ort::ThrowOnError(GetApi().HasSessionConfigEntry(this->p_, config_key, &out)); - return static_cast(out); -} - -template -inline std::string ConstSessionOptionsImpl::GetConfigEntryOrDefault(const char* config_key, const std::string& def) { - if (!this->HasConfigEntry(config_key)) { - return def; - } - - return this->GetConfigEntry(config_key); -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetIntraOpNumThreads(int intra_op_num_threads) { - ThrowOnError(GetApi().SetIntraOpNumThreads(this->p_, intra_op_num_threads)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetInterOpNumThreads(int inter_op_num_threads) { - ThrowOnError(GetApi().SetInterOpNumThreads(this->p_, inter_op_num_threads)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) { - ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(this->p_, graph_optimization_level)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetDeterministicCompute(bool value) { - ThrowOnError(GetApi().SetDeterministicCompute(this->p_, value)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_filepath) { - ThrowOnError(GetApi().SetOptimizedModelFilePath(this->p_, optimized_model_filepath)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableProfiling(const ORTCHAR_T* profile_file_prefix) { - ThrowOnError(GetApi().EnableProfiling(this->p_, profile_file_prefix)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisableProfiling() { - ThrowOnError(GetApi().DisableProfiling(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableOrtCustomOps() { - ThrowOnError(GetApi().EnableOrtCustomOps(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableMemPattern() { - ThrowOnError(GetApi().EnableMemPattern(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisableMemPattern() { - ThrowOnError(GetApi().DisableMemPattern(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableCpuMemArena() { - ThrowOnError(GetApi().EnableCpuMemArena(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisableCpuMemArena() { - ThrowOnError(GetApi().DisableCpuMemArena(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetExecutionMode(ExecutionMode execution_mode) { - ThrowOnError(GetApi().SetSessionExecutionMode(this->p_, execution_mode)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetLogId(const char* logid) { - ThrowOnError(GetApi().SetSessionLogId(this->p_, logid)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetLogSeverityLevel(int level) { - ThrowOnError(GetApi().SetSessionLogSeverityLevel(this->p_, level)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::Add(OrtCustomOpDomain* custom_op_domain) { - ThrowOnError(GetApi().AddCustomOpDomain(this->p_, custom_op_domain)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddConfigEntry(const char* config_key, const char* config_value) { - ThrowOnError(GetApi().AddSessionConfigEntry(this->p_, config_key, config_value)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddInitializer(const char* name, const OrtValue* ort_val) { - ThrowOnError(GetApi().AddInitializer(this->p_, name, ort_val)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisablePerSessionThreads() { - ThrowOnError(GetApi().DisablePerSessionThreads(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddExternalInitializers(const std::vector& names, - const std::vector& ort_values) { - const size_t inputs_num = names.size(); - if (inputs_num != ort_values.size()) { - ORT_CXX_API_THROW("Expecting names and ort_values to have the same length", ORT_INVALID_ARGUMENT); - } - std::vector names_ptr; - std::vector ort_values_ptrs; - names_ptr.reserve(inputs_num); - ort_values_ptrs.reserve(inputs_num); - for (size_t i = 0; i < inputs_num; ++i) { - names_ptr.push_back(names[i].c_str()); - ort_values_ptrs.push_back(ort_values[i]); - } - ThrowOnError(GetApi().AddExternalInitializers(this->p_, names_ptr.data(), ort_values_ptrs.data(), inputs_num)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddExternalInitializersFromFilesInMemory(const std::vector>& file_names, - const std::vector& buffer_array, - const std::vector& file_lengths) { - const size_t inputs_num = file_names.size(); - if (inputs_num != buffer_array.size()) { - ORT_CXX_API_THROW("Expecting names and buffer_array to have the same length", ORT_INVALID_ARGUMENT); - } - if (inputs_num != file_lengths.size()) { - ORT_CXX_API_THROW("Expecting names and file_lengths to have the same length", ORT_INVALID_ARGUMENT); - } - std::vector names_ptr; - names_ptr.reserve(inputs_num); - for (size_t i = 0; i < inputs_num; ++i) { - names_ptr.push_back(file_names[i].c_str()); - } - ThrowOnError(GetApi().AddExternalInitializersFromFilesInMemory(this->p_, names_ptr.data(), buffer_array.data(), - file_lengths.data(), inputs_num)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA_V2(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_ROCM(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT_V2(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_MIGraphX(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CANN(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_Dnnl(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider( - const std::string& provider_name, - const std::unordered_map& provider_options) { - auto num_entries = provider_options.size(); - std::vector keys, values; - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - - for (const auto& entry : provider_options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider(this->p_, provider_name.c_str(), - keys.data(), values.data(), num_entries)); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { - ThrowOnError(GetApi().SessionOptionsSetCustomCreateThreadFn(this->p_, ort_custom_create_thread_fn)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options) { - ThrowOnError(GetApi().SessionOptionsSetCustomThreadCreationOptions(this->p_, ort_custom_thread_creation_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn) { - ThrowOnError(GetApi().SessionOptionsSetCustomJoinThreadFn(this->p_, ort_custom_join_thread_fn)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_OpenVINO_V2(const std::unordered_map& provider_options) { - auto num_entries = provider_options.size(); - std::vector keys, values; - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - - for (const auto& entry : provider_options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO_V2(this->p_, - keys.data(), values.data(), num_entries)); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_VitisAI(const std::unordered_map& provider_options) { - auto num_entries = provider_options.size(); - std::vector keys, values; - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - - for (const auto& entry : provider_options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_VitisAI(this->p_, keys.data(), values.data(), num_entries)); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, - const CustomOpConfigs& custom_op_configs) { - // Add custom op config entries before registering the custom op library. Otherwise, the config entries _may_ be ignored by - // the custom op library. - for (const auto& config_iter : custom_op_configs.GetFlattenedConfigs()) { - AddConfigEntry(config_iter.first.c_str(), config_iter.second.c_str()); - } - - ThrowOnError(GetApi().RegisterCustomOpsLibrary_V2(this->p_, library_name)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsUsingFunction(const char* registration_function_name) { - ThrowOnError(GetApi().RegisterCustomOpsUsingFunction(this->p_, registration_function_name)); - return *this; -} - -/// Session -template -inline size_t ConstSessionImpl::GetInputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetInputCount(this->p_, &out)); - return out; -} - -template -inline size_t ConstSessionImpl::GetOutputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOutputCount(this->p_, &out)); - return out; -} - -template -inline size_t ConstSessionImpl::GetOverridableInitializerCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOverridableInitializerCount(this->p_, &out)); - return out; -} - -template -inline AllocatedStringPtr ConstSessionImpl::GetInputNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetInputName(this->p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -template -inline AllocatedStringPtr ConstSessionImpl::GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOutputName(this->p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -template -inline AllocatedStringPtr ConstSessionImpl::GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOverridableInitializerName(this->p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -template -inline uint64_t ConstSessionImpl::GetProfilingStartTimeNs() const { - uint64_t out; - ThrowOnError(GetApi().SessionGetProfilingStartTimeNs(this->p_, &out)); - return out; -} - -template -inline ModelMetadata ConstSessionImpl::GetModelMetadata() const { - OrtModelMetadata* out; - ThrowOnError(GetApi().SessionGetModelMetadata(this->p_, &out)); - return ModelMetadata{out}; -} - -template -inline TypeInfo ConstSessionImpl::GetInputTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetInputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline TypeInfo ConstSessionImpl::GetOutputTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetOutputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline TypeInfo ConstSessionImpl::GetOverridableInitializerTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline std::vector SessionImpl::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, size_t output_count) { - std::vector output_values; - output_values.reserve(output_count); - for (size_t i = 0; i < output_count; i++) - output_values.emplace_back(nullptr); - Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_count); - return output_values; -} - -template -inline void SessionImpl::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count) { - static_assert(sizeof(Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); - auto ort_input_values = reinterpret_cast(input_values); - auto ort_output_values = reinterpret_cast(output_values); - ThrowOnError(GetApi().Run(this->p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values)); -} - -template -inline void SessionImpl::Run(const RunOptions& run_options, const IoBinding& io_binding) { - ThrowOnError(GetApi().RunWithBinding(this->p_, run_options, io_binding)); -} - -template -inline void SessionImpl::RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data) { - auto ort_input_values = reinterpret_cast(input_values); - auto ort_output_values = reinterpret_cast(output_values); - ThrowOnError(GetApi().RunAsync(this->p_, run_options, input_names, - ort_input_values, input_count, output_names, output_count, - ort_output_values, callback, user_data)); -} - -template -inline AllocatedStringPtr SessionImpl::EndProfilingAllocated(OrtAllocator* allocator) { - char* out = nullptr; - ThrowOnError(GetApi().SessionEndProfiling(this->p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -} // namespace detail - -inline SessionOptions::SessionOptions() { - ThrowOnError(GetApi().CreateSessionOptions(&this->p_)); -} - -/// CustomOpConfigs -inline std::string detail::MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config) { - std::string config_key = "custom_op."; - - config_key += custom_op_name; - config_key += "."; - config_key += config; - - return config_key; -} - -inline CustomOpConfigs& CustomOpConfigs::AddConfig(const char* custom_op_name, const char* config_key, const char* config_value) { - const std::string full_flat_key = detail::MakeCustomOpConfigEntryKey(custom_op_name, config_key); - flat_configs_[full_flat_key] = config_value; - return *this; -} - -inline const std::unordered_map& CustomOpConfigs::GetFlattenedConfigs() const { - return flat_configs_; -} - -inline Session::Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) { - ThrowOnError(GetApi().CreateSession(env, model_path, options, &this->p_)); -} - -inline Session::Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container) { - ThrowOnError(GetApi().CreateSessionWithPrepackedWeightsContainer(env, model_path, options, prepacked_weights_container, &this->p_)); -} - -inline Session::Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options) { - ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &this->p_)); -} - -inline Session::Session(const Env& env, const void* model_data, size_t model_data_length, - const SessionOptions& options, OrtPrepackedWeightsContainer* prepacked_weights_container) { - ThrowOnError(GetApi().CreateSessionFromArrayWithPrepackedWeightsContainer(env, model_data, model_data_length, options, - prepacked_weights_container, &this->p_)); -} - -inline AllocatedStringPtr ModelMetadata::GetProducerNameAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::GetGraphNameAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::GetDomainAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr Ort::ModelMetadata::GetDescriptionAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::GetGraphDescriptionAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphDescription(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline std::vector ModelMetadata::GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const { - auto deletor = detail::AllocatedFree(allocator); - std::vector result; - - char** out = nullptr; - int64_t num_keys = 0; - ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys)); - if (num_keys <= 0) { - return result; - } - - // array of pointers will be freed - std::unique_ptr array_guard(out, deletor); - // reserve may throw - auto strings_deletor = [&deletor, num_keys](char** out) { for(int64_t i = 0; i < num_keys; ++i) deletor(out[i]); }; - std::unique_ptr strings_guard(out, strings_deletor); - result.reserve(static_cast(num_keys)); - strings_guard.release(); - for (int64_t i = 0; i < num_keys; ++i) { - result.push_back(AllocatedStringPtr(out[i], deletor)); - } - - return result; -} - -inline int64_t ModelMetadata::GetVersion() const { - int64_t out; - ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out)); - return out; -} - -namespace detail { - -template -inline ONNXTensorElementDataType TensorTypeAndShapeInfoImpl::GetElementType() const { - ONNXTensorElementDataType out; - ThrowOnError(GetApi().GetTensorElementType(this->p_, &out)); - return out; -} - -template -inline size_t TensorTypeAndShapeInfoImpl::GetElementCount() const { - size_t out; - ThrowOnError(GetApi().GetTensorShapeElementCount(this->p_, &out)); - return static_cast(out); -} - -template -inline size_t TensorTypeAndShapeInfoImpl::GetDimensionsCount() const { - size_t out; - ThrowOnError(GetApi().GetDimensionsCount(this->p_, &out)); - return out; -} - -template -inline void TensorTypeAndShapeInfoImpl::GetDimensions(int64_t* values, size_t values_count) const { - ThrowOnError(GetApi().GetDimensions(this->p_, values, values_count)); -} - -template -inline void TensorTypeAndShapeInfoImpl::GetSymbolicDimensions(const char** values, size_t values_count) const { - ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, values, values_count)); -} - -template -inline std::vector TensorTypeAndShapeInfoImpl::GetShape() const { - std::vector out(GetDimensionsCount(), 0); - ThrowOnError(GetApi().GetDimensions(this->p_, out.data(), out.size())); - return out; -} - -template -inline ConstTensorTypeAndShapeInfo TypeInfoImpl::GetTensorTypeAndShapeInfo() const { - const OrtTensorTypeAndShapeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToTensorInfo(this->p_, &out)); - return ConstTensorTypeAndShapeInfo{out}; -} - -template -inline ConstSequenceTypeInfo TypeInfoImpl::GetSequenceTypeInfo() const { - const OrtSequenceTypeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(this->p_, &out)); - return ConstSequenceTypeInfo{out}; -} - -template -inline ConstMapTypeInfo TypeInfoImpl::GetMapTypeInfo() const { - const OrtMapTypeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(this->p_, &out)); - return ConstMapTypeInfo{out}; -} - -template -inline ONNXType TypeInfoImpl::GetONNXType() const { - ONNXType out; - ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(this->p_, &out)); - return out; -} - -template -inline TypeInfo SequenceTypeInfoImpl::GetSequenceElementType() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetSequenceElementType(this->p_, &output)); - return TypeInfo{output}; -} - -template -inline TypeInfo OptionalTypeInfoImpl::GetOptionalElementType() const { - OrtTypeInfo* info; - ThrowOnError(GetApi().GetOptionalContainedTypeInfo(this->p_, &info)); - return TypeInfo{info}; -} - -template -inline ONNXTensorElementDataType MapTypeInfoImpl::GetMapKeyType() const { - ONNXTensorElementDataType out; - ThrowOnError(GetApi().GetMapKeyType(this->p_, &out)); - return out; -} - -template -inline TypeInfo MapTypeInfoImpl::GetMapValueType() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetMapValueType(this->p_, &output)); - return TypeInfo{output}; -} - -template -inline ConstOptionalTypeInfo TypeInfoImpl::GetOptionalTypeInfo() const { - const OrtOptionalTypeInfo* info; - ThrowOnError(GetApi().CastTypeInfoToOptionalTypeInfo(this->p_, &info)); - return ConstOptionalTypeInfo{info}; -} - -} // namespace detail - -namespace detail { - -template -template -inline void ConstValueImpl::GetOpaqueData(const char* domain, const char* type_name, R& out) const { - ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, this->p_, &out, sizeof(R))); -} - -template -inline bool ConstValueImpl::IsTensor() const { - int out; - ThrowOnError(GetApi().IsTensor(this->p_, &out)); - return out != 0; -} - -template -inline bool ConstValueImpl::HasValue() const { - int out; - ThrowOnError(GetApi().HasValue(this->p_, &out)); - return out != 0; -} - -template -inline size_t ConstValueImpl::GetCount() const { - size_t out; - ThrowOnError(GetApi().GetValueCount(this->p_, &out)); - return out; -} - -template -inline Value ConstValueImpl::GetValue(int index, OrtAllocator* allocator) const { - OrtValue* out; - ThrowOnError(GetApi().GetValue(this->p_, index, allocator, &out)); - return Value{out}; -} - -template -inline size_t ConstValueImpl::GetStringTensorDataLength() const { - size_t out; - ThrowOnError(GetApi().GetStringTensorDataLength(this->p_, &out)); - return out; -} - -template -inline size_t ConstValueImpl::GetStringTensorElementLength(size_t element_index) const { - size_t out; - ThrowOnError(GetApi().GetStringTensorElementLength(this->p_, element_index, &out)); - return out; -} - -template -template -inline const R* ConstValueImpl::GetTensorData() const { - R* out; - ThrowOnError(GetApi().GetTensorMutableData(const_cast(this->p_), (void**)&out)); - return out; -} - -template -inline const void* ConstValueImpl::GetTensorRawData() const { - void* out; - ThrowOnError(GetApi().GetTensorMutableData(const_cast(this->p_), &out)); - return out; -} - -template -inline TypeInfo ConstValueImpl::GetTypeInfo() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetTypeInfo(this->p_, &output)); - return TypeInfo{output}; -} - -template -inline TensorTypeAndShapeInfo ConstValueImpl::GetTensorTypeAndShapeInfo() const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetTensorTypeAndShape(this->p_, &output)); - return TensorTypeAndShapeInfo{output}; -} - -template -inline ConstMemoryInfo ConstValueImpl::GetTensorMemoryInfo() const { - const OrtMemoryInfo* mem_info; - ThrowOnError(GetApi().GetTensorMemoryInfo(this->p_, &mem_info)); - return ConstMemoryInfo(mem_info); -} - -template -inline void ConstValueImpl::GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const { - ThrowOnError(GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, buffer)); -} - -template -inline std::string ConstValueImpl::GetStringTensorElement(size_t element_index) const { - size_t buffer_length; - ThrowOnError(GetApi().GetStringTensorElementLength(this->p_, element_index, &buffer_length)); - - std::string s; - s.resize(buffer_length); - ThrowOnError(GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, &s[0])); - return s; -} - -template -inline void ConstValueImpl::GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const { - ThrowOnError(GetApi().GetStringTensorContent(this->p_, buffer, buffer_length, offsets, offsets_count)); -} - -#if !defined(DISABLE_SPARSE_TENSORS) -template -inline OrtSparseFormat ConstValueImpl::GetSparseFormat() const { - OrtSparseFormat format; - ThrowOnError(GetApi().GetSparseTensorFormat(this->p_, &format)); - return format; -} - -template -inline TensorTypeAndShapeInfo ConstValueImpl::GetSparseTensorValuesTypeAndShapeInfo() const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetSparseTensorValuesTypeAndShape(this->p_, &output)); - return TensorTypeAndShapeInfo{output}; -} - -template -inline TensorTypeAndShapeInfo ConstValueImpl::GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat indices_format) const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetSparseTensorIndicesTypeShape(this->p_, indices_format, &output)); - return TensorTypeAndShapeInfo{output}; -} - -template -template -inline const R* ConstValueImpl::GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const { - const void* out; - ThrowOnError(GetApi().GetSparseTensorIndices(this->p_, indices_format, &num_indices, &out)); - return reinterpret_cast(out); -} - -template -inline bool ConstValueImpl::IsSparseTensor() const { - int out; - ThrowOnError(GetApi().IsSparseTensor(this->p_, &out)); - return out != 0; -} - -template -template -inline const R* ConstValueImpl::GetSparseTensorValues() const { - const void* out; - ThrowOnError(GetApi().GetSparseTensorValues(this->p_, &out)); - return reinterpret_cast(out); -} - -#endif - -template -void ValueImpl::FillStringTensor(const char* const* s, size_t s_len) { - ThrowOnError(GetApi().FillStringTensor(this->p_, s, s_len)); -} - -template -void ValueImpl::FillStringTensorElement(const char* s, size_t index) { - ThrowOnError(GetApi().FillStringTensorElement(this->p_, s, index)); -} - -template -inline char* ValueImpl::GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length) { - char* result; - ThrowOnError(GetApi().GetResizedStringTensorElementBuffer(this->p_, index, buffer_length, &result)); - return result; -} - -template -void* ValueImpl::GetTensorMutableRawData() { - void* out; - ThrowOnError(GetApi().GetTensorMutableData(this->p_, &out)); - return out; -} - -template -template -R* ValueImpl::GetTensorMutableData() { - R* out; - ThrowOnError(GetApi().GetTensorMutableData(this->p_, (void**)&out)); - return out; -} - -template -template -R& ValueImpl::At(const std::vector& location) { - static_assert(!std::is_same::value, "this api does not support std::string"); - R* out; - ThrowOnError(GetApi().TensorAt(this->p_, location.data(), location.size(), (void**)&out)); - return *out; -} - -#if !defined(DISABLE_SPARSE_TENSORS) -template -void ValueImpl::UseCooIndices(int64_t* indices_data, size_t indices_num) { - ThrowOnError(GetApi().UseCooIndices(this->p_, indices_data, indices_num)); -} - -template -void ValueImpl::UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num) { - ThrowOnError(GetApi().UseCsrIndices(this->p_, inner_data, inner_num, outer_data, outer_num)); -} - -template -void ValueImpl::UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data) { - ThrowOnError(GetApi().UseBlockSparseIndices(this->p_, indices_shape.shape, indices_shape.shape_len, indices_data)); -} - -template -void ValueImpl::FillSparseTensorCoo(const OrtMemoryInfo* mem_info, const OrtSparseValuesParam& values_param, - const int64_t* indices_data, size_t indices_num) { - ThrowOnError(GetApi().FillSparseTensorCoo(this->p_, mem_info, values_param.values_shape, - values_param.values_shape_len, values_param.data.p_data, - indices_data, indices_num)); -} - -template -void ValueImpl::FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const int64_t* inner_indices_data, size_t inner_indices_num, - const int64_t* outer_indices_data, size_t outer_indices_num) { - ThrowOnError(GetApi().FillSparseTensorCsr(this->p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, - inner_indices_data, inner_indices_num, - outer_indices_data, outer_indices_num)); -} - -template -void ValueImpl::FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const Shape& indices_shape, - const int32_t* indices_data) { - ThrowOnError(GetApi().FillSparseTensorBlockSparse(this->p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, - indices_shape.shape, indices_shape.shape_len, - indices_data)); -} - -#endif // !defined(DISABLE_SPARSE_TENSORS) - -} // namespace detail - -template -inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len) { - return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out)); - return Value{out}; -} - -template -inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) { - return CreateTensor(allocator, shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out)); - return Value{out}; -} - -#if !defined(DISABLE_SPARSE_TENSORS) - -template -inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, - const Shape& values_shape) { - return CreateSparseTensor(info, p_data, dense_shape, values_shape, TypeToTensorType::type); -} - -inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, - const Shape& values_shape, ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateSparseTensorWithValuesAsOrtValue(info, p_data, dense_shape.shape, dense_shape.shape_len, - values_shape.shape, values_shape.shape_len, type, &out)); - return Value{out}; -} - -template -inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape) { - return CreateSparseTensor(allocator, dense_shape, TypeToTensorType::type); -} - -inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, - ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateSparseTensorAsOrtValue(allocator, dense_shape.shape, dense_shape.shape_len, type, &out)); - return Value{out}; -} -#endif // !defined(DISABLE_SPARSE_TENSORS) - -inline Value Value::CreateMap(const Value& keys, const Value& values) { - OrtValue* out; - const OrtValue* inputs[2] = {keys, values}; - ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out)); - return Value{out}; -} - -inline Value Value::CreateSequence(const std::vector& values) { - OrtValue* out; - std::vector values_ort{values.data(), values.data() + values.size()}; - ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out)); - return Value{out}; -} - -template -inline Value Value::CreateOpaque(const char* domain, const char* type_name, const T& data_container) { - OrtValue* out; - ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out)); - return Value{out}; -} - -// -// Custom OP Inlines -// -inline Logger::Logger(const OrtLogger* logger) : logger_(logger) { - Ort::ThrowOnError(GetApi().Logger_GetLoggingSeverityLevel(this->logger_, &this->cached_severity_level_)); -} - -inline OrtLoggingLevel Logger::GetLoggingSeverityLevel() const noexcept { - return cached_severity_level_; -} - -inline Status Logger::LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, - const char* func_name, const char* message) const noexcept { - OrtStatus* status = GetApi().Logger_LogMessage(logger_, log_severity_level, message, file_path, line_number, - func_name); - return Status{status}; -} - -// Disable warnings about the format string not being a literal (-Wformat-nonliteral and -Wformat-security) -// for gcc and clang. The alternative is to use actual C-style variadic parameters and apply -// __attribute__(format(printf...)), which does not work with variadic templates. -#if defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat-nonliteral" -#pragma GCC diagnostic ignored "-Wformat-security" -#elif defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wformat-nonliteral" -#pragma clang diagnostic ignored "-Wformat-security" -#endif -template -inline Status Logger::LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, - int line_number, const char* func_name, const char* format, - Args&&... args) const noexcept { - int msg_len = std::snprintf(nullptr, 0U, format, std::forward(args)...); - - if (msg_len < 0) { // Formatting error - return Status("Failed to log message due to formatting error", OrtErrorCode::ORT_FAIL); - } - - OrtStatus* status = nullptr; - const size_t buffer_size = static_cast(msg_len) + 1U; - - constexpr size_t kStackBufferSize = 1024; - - if (buffer_size < kStackBufferSize) { - char buffer[kStackBufferSize]; - snprintf(buffer, kStackBufferSize, format, std::forward(args)...); - status = GetApi().Logger_LogMessage(logger_, log_severity_level, buffer, file_path, line_number, func_name); - } else { - // std::make_unique is only supported starting at C++14. -#if (__cplusplus >= 201402L) || (_MSC_VER >= 1900) - auto buffer = std::make_unique(buffer_size); -#else - std::unique_ptr buffer(new char[buffer_size]); -#endif - std::snprintf(buffer.get(), buffer_size, format, std::forward(args)...); - status = GetApi().Logger_LogMessage(logger_, log_severity_level, buffer.get(), file_path, line_number, func_name); - } - - return Status{status}; -} -// Re-enable -Wformat-nonliteral and -Wformat-security -#if defined(__GNUC__) -#pragma GCC diagnostic pop -#elif defined(__clang__) -#pragma clang diagnostic pop -#endif - -inline KernelContext::KernelContext(OrtKernelContext* context) : ctx_(context) { -} - -inline size_t KernelContext::GetInputCount() const { - size_t out = 0; - Ort::ThrowOnError(GetApi().KernelContext_GetInputCount(ctx_, &out)); - return out; -} - -inline size_t KernelContext::GetOutputCount() const { - size_t out = 0; - Ort::ThrowOnError(GetApi().KernelContext_GetOutputCount(ctx_, &out)); - return out; -} - -inline ConstValue KernelContext::GetInput(size_t index) const { - const OrtValue* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetInput(ctx_, index, &out)); - return ConstValue{out}; -} - -inline UnownedValue KernelContext::GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const { - OrtValue* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dim_values, dim_count, &out)); - return UnownedValue(out); -} - -inline UnownedValue KernelContext::GetOutput(size_t index, const std::vector& dims) const { - OrtValue* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dims.data(), dims.size(), &out)); - return UnownedValue(out); -} - -inline void* KernelContext::GetGPUComputeStream() const { - void* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetGPUComputeStream(ctx_, &out)); - return out; -} - -inline OrtAllocator* KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const { - OrtAllocator* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetAllocator(ctx_, &memory_info, &out)); - return out; -} - -inline Logger KernelContext::GetLogger() const { - const OrtLogger* out = nullptr; - ThrowOnError(GetApi().KernelContext_GetLogger(this->ctx_, &out)); - return Logger{out}; -} - -inline void KernelContext::ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const { - ThrowOnError(GetApi().KernelContext_ParallelFor(ctx_, fn, total, num_batch, usr_data)); -} - -inline OpAttr::OpAttr(const char* name, const void* data, int len, OrtOpAttrType type) { - Ort::ThrowOnError(GetApi().CreateOpAttr(name, data, len, type, &p_)); -} - -namespace detail { -template -inline KernelInfo KernelInfoImpl::Copy() const { - OrtKernelInfo* info_copy = nullptr; - Ort::ThrowOnError(GetApi().CopyKernelInfo(this->p_, &info_copy)); - return KernelInfo{info_copy}; -} - -template -inline size_t KernelInfoImpl::GetInputCount() const { - size_t out = 0; - ThrowOnError(GetApi().KernelInfo_GetInputCount(this->p_, &out)); - return out; -} - -template -inline size_t KernelInfoImpl::GetOutputCount() const { - size_t out = 0; - ThrowOnError(GetApi().KernelInfo_GetOutputCount(this->p_, &out)); - return out; -} - -template -inline std::string KernelInfoImpl::GetInputName(size_t index) const { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline std::string KernelInfoImpl::GetOutputName(size_t index) const { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline TypeInfo KernelInfoImpl::GetInputTypeInfo(size_t index) const { - OrtTypeInfo* out = nullptr; - ThrowOnError(GetApi().KernelInfo_GetInputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline TypeInfo KernelInfoImpl::GetOutputTypeInfo(size_t index) const { - OrtTypeInfo* out = nullptr; - ThrowOnError(GetApi().KernelInfo_GetOutputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline Value KernelInfoImpl::GetTensorAttribute(const char* name, OrtAllocator* allocator) const { - OrtValue* out = nullptr; - ThrowOnError(GetApi().KernelInfoGetAttribute_tensor(this->p_, name, allocator, &out)); - return Value{out}; -} - -template -inline ConstValue KernelInfoImpl::GetTensorConstantInput(size_t index, int* is_constant) const { - const OrtValue* out = nullptr; - ThrowOnError(GetApi().KernelInfoGetConstantInput_tensor(this->p_, index, is_constant, &out)); - return ConstValue{out}; -} - -template -inline std::string KernelInfoImpl::GetNodeName() const { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().KernelInfo_GetNodeName(this->p_, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfo_GetNodeName(this->p_, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline Logger KernelInfoImpl::GetLogger() const { - const OrtLogger* out = nullptr; - ThrowOnError(GetApi().KernelInfo_GetLogger(this->p_, &out)); - return Logger{out}; -} - -inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, float& out) { - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_float(p, name, &out)); -} - -inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, int64_t& out) { - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_int64(p, name, &out)); -} - -inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, std::string& result) { - size_t size = 0; - // Feed nullptr for the data buffer to query the true size of the string attribute - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_string(p, name, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_string(p, name, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - out.swap(result); -} - -inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std::vector& result) { - size_t size = 0; - // Feed nullptr for the data buffer to query the true size of the attribute - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_float(p, name, nullptr, &size)); - - std::vector out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_float(p, name, out.data(), &size)); - out.swap(result); -} - -inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std::vector& result) { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the attribute - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_int64(p, name, nullptr, &size)); - - std::vector out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_int64(p, name, out.data(), &size)); - out.swap(result); -} -} // namespace detail - -inline KernelInfo::KernelInfo(OrtKernelInfo* info) : detail::KernelInfoImpl{info} {} - -inline Op::Op(OrtOp* p) : Base(p) {} - -inline Op Op::Create(const OrtKernelInfo* info, const char* op_name, const char* domain, int version, - const char** type_constraint_names, - const ONNXTensorElementDataType* type_constraint_values, - size_t type_constraint_count, - const OpAttr* attr_values, size_t attr_count, - size_t input_count, size_t output_count) { - static_assert(sizeof(OpAttr) == sizeof(OrtOpAttr*), - "OpAttr's is expected to be just an array of OrtOpAttr in memory so we can reinterpret safely"); - auto attr_input_values = reinterpret_cast(attr_values); - OrtOp* op; - Ort::ThrowOnError(GetApi().CreateOp(info, op_name, domain, version, type_constraint_names, type_constraint_values, - static_cast(type_constraint_count), - attr_input_values, - static_cast(attr_count), - static_cast(input_count), - static_cast(output_count), &op)); - return Op{op}; -} - -inline void Op::Invoke(const OrtKernelContext* context, - const Value* input_values, - size_t input_count, - Value* output_values, - size_t output_count) { - static_assert(sizeof(Value) == sizeof(OrtValue*), - "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); - auto ort_input_values = reinterpret_cast(input_values); - auto ort_output_values = reinterpret_cast(output_values); - Ort::ThrowOnError(GetApi().InvokeOp(context, p_, ort_input_values, static_cast(input_count), - ort_output_values, static_cast(output_count))); -} - -inline void Op::Invoke(const OrtKernelContext* context, - const OrtValue* const* input_values, - size_t input_count, - OrtValue* const* output_values, - size_t output_count) { - Ort::ThrowOnError(GetApi().InvokeOp(context, p_, input_values, static_cast(input_count), - output_values, static_cast(output_count))); -} - -inline std::string GetVersionString() { - return OrtGetApiBase()->GetVersionString(); -} - -inline std::string GetBuildInfoString() { - return GetApi().GetBuildInfoString(); -} - -inline std::vector GetAvailableProviders() { - char** providers; - int len; - - auto release_fn = [&len](char** providers) { - // This should always return nullptr. - ThrowOnError(GetApi().ReleaseAvailableProviders(providers, len)); - }; - - ThrowOnError(GetApi().GetAvailableProviders(&providers, &len)); - std::unique_ptr guard(providers, release_fn); - std::vector available_providers; - available_providers.reserve(static_cast(len)); - for (int i = 0; i < len; ++i) { - available_providers.emplace_back(providers[i]); - } - return available_providers; -} - -template -void CustomOpBase::GetSessionConfigs(std::unordered_map& out, - ConstSessionOptions options) const { - const TOp* derived = static_cast(this); - std::vector keys = derived->GetSessionConfigKeys(); - - out.reserve(keys.size()); - - std::string config_entry_key = detail::MakeCustomOpConfigEntryKey(derived->GetName(), ""); - const size_t prefix_size = config_entry_key.length(); - - for (const auto& key : keys) { - config_entry_key.resize(prefix_size); - config_entry_key.append(key); - out[key] = options.GetConfigEntryOrDefault(config_entry_key.c_str(), ""); - } -} - -inline ShapeInferContext::ShapeInferContext(const OrtApi* ort_api, - OrtShapeInferContext* ctx) : ort_api_(ort_api), ctx_(ctx) { - size_t input_count = 0; - Ort::ThrowOnError(ort_api_->ShapeInferContext_GetInputCount(ctx_, &input_count)); - for (size_t ith_input = 0; ith_input < input_count; ++ith_input) { - OrtTensorTypeAndShapeInfo* info{}; - Ort::ThrowOnError(ort_api_->ShapeInferContext_GetInputTypeShape(ctx, ith_input, &info)); - TensorTypeAndShapeInfo type_shape_info(info); - auto integer_shape = type_shape_info.GetShape(); - std::vector symbolic_shape(integer_shape.size(), {}); - type_shape_info.GetSymbolicDimensions(&symbolic_shape[0], integer_shape.size()); - Shape shape; - for (size_t ith = 0; ith < integer_shape.size(); ++ith) { - if (symbolic_shape[ith] && std::string{symbolic_shape[ith]}.size() > 0) { - shape.emplace_back(symbolic_shape[ith]); - } else { - shape.emplace_back(integer_shape[ith]); - } - } - input_shapes_.push_back(std::move(shape)); - type_shape_info.release(); - } -} - -inline Status ShapeInferContext::SetOutputShape(size_t indice, const Shape& shape) { - OrtTensorTypeAndShapeInfo* info = {}; - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->CreateTensorTypeAndShapeInfo(&info)); - - using InfoPtr = std::unique_ptr>; - - InfoPtr info_ptr(info, [this](OrtTensorTypeAndShapeInfo* obj) { - ort_api_->ReleaseTensorTypeAndShapeInfo(obj); - }); - - std::vector integer_dims; - std::vector symbolic_dims; - - for (const auto dim : shape) { - if (dim.IsInt()) { - integer_dims.push_back(dim.IsInt()); - symbolic_dims.push_back(""); - } else { - if (!dim.AsSym() || std::string{dim.AsSym()}.empty()) { - ORT_CXX_API_THROW("Symbolic dim must not be an empty string", ORT_INVALID_ARGUMENT); - } - integer_dims.push_back(SymbolicInteger::INVALID_INT_DIM); - symbolic_dims.push_back(dim.AsSym()); - } - } - - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetDimensions(info, integer_dims.data(), integer_dims.size())); - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetSymbolicDimensions(info, symbolic_dims.data(), symbolic_dims.size())); - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->ShapeInferContext_SetOutputTypeShape(ctx_, indice, info)); - return Status{nullptr}; -} - -inline int64_t ShapeInferContext::GetAttrInt(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - int64_t i = {}; - size_t out = {}; - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INT, &i, sizeof(i), &out)); - return i; -} - -inline ShapeInferContext::Ints ShapeInferContext::GetAttrInts(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - int64_t i = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, &i, sizeof(i), &out); - if (status) { - size_t num_i = out / sizeof(int64_t); - ShapeInferContext::Ints ints(num_i, 0); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, ints.data(), out, &out)); - return ints; - } else { - return {i}; - } -} - -inline float ShapeInferContext::GetAttrFloat(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - float f = {}; - size_t out = {}; - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOAT, &f, sizeof(f), &out)); - return f; -} - -inline ShapeInferContext::Floats ShapeInferContext::GetAttrFloats(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - float f = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, &f, sizeof(f), &out); - if (status) { - size_t num_f = out / sizeof(float); - ShapeInferContext::Floats floats(num_f, 0); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, floats.data(), out, &out)); - return floats; - } else { - return {f}; - } -} - -inline std::string ShapeInferContext::GetAttrString(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - char c = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, &c, sizeof(char), &out); - if (status) { - std::vector chars(out, '\0'); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, chars.data(), out, &out)); - return {chars.data()}; - } else { - return {c}; - } -} - -inline ShapeInferContext::Strings ShapeInferContext::GetAttrStrings(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - char c = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, &c, sizeof(char), &out); - if (status) { - std::vector chars(out, '\0'); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, chars.data(), out, &out)); - ShapeInferContext::Strings strings; - char* char_st = chars.data(); - char* char_ed = char_st + out; - while (char_st < char_ed) { - strings.emplace_back(char_st); - while (*char_st != '\0') { - char_st++; - } - char_st++; - } - return strings; - } else { - return {std::string{c}}; - } -} - -inline const OrtOpAttr* ShapeInferContext::GetAttrHdl(const char* attr_name) const { - const OrtOpAttr* attr_hdl = {}; - Ort::ThrowOnError(ort_api_->ShapeInferContext_GetAttribute(ctx_, attr_name, &attr_hdl)); - return attr_hdl; -} - -} // namespace Ort diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_float16.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_float16.h deleted file mode 100644 index 0b066a9..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_float16.h +++ /dev/null @@ -1,540 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include -#include -#include -#include - -namespace onnxruntime_float16 { - -namespace detail { - -enum class endian { -#if defined(_WIN32) - little = 0, - big = 1, - native = little, -#elif defined(__GNUC__) || defined(__clang__) - little = __ORDER_LITTLE_ENDIAN__, - big = __ORDER_BIG_ENDIAN__, - native = __BYTE_ORDER__, -#else -#error onnxruntime_float16::detail::endian is not implemented in this environment. -#endif -}; - -static_assert( - endian::native == endian::little || endian::native == endian::big, - "Only little-endian or big-endian native byte orders are supported."); - -} // namespace detail - -/// -/// Shared implementation between public and internal classes. CRTP pattern. -/// -template -struct Float16Impl { - protected: - /// - /// Converts from float to uint16_t float16 representation - /// - /// - /// - constexpr static uint16_t ToUint16Impl(float v) noexcept; - - /// - /// Converts float16 to float - /// - /// float representation of float16 value - float ToFloatImpl() const noexcept; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - uint16_t AbsImpl() const noexcept { - return static_cast(val & ~kSignMask); - } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - uint16_t NegateImpl() const noexcept { - return IsNaN() ? val : static_cast(val ^ kSignMask); - } - - public: - // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7C00U; - static constexpr uint16_t kPositiveInfinityBits = 0x7C00U; - static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; - static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; - static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; - static constexpr uint16_t kEpsilonBits = 0x4170U; - static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number - static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number - static constexpr uint16_t kOneBits = 0x3C00U; - static constexpr uint16_t kMinusOneBits = 0xBC00U; - - uint16_t val{0}; - - Float16Impl() = default; - - /// - /// Checks if the value is negative - /// - /// true if negative - bool IsNegative() const noexcept { - return static_cast(val) < 0; - } - - /// - /// Tests if the value is NaN - /// - /// true if NaN - bool IsNaN() const noexcept { - return AbsImpl() > kPositiveInfinityBits; - } - - /// - /// Tests if the value is finite - /// - /// true if finite - bool IsFinite() const noexcept { - return AbsImpl() < kPositiveInfinityBits; - } - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - bool IsPositiveInfinity() const noexcept { - return val == kPositiveInfinityBits; - } - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - bool IsNegativeInfinity() const noexcept { - return val == kNegativeInfinityBits; - } - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - bool IsInfinity() const noexcept { - return AbsImpl() == kPositiveInfinityBits; - } - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - bool IsNaNOrZero() const noexcept { - auto abs = AbsImpl(); - return (abs == 0 || abs > kPositiveInfinityBits); - } - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - bool IsNormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) - } - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - bool IsSubnormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) - } - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) noexcept { - return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; - } - - bool operator==(const Float16Impl& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is not equal to anything, including itself. - return false; - } - return val == rhs.val; - } - - bool operator!=(const Float16Impl& rhs) const noexcept { return !(*this == rhs); } - - bool operator<(const Float16Impl& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is unordered with respect to everything, including itself. - return false; - } - - const bool left_is_negative = IsNegative(); - if (left_is_negative != rhs.IsNegative()) { - // When the signs of left and right differ, we know that left is less than right if it is - // the negative value. The exception to this is if both values are zero, in which case IEEE - // says they should be equal, even if the signs differ. - return left_is_negative && !AreZero(*this, rhs); - } - return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); - } -}; - -// The following Float16_t conversions are based on the code from -// Eigen library. - -// The conversion routines are Copyright (c) Fabian Giesen, 2016. -// The original license follows: -// -// Copyright (c) Fabian Giesen, 2016 -// All rights reserved. -// Redistribution and use in source and binary forms, with or without -// modification, are permitted. -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -namespace detail { -union float32_bits { - unsigned int u; - float f; -}; -} // namespace detail - -template -inline constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept { - detail::float32_bits f{}; - f.f = v; - - constexpr detail::float32_bits f32infty = {255 << 23}; - constexpr detail::float32_bits f16max = {(127 + 16) << 23}; - constexpr detail::float32_bits denorm_magic = {((127 - 15) + (23 - 10) + 1) << 23}; - constexpr unsigned int sign_mask = 0x80000000u; - uint16_t val = static_cast(0x0u); - - unsigned int sign = f.u & sign_mask; - f.u ^= sign; - - // NOTE all the integer compares in this function can be safely - // compiled into signed compares since all operands are below - // 0x80000000. Important if you want fast straight SSE2 code - // (since there's no unsigned PCMPGTD). - - if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) - val = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf - } else { // (De)normalized number or zero - if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero - // use a magic value to align our 10 mantissa bits at the bottom of - // the float. as long as FP addition is round-to-nearest-even this - // just works. - f.f += denorm_magic.f; - - // and one integer subtract of the bias later, we have our final float! - val = static_cast(f.u - denorm_magic.u); - } else { - unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd - - // update exponent, rounding bias part 1 - // Equivalent to `f.u += ((unsigned int)(15 - 127) << 23) + 0xfff`, but - // without arithmetic overflow. - f.u += 0xc8000fffU; - // rounding bias part 2 - f.u += mant_odd; - // take the bits! - val = static_cast(f.u >> 13); - } - } - - val |= static_cast(sign >> 16); - return val; -} - -template -inline float Float16Impl::ToFloatImpl() const noexcept { - constexpr detail::float32_bits magic = {113 << 23}; - constexpr unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift - detail::float32_bits o{}; - - o.u = (val & 0x7fff) << 13; // exponent/mantissa bits - unsigned int exp = shifted_exp & o.u; // just the exponent - o.u += (127 - 15) << 23; // exponent adjust - - // handle exponent special cases - if (exp == shifted_exp) { // Inf/NaN? - o.u += (128 - 16) << 23; // extra exp adjust - } else if (exp == 0) { // Zero/Denormal? - o.u += 1 << 23; // extra exp adjust - o.f -= magic.f; // re-normalize - } - - // Attempt to workaround the Internal Compiler Error on ARM64 - // for bitwise | operator, including std::bitset -#if (defined _MSC_VER) && (defined _M_ARM || defined _M_ARM64 || defined _M_ARM64EC) - if (IsNegative()) { - return -o.f; - } -#else - // original code: - o.u |= (val & 0x8000U) << 16U; // sign bit -#endif - return o.f; -} - -/// Shared implementation between public and internal classes. CRTP pattern. -template -struct BFloat16Impl { - protected: - /// - /// Converts from float to uint16_t float16 representation - /// - /// - /// - static uint16_t ToUint16Impl(float v) noexcept; - - /// - /// Converts bfloat16 to float - /// - /// float representation of bfloat16 value - float ToFloatImpl() const noexcept; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - uint16_t AbsImpl() const noexcept { - return static_cast(val & ~kSignMask); - } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - uint16_t NegateImpl() const noexcept { - return IsNaN() ? val : static_cast(val ^ kSignMask); - } - - public: - // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7F80U; - static constexpr uint16_t kPositiveInfinityBits = 0x7F80U; - static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; - static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; - static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; - static constexpr uint16_t kSignaling_NaNBits = 0x7F80U; - static constexpr uint16_t kEpsilonBits = 0x0080U; - static constexpr uint16_t kMinValueBits = 0xFF7FU; - static constexpr uint16_t kMaxValueBits = 0x7F7FU; - static constexpr uint16_t kRoundToNearest = 0x7FFFU; - static constexpr uint16_t kOneBits = 0x3F80U; - static constexpr uint16_t kMinusOneBits = 0xBF80U; - - uint16_t val{0}; - - BFloat16Impl() = default; - - /// - /// Checks if the value is negative - /// - /// true if negative - bool IsNegative() const noexcept { - return static_cast(val) < 0; - } - - /// - /// Tests if the value is NaN - /// - /// true if NaN - bool IsNaN() const noexcept { - return AbsImpl() > kPositiveInfinityBits; - } - - /// - /// Tests if the value is finite - /// - /// true if finite - bool IsFinite() const noexcept { - return AbsImpl() < kPositiveInfinityBits; - } - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - bool IsPositiveInfinity() const noexcept { - return val == kPositiveInfinityBits; - } - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - bool IsNegativeInfinity() const noexcept { - return val == kNegativeInfinityBits; - } - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - bool IsInfinity() const noexcept { - return AbsImpl() == kPositiveInfinityBits; - } - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - bool IsNaNOrZero() const noexcept { - auto abs = AbsImpl(); - return (abs == 0 || abs > kPositiveInfinityBits); - } - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - bool IsNormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) - } - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - bool IsSubnormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) - } - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) noexcept { - // IEEE defines that positive and negative zero are equal, this gives us a quick equality check - // for two values by or'ing the private bits together and stripping the sign. They are both zero, - // and therefore equivalent, if the resulting value is still zero. - return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; - } -}; - -template -inline uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept { - uint16_t result; - if (std::isnan(v)) { - result = kPositiveQNaNBits; - } else { - auto get_msb_half = [](float fl) { - uint16_t result; -#ifdef __cpp_if_constexpr - if constexpr (detail::endian::native == detail::endian::little) { -#else - if (detail::endian::native == detail::endian::little) { -#endif - std::memcpy(&result, reinterpret_cast(&fl) + sizeof(uint16_t), sizeof(uint16_t)); - } else { - std::memcpy(&result, &fl, sizeof(uint16_t)); - } - return result; - }; - - uint16_t upper_bits = get_msb_half(v); - union { - uint32_t U32; - float F32; - }; - F32 = v; - U32 += (upper_bits & 1) + kRoundToNearest; - result = get_msb_half(F32); - } - return result; -} - -template -inline float BFloat16Impl::ToFloatImpl() const noexcept { - if (IsNaN()) { - return std::numeric_limits::quiet_NaN(); - } - float result; - char* const first = reinterpret_cast(&result); - char* const second = first + sizeof(uint16_t); -#ifdef __cpp_if_constexpr - if constexpr (detail::endian::native == detail::endian::little) { -#else - if (detail::endian::native == detail::endian::little) { -#endif - std::memset(first, 0, sizeof(uint16_t)); - std::memcpy(second, &val, sizeof(uint16_t)); - } else { - std::memcpy(first, &val, sizeof(uint16_t)); - std::memset(second, 0, sizeof(uint16_t)); - } - return result; -} - -} // namespace onnxruntime_float16 diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_lite_custom_op.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_lite_custom_op.h deleted file mode 100644 index ee60f25..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_lite_custom_op.h +++ /dev/null @@ -1,1119 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Summary -// The header has APIs to save custom op authors the trouble of defining schemas, -// which will be inferred by functions' signature, as long as their argument list has types supported here. -// Input could be: -// 1. Tensor of onnx data types. -// 2. Span of onnx data types. -// 3. Scalar of onnx data types. -// A input could be optional if indicated as std::optional<...>. -// For an output, it must be a tensor of onnx data types. -// Further, the header also has utility for a simple custom struct, where resources could be kept, to be registered as a custom op. -// For concrete examples, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". -// Note - all APIs in this header are ABI. - -#pragma once -#include "onnxruntime_cxx_api.h" -#include -#include -#include -#include - -namespace Ort { -namespace Custom { - -class ArgBase { - public: - ArgBase(OrtKernelContext* ctx, - size_t indice, - bool is_input) : ctx_(ctx), indice_(indice), is_input_(is_input) {} - virtual ~ArgBase(){}; - - protected: - struct KernelContext ctx_; - size_t indice_; - bool is_input_; -}; - -using ArgPtr = std::unique_ptr; -using ArgPtrs = std::vector; - -class TensorBase : public ArgBase { - public: - TensorBase(OrtKernelContext* ctx, - size_t indice, - bool is_input) : ArgBase(ctx, indice, is_input) {} - - operator bool() const { - return shape_.has_value(); - } - - const std::vector& Shape() const { - if (!shape_.has_value()) { - ORT_CXX_API_THROW("tensor shape is not yet initialized", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return shape_.value(); - } - - ONNXTensorElementDataType Type() const { - return type_; - } - - int64_t NumberOfElement() const { - if (shape_.has_value()) { - return std::accumulate(shape_->begin(), shape_->end(), 1LL, std::multiplies()); - } else { - return 0; - } - } - - std::string Shape2Str() const { - if (shape_.has_value()) { - std::string shape_str; - for (const auto& dim : *shape_) { - shape_str.append(std::to_string(dim)); - shape_str.append(", "); - } - return shape_str; - } else { - return "empty"; - } - } - - bool IsCpuTensor() const { - return strcmp("Cpu", mem_type_) == 0; - } - - virtual const void* DataRaw() const = 0; - virtual size_t SizeInBytes() const = 0; - - protected: - std::optional> shape_; - ONNXTensorElementDataType type_ = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; - const char* mem_type_ = "Cpu"; -}; - -template -struct Span { - const T* data_ = {}; - size_t size_ = {}; - void Assign(const T* data, size_t size) { - data_ = data; - size_ = size; - } - size_t size() const { return size_; } - T operator[](size_t indice) const { - return data_[indice]; - } - const T* data() const { return data_; } -}; - -template -class Tensor : public TensorBase { - public: - using TT = typename std::remove_reference::type; - Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { - if (is_input_) { - if (indice >= ctx_.GetInputCount()) { - ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - const_value_ = ctx_.GetInput(indice); - auto type_shape_info = const_value_.GetTensorTypeAndShapeInfo(); - shape_ = type_shape_info.GetShape(); - } - } - const TT* Data() const { - return reinterpret_cast(const_value_.GetTensorRawData()); - } - TT* Allocate(const std::vector& shape) { - shape_ = shape; - if (!data_) { - shape_ = shape; - data_ = ctx_.GetOutput(indice_, shape).template GetTensorMutableData(); - } - return data_; - } - static TT GetT() { return (TT)0; } - const Span& AsSpan() { - if (!shape_.has_value() || shape_->size() != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a span out of Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - span_.Assign(Data(), static_cast((*shape_)[0])); - return span_; - } - const T& AsScalar() { - if (!shape_.has_value() || shape_->size() != 1 || (*shape_)[0] != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a scalar from Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return *Data(); - } - const void* DataRaw() const override { - return reinterpret_cast(Data()); - } - - size_t SizeInBytes() const override { - return sizeof(TT) * static_cast(NumberOfElement()); - } - - private: - ConstValue const_value_; // for input - TT* data_{}; // for output - Span span_; -}; - -template <> -class Tensor : public TensorBase { - public: - using strings = std::vector; - - Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { - if (is_input_) { - if (indice >= ctx_.GetInputCount()) { - ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - auto const_value = ctx_.GetInput(indice); - auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); - shape_ = type_shape_info.GetShape(); - auto num_chars = const_value.GetStringTensorDataLength(); - // note - there will be copy ... - auto num_strings = static_cast(NumberOfElement()); - if (num_strings) { - std::vector chars(num_chars + 1, '\0'); - std::vector offsets(num_strings); - const_value.GetStringTensorContent(static_cast(chars.data()), num_chars, offsets.data(), offsets.size()); - auto upper_bound = num_strings - 1; - input_strings_.resize(num_strings); - for (size_t i = upper_bound;; --i) { - if (i < upper_bound) { - chars[offsets[i + 1]] = '\0'; - } - input_strings_[i] = chars.data() + offsets[i]; - if (0 == i) { - break; - } - } - } - } - } - const strings& Data() const { - return input_strings_; - } - const void* DataRaw() const override { - if (input_strings_.size() != 1) { - ORT_CXX_API_THROW("DataRaw() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return reinterpret_cast(input_strings_[0].c_str()); - } - size_t SizeInBytes() const override { - if (input_strings_.size() != 1) { - ORT_CXX_API_THROW("SizeInBytes() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return input_strings_[0].size(); - } - void SetStringOutput(const strings& ss, const std::vector& dims) { - shape_ = dims; - std::vector raw; - for (const auto& s : ss) { - raw.push_back(s.data()); - } - auto output = ctx_.GetOutput(indice_, dims.data(), dims.size()); - // note - there will be copy ... - output.FillStringTensor(raw.data(), raw.size()); - } - const Span& AsSpan() { - ORT_CXX_API_THROW("span for TensorT of string not implemented", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - const std::string& AsScalar() { - if (input_strings_.size() != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a scalar string from Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return input_strings_[0]; - } - - private: - std::vector input_strings_; // for input -}; - -template <> -class Tensor : public TensorBase { - public: - using strings = std::vector; - using string_views = std::vector; - - Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { - if (is_input_) { - if (indice >= ctx_.GetInputCount()) { - ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - auto const_value = ctx_.GetInput(indice); - auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); - shape_ = type_shape_info.GetShape(); - auto num_chars = const_value.GetStringTensorDataLength(); - chars_.resize(num_chars + 1, '\0'); - auto num_strings = static_cast(NumberOfElement()); - if (num_strings) { - std::vector offsets(num_strings); - const_value.GetStringTensorContent(static_cast(chars_.data()), num_chars, offsets.data(), offsets.size()); - offsets.push_back(num_chars); - for (size_t i = 0; i < num_strings; ++i) { - input_string_views_.emplace_back(chars_.data() + offsets[i], offsets[i + 1] - offsets[i]); - } - } - } - } - const string_views& Data() const { - return input_string_views_; - } - const void* DataRaw() const override { - if (input_string_views_.size() != 1) { - ORT_CXX_API_THROW("DataRaw() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return reinterpret_cast(input_string_views_[0].data()); - } - size_t SizeInBytes() const override { - if (input_string_views_.size() != 1) { - ORT_CXX_API_THROW("SizeInBytes() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return input_string_views_[0].size(); - } - void SetStringOutput(const strings& ss, const std::vector& dims) { - shape_ = dims; - std::vector raw; - for (const auto& s : ss) { - raw.push_back(s.data()); - } - auto output = ctx_.GetOutput(indice_, dims.data(), dims.size()); - // note - there will be copy ... - output.FillStringTensor(raw.data(), raw.size()); - } - const Span& AsSpan() { - ORT_CXX_API_THROW("span for TensorT of string view not implemented", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - std::string_view AsScalar() { - if (input_string_views_.size() != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a scalar string view from Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return input_string_views_[0]; - } - - private: - std::vector chars_; // for input - std::vector input_string_views_; // for input -}; - -using TensorPtr = std::unique_ptr; -using TensorPtrs = std::vector; - -struct TensorArray : public ArgBase { - TensorArray(OrtKernelContext* ctx, - size_t start_indice, - bool is_input) : ArgBase(ctx, - start_indice, - is_input) { - if (is_input) { - auto input_count = ctx_.GetInputCount(); - for (size_t ith_input = start_indice; ith_input < input_count; ++ith_input) { - auto const_value = ctx_.GetInput(start_indice); - auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); - auto type = type_shape_info.GetElementType(); - TensorPtr tensor; - switch (type) { - case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: - tensor = std::make_unique>(ctx, ith_input, true); - break; - default: - ORT_CXX_API_THROW("unknow input type", ORT_RUNTIME_EXCEPTION); - break; - } - tensors_.emplace_back(tensor.release()); - } // for - } - } - template - T* AllocateOutput(size_t ith_output, const std::vector& shape) { - // ith_output is the indice of output relative to the tensor array - // indice_ + ith_output is the indice relative to context - auto tensor = std::make_unique>(ctx_.GetOrtKernelContext(), indice_ + ith_output, false); - auto raw_output = tensor.get()->Allocate(shape); - tensors_.emplace_back(tensor.release()); - return raw_output; - } - Tensor& AllocateStringTensor(size_t ith_output) { - // ith_output is the indice of output relative to the tensor array - // indice_ + ith_output is the indice relative to context - auto tensor = std::make_unique>(ctx_.GetOrtKernelContext(), indice_ + ith_output, false); - Tensor& output = *tensor; - tensors_.emplace_back(tensor.release()); - return output; - } - size_t Size() const { - return tensors_.size(); - } - const TensorPtr& operator[](size_t ith_input) const { - // ith_input is the indice of output relative to the tensor array - return tensors_.at(ith_input); - } - - private: - TensorPtrs tensors_; -}; - -using Variadic = TensorArray; - -/* -Note: -OrtLiteCustomOp inherits from OrtCustomOp to bridge tween a custom func/struct and ort core. -The lifetime of an OrtLiteCustomOp instance is managed by customer code, not ort, so: -1. DO NOT cast OrtLiteCustomOp to OrtCustomOp and release since there is no virtual destructor in the hierachy. -2. OrtLiteCustomFunc and OrtLiteCustomStruct, as two sub-structs, can be released in form of OrtLiteCustomOp since all members are kept in the OrtLiteCustomOp, - hence memory could still be recycled properly. -Further, OrtCustomOp is a c struct bearing no v-table, so offspring structs are by design to be of zero virtual functions to maintain cast safety. -*/ -struct OrtLiteCustomOp : public OrtCustomOp { - using ConstOptionalFloatTensor = std::optional&>; - using OptionalFloatTensor = std::optional>; - - // CreateTuple - template - static typename std::enable_if>::type - CreateTuple(OrtKernelContext*, ArgPtrs&, size_t, size_t, const std::string&) { - return std::make_tuple(); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - std::tuple current = std::tuple{context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - std::tuple current = std::tuple{*context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - -#ifdef ORT_CUDA_CTX - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - thread_local CudaContext cuda_context; - cuda_context.Init(*context); - std::tuple current = std::tuple{cuda_context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } -#endif - -#ifdef ORT_ROCM_CTX - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - thread_local RocmContext rocm_context; - rocm_context.Init(*context); - std::tuple current = std::tuple{rocm_context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } -#endif - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_input, true)); - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_input, true)); - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_output, false)); - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_output, false)); - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - -#define CREATE_TUPLE_INPUT(data_type) \ - template \ - static typename std::enable_if*>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if&>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if*>>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_input < num_input) { \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } \ - template \ - static typename std::enable_if*>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{&reinterpret_cast*>(args.back().get())->AsSpan()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if&>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsSpan()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if*>>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_input < num_input) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{&reinterpret_cast*>(args.back().get())->AsSpan()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } \ - template \ - static typename std::enable_if::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("scalar input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsScalar()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_input < num_input) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("scalar input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsScalar()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } -#define CREATE_TUPLE_OUTPUT(data_type) \ - template \ - static typename std::enable_if*>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_output, false)); \ - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if&>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_output, false)); \ - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if*>>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_output < num_output) { \ - args.push_back(std::make_unique>(context, ith_output, false)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } -#define CREATE_TUPLE(data_type) \ - CREATE_TUPLE_INPUT(data_type) \ - CREATE_TUPLE_OUTPUT(data_type) - - CREATE_TUPLE(bool) - CREATE_TUPLE(float) - CREATE_TUPLE(Ort::Float16_t) - CREATE_TUPLE(Ort::BFloat16_t) - CREATE_TUPLE(double) - CREATE_TUPLE(int8_t) - CREATE_TUPLE(int16_t) - CREATE_TUPLE(int32_t) - CREATE_TUPLE(int64_t) - CREATE_TUPLE(uint8_t) - CREATE_TUPLE(uint16_t) - CREATE_TUPLE(uint32_t) - CREATE_TUPLE(uint64_t) - CREATE_TUPLE(std::string) - CREATE_TUPLE_INPUT(std::string_view) - CREATE_TUPLE(Ort::Float8E4M3FN_t) - CREATE_TUPLE(Ort::Float8E4M3FNUZ_t) - CREATE_TUPLE(Ort::Float8E5M2_t) - CREATE_TUPLE(Ort::Float8E5M2FNUZ_t) - - // ParseArgs ... - template - static typename std::enable_if<0 == sizeof...(Ts)>::type - ParseArgs(std::vector&, std::vector&) { - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } - -#ifdef ORT_CUDA_CTX - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } -#endif - -#ifdef ORT_ROCM_CTX - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } -#endif - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - input_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - input_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - output_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - output_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - -#define PARSE_INPUT_BASE(pack_type, onnx_type) \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - input_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - input_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - input_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } - -#define PARSE_INPUT(data_type, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Tensor*, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Tensor&, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Span*, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Span&, onnx_type) \ - PARSE_INPUT_BASE(data_type, onnx_type) - -#define PARSE_OUTPUT(data_type, onnx_type) \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same*>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - output_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same&>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - output_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same*>>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - output_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } - -#define PARSE_ARGS(data_type, onnx_type) \ - PARSE_INPUT(data_type, onnx_type) \ - PARSE_OUTPUT(data_type, onnx_type) - - PARSE_ARGS(bool, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL) - PARSE_ARGS(float, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) - PARSE_ARGS(Ort::Float16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) - PARSE_ARGS(Ort::BFloat16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16) - PARSE_ARGS(double, ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE) - PARSE_ARGS(int8_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) - PARSE_ARGS(int16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16) - PARSE_ARGS(int32_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) - PARSE_ARGS(int64_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) - PARSE_ARGS(uint8_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8) - PARSE_ARGS(uint16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16) - PARSE_ARGS(uint32_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32) - PARSE_ARGS(uint64_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64) - PARSE_ARGS(std::string, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) - PARSE_ARGS(std::string_view, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) // todo - remove string_view output - PARSE_ARGS(Ort::Float8E4M3FN_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN) - PARSE_ARGS(Ort::Float8E4M3FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ) - PARSE_ARGS(Ort::Float8E5M2_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2) - PARSE_ARGS(Ort::Float8E5M2FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ) - - OrtLiteCustomOp(const char* op_name, - const char* execution_provider, - ShapeInferFn shape_infer_fn, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : op_name_(op_name), - execution_provider_(execution_provider), - shape_infer_fn_(shape_infer_fn), - start_ver_(start_ver), - end_ver_(end_ver) { - OrtCustomOp::version = ORT_API_VERSION; - - OrtCustomOp::GetName = [](const OrtCustomOp* op) { return static_cast(op)->op_name_.c_str(); }; - OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* op) { return ((OrtLiteCustomOp*)op)->execution_provider_.c_str(); }; - OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp*, size_t) { return OrtMemTypeDefault; }; - - OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->input_types_.size(); - }; - - OrtCustomOp::GetInputType = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->input_types_[indice]; - }; - - OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->output_types_.size(); - }; - - OrtCustomOp::GetOutputType = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->output_types_[indice]; - }; - - OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->input_types_[indice] == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED ? INPUT_OUTPUT_VARIADIC : INPUT_OUTPUT_OPTIONAL; - }; - - OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->output_types_[indice] == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED ? INPUT_OUTPUT_VARIADIC : INPUT_OUTPUT_OPTIONAL; - }; - - OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp*) { - return 1; - }; - - OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp*) { - return 0; - }; - - OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp*) { - return 1; - }; - - OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp*) { - return 0; - }; - - OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp*) { return 0; }; - OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp*) { return 0; }; - OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp*) { return 0; }; - OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp*) { return 0; }; - - OrtCustomOp::CreateKernelV2 = {}; - OrtCustomOp::KernelComputeV2 = {}; - OrtCustomOp::KernelCompute = {}; - - OrtCustomOp::InferOutputShapeFn = {}; - - OrtCustomOp::GetStartVersion = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->start_ver_; - }; - - OrtCustomOp::GetEndVersion = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->end_ver_; - }; - - OrtCustomOp::GetMayInplace = {}; - OrtCustomOp::ReleaseMayInplace = {}; - OrtCustomOp::GetAliasMap = {}; - OrtCustomOp::ReleaseAliasMap = {}; - } - - const std::string op_name_; - const std::string execution_provider_; - - std::vector input_types_; - std::vector output_types_; - - ShapeInferFn shape_infer_fn_ = {}; - - int start_ver_ = 1; - int end_ver_ = MAX_CUSTOM_OP_END_VER; - - void* compute_fn_ = {}; - void* compute_fn_return_status_ = {}; -}; - -//////////////////////////// OrtLiteCustomFunc //////////////////////////////// -// The struct is to implement function-as-op. -// E.g. a function might be defined as: -// void Filter(const Ort::Custom::Tensor& floats_in, Ort::Custom::Tensor& floats_out) { ... } -// It could be registered this way: -// Ort::CustomOpDomain v2_domain{"v2"}; -// std::unique_ptr fil_op_ptr{Ort::Custom::CreateLiteCustomOp("Filter", "CPUExecutionProvider", Filter)}; -// v2_domain.Add(fil_op_ptr.get()); -// session_options.Add(v2_domain); -// For the complete example, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". -template -struct OrtLiteCustomFunc : public OrtLiteCustomOp { - using ComputeFn = void (*)(Args...); - using ComputeFnReturnStatus = Status (*)(Args...); - using MyType = OrtLiteCustomFunc; - - struct Kernel { - size_t num_input_{}; - size_t num_output_{}; - ComputeFn compute_fn_{}; - ComputeFnReturnStatus compute_fn_return_status_{}; - std::string ep_{}; - }; - - OrtLiteCustomFunc(const char* op_name, - const char* execution_provider, - ComputeFn compute_fn, - ShapeInferFn shape_infer_fn = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, shape_infer_fn, start_ver, end_ver) { - compute_fn_ = reinterpret_cast(compute_fn); - ParseArgs(input_types_, output_types_); - - OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { - auto kernel = reinterpret_cast(op_kernel); - std::vector args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - std::apply([kernel](Args const&... t_args) { kernel->compute_fn_(t_args...); }, t); - }; - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { - auto kernel = std::make_unique(); - auto me = static_cast(this_); - kernel->compute_fn_ = reinterpret_cast(me->compute_fn_); - Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); - Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); - auto self = static_cast(this_); - kernel->ep_ = self->execution_provider_; - return reinterpret_cast(kernel.release()); - }; - - OrtCustomOp::KernelDestroy = [](void* op_kernel) { - delete reinterpret_cast(op_kernel); - }; - - if (shape_infer_fn_) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp* op, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - auto shape_info_fn = static_cast(op)->shape_infer_fn_; - ShapeInferContext ctx(&GetApi(), ort_ctx); - return shape_info_fn(ctx); - }; - } - } - - OrtLiteCustomFunc(const char* op_name, - const char* execution_provider, - ComputeFnReturnStatus compute_fn_return_status, - ShapeInferFn shape_infer_fn = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, shape_infer_fn, start_ver, end_ver) { - compute_fn_return_status_ = reinterpret_cast(compute_fn_return_status); - ParseArgs(input_types_, output_types_); - - OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { - auto kernel = reinterpret_cast(op_kernel); - std::vector args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - return std::apply([kernel](Args const&... t_args) { Status status = kernel->compute_fn_return_status_(t_args...); return status.release(); }, t); - }; - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { - auto kernel = std::make_unique(); - auto me = static_cast(this_); - kernel->compute_fn_return_status_ = reinterpret_cast(me->compute_fn_return_status_); - Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); - Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); - auto self = static_cast(this_); - kernel->ep_ = self->execution_provider_; - return reinterpret_cast(kernel.release()); - }; - - OrtCustomOp::KernelDestroy = [](void* op_kernel) { - delete reinterpret_cast(op_kernel); - }; - - if (shape_infer_fn_) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp* op, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - auto shape_info_fn = static_cast(op)->shape_infer_fn_; - ShapeInferContext ctx(&GetApi(), ort_ctx); - return shape_info_fn(ctx); - }; - } - } -}; // struct OrtLiteCustomFunc - -/////////////////////////// OrtLiteCustomStruct /////////////////////////// -// The struct is to implement struct-as-op. -// E.g. a struct might be defined as: -// struct Merge { -// Merge(const OrtApi* ort_api, const OrtKernelInfo* info) {...} -// void Compute(const Ort::Custom::Tensor& strings_in, -// std::string_view string_in, -// Ort::Custom::Tensor* strings_out) {...} -// bool reverse_ = false; -// }; -// It could be registered this way: -// Ort::CustomOpDomain v2_domain{"v2"}; -// std::unique_ptr mrg_op_ptr{Ort::Custom::CreateLiteCustomOp("Merge", "CPUExecutionProvider")}; -// v2_domain.Add(mrg_op_ptr.get()); -// session_options.Add(v2_domain); -// For the complete example, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". -template -struct OrtLiteCustomStruct : public OrtLiteCustomOp { - template - using CustomComputeFn = void (CustomOp::*)(Args...); - - template - using CustomComputeFnReturnStatus = Status (CustomOp::*)(Args...); - - using MyType = OrtLiteCustomStruct; - - struct Kernel { - size_t num_input_{}; - size_t num_output_{}; - std::unique_ptr custom_op_; - std::string ep_{}; - }; - - OrtLiteCustomStruct(const char* op_name, - const char* execution_provider, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, {}, start_ver, end_ver) { - SetCompute(&CustomOp::Compute); - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { - auto kernel = std::make_unique(); - Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); - Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); - kernel->custom_op_ = std::make_unique(ort_api, info); - auto self = static_cast(this_); - kernel->ep_ = self->execution_provider_; - return reinterpret_cast(kernel.release()); - }; - - OrtCustomOp::KernelDestroy = [](void* op_kernel) { - delete reinterpret_cast(op_kernel); - }; - - SetShapeInfer(0); - } - - template - void SetCompute(CustomComputeFn) { - ParseArgs(input_types_, output_types_); - OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { - auto kernel = reinterpret_cast(op_kernel); - ArgPtrs args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - std::apply([kernel](Args const&... t_args) { kernel->custom_op_->Compute(t_args...); }, t); - }; - } - - template - void SetCompute(CustomComputeFnReturnStatus) { - ParseArgs(input_types_, output_types_); - OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { - auto kernel = reinterpret_cast(op_kernel); - ArgPtrs args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - return std::apply([kernel](Args const&... t_args) { Status status = kernel->custom_op_->Compute(t_args...); return status.release(); }, t); - }; - } - - template - decltype(&C::InferOutputShape) SetShapeInfer(decltype(&C::InferOutputShape)) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp*, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - ShapeInferContext ctx(&GetApi(), ort_ctx); - return C::InferOutputShape(ctx); - }; - return {}; - } - - template - void SetShapeInfer(...) { - OrtCustomOp::InferOutputShapeFn = {}; - } -}; // struct OrtLiteCustomStruct - -/////////////////////////// CreateLiteCustomOp //////////////////////////// - -template -OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, - const char* execution_provider, - void (*custom_compute_fn)(Args...), - Status (*shape_infer_fn)(ShapeInferContext&) = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) { - using LiteOp = OrtLiteCustomFunc; - return std::make_unique(op_name, execution_provider, custom_compute_fn, shape_infer_fn, start_ver, end_ver).release(); -} - -template -OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, - const char* execution_provider, - Status (*custom_compute_fn_v2)(Args...), - Status (*shape_infer_fn)(ShapeInferContext&) = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) { - using LiteOp = OrtLiteCustomFunc; - return std::make_unique(op_name, execution_provider, custom_compute_fn_v2, shape_infer_fn, start_ver, end_ver).release(); -} - -template -OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, - const char* execution_provider, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) { - using LiteOp = OrtLiteCustomStruct; - return std::make_unique(op_name, execution_provider, start_ver, end_ver).release(); -} - -} // namespace Custom -} // namespace Ort diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_run_options_config_keys.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_run_options_config_keys.h deleted file mode 100644 index c80b8c0..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_run_options_config_keys.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -/* - * This file defines RunOptions Config Keys and format of the Config Values. - * - * The Naming Convention for a RunOptions Config Key, - * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" - * Such as "ep.cuda.use_arena" - * The Config Key cannot be empty - * The maximum length of the Config Key is 128 - * - * The string format of a RunOptions Config Value is defined individually for each Config. - * The maximum length of the Config Value is 1024 - */ - -// Key for enabling shrinkages of user listed device memory arenas. -// Expects a list of semi-colon separated key value pairs separated by colon in the following format: -// "device_0:device_id_0;device_1:device_id_1" -// No white-spaces allowed in the provided list string. -// Currently, the only supported devices are : "cpu", "gpu" (case sensitive). -// If "cpu" is included in the list, DisableCpuMemArena() API must not be called (i.e.) arena for cpu should be enabled. -// Example usage: "cpu:0;gpu:0" (or) "gpu:0" -// By default, the value for this key is empty (i.e.) no memory arenas are shrunk -static const char* const kOrtRunOptionsConfigEnableMemoryArenaShrinkage = "memory.enable_memory_arena_shrinkage"; - -// Set to '1' to not synchronize execution providers with CPU at the end of session run. -// Per default it will be set to '0' -// Taking CUDA EP as an example, it omit triggering cudaStreamSynchronize on the compute stream. -static const char* const kOrtRunOptionsConfigDisableSynchronizeExecutionProviders = "disable_synchronize_execution_providers"; - -// Set HTP performance mode for QNN HTP backend before session run. -// options for HTP performance mode: "burst", "balanced", "default", "high_performance", -// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", -// "sustained_high_performance". Default to "default". -static const char* const kOrtRunOptionsConfigQnnPerfMode = "qnn.htp_perf_mode"; - -// Set HTP performance mode for QNN HTP backend post session run. -static const char* const kOrtRunOptionsConfigQnnPerfModePostRun = "qnn.htp_perf_mode_post_run"; - -// Set RPC control latency for QNN HTP backend -static const char* const kOrtRunOptionsConfigQnnRpcControlLatency = "qnn.rpc_control_latency"; - -// Set graph annotation id for CUDA EP. Use with enable_cuda_graph=true. -// The value should be an integer. If the value is not set, the default value is 0 and -// ORT session only captures one cuda graph before another capture is requested. -// If the value is set to -1, cuda graph capture/replay is disabled in that run. -// User are not expected to set the value to 0 as it is reserved for internal use. -static const char* const kOrtRunOptionsConfigCudaGraphAnnotation = "gpu_graph_id"; diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_session_options_config_keys.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_session_options_config_keys.h deleted file mode 100644 index bb5e034..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_session_options_config_keys.h +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -/* - * This file defines SessionOptions Config Keys and format of the Config Values. - * - * The Naming Convention for a SessionOptions Config Key, - * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" - * Such as "ep.cuda.use_arena" - * The Config Key cannot be empty - * The maximum length of the Config Key is 128 - * - * The string format of a SessionOptions Config Value is defined individually for each Config. - * The maximum length of the Config Value is 1024 - */ - -// Key for disable PrePacking, -// If the config value is set to "1" then the prepacking is disabled, otherwise prepacking is enabled (default value) -static const char* const kOrtSessionOptionsConfigDisablePrepacking = "session.disable_prepacking"; - -// A value of "1" means allocators registered in the env will be used. "0" means the allocators created in the session -// will be used. Use this to override the usage of env allocators on a per session level. -static const char* const kOrtSessionOptionsConfigUseEnvAllocators = "session.use_env_allocators"; - -// Set to 'ORT' (case sensitive) to load an ORT format model. -// If unset, model type will default to ONNX unless inferred from filename ('.ort' == ORT format) or bytes to be ORT -static const char* const kOrtSessionOptionsConfigLoadModelFormat = "session.load_model_format"; - -// Set to 'ORT' (case sensitive) to save optimized model in ORT format when SessionOptions.optimized_model_path is set. -// If unset, format will default to ONNX unless optimized_model_filepath ends in '.ort'. -static const char* const kOrtSessionOptionsConfigSaveModelFormat = "session.save_model_format"; - -// If a value is "1", flush-to-zero and denormal-as-zero are applied. The default is "0". -// When multiple sessions are created, a main thread doesn't override changes from succeeding session options, -// but threads in session thread pools follow option changes. -// When ORT runs with OpenMP, the same rule is applied, i.e. the first session option to flush-to-zero and -// denormal-as-zero is only applied to global OpenMP thread pool, which doesn't support per-session thread pool. -// Note that an alternative way not using this option at runtime is to train and export a model without denormals -// and that's recommended because turning this option on may hurt model accuracy. -static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.set_denormal_as_zero"; - -// It controls to run quantization model in QDQ (QuantizelinearDeQuantizelinear) format or not. -// "0": enable. ORT does fusion logic for QDQ format. -// "1": disable. ORT doesn't do fusion logic for QDQ format. -// Its default value is "0" unless the DirectML execution provider is registered, in which case it defaults to "1". -static const char* const kOrtSessionOptionsDisableQuantQDQ = "session.disable_quant_qdq"; - -// It controls whether to enable Double QDQ remover and Identical Children Consolidation -// "0": not to disable. ORT does remove the middle 2 Nodes from a Q->(QD->Q)->QD pairs -// "1": disable. ORT doesn't remove the middle 2 Nodes from a Q->(QD->Q)->QD pairs -// Its default value is "0" -static const char* const kOrtSessionOptionsDisableDoubleQDQRemover = "session.disable_double_qdq_remover"; - -// If set to "1", enables the removal of QuantizeLinear/DequantizeLinear node pairs once all QDQ handling has been -// completed. e.g. If after all QDQ handling has completed and we have -> FloatOp -> Q -> DQ -> FloatOp -> the -// Q -> DQ could potentially be removed. This will provide a performance benefit by avoiding going from float to -// 8-bit and back to float, but could impact accuracy. The impact on accuracy will be model specific and depend on -// other factors like whether the model was created using Quantization Aware Training or Post Training Quantization. -// As such, it's best to test to determine if enabling this works well for your scenario. -// The default value is "0" -// Available since version 1.11. -static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enable_quant_qdq_cleanup"; - -// Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0". -// GeluApproximation has side effects which may change the inference results. It is disabled by default due to this. -static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation"; - -// This setting controls whether to enable AheadOfTime function inlining. -// AOT function inlining examines the graph and attempts to inline as many locally defined functions in the model -// as possible with the help of enabled execution providers. -// This can reduce the number of function calls and improve performance because it is done before -// Level1 optimizers and constant folding. However, under some circumstances, when the EPs are not available, -// one can disable the AOT inlining, produce an optimized model and postpone AOT until run time. -// "0": enable; "1": disable. -// Its default value is "0". -static const char* const kOrtSessionOptionsDisableAheadOfTimeFunctionInlining = "session.disable_aot_function_inlining"; - -#ifdef ENABLE_TRAINING -// Specifies a list of op types for memory footprint reduction. -// The value should be a ","-delimited list of pair of -// . -// For example, "Gelu+Cast+:1:0,Dropout+:1:1". -// A valid "subgraph string" should be one subgraph representation output by ORT graph transformations. -// "optimization strategy" currently has valid values: 0 - disabled, 1 - recompute. -// "number of subgraph to apply" is used to control how many subgraphs to apply optimization, to avoid "oversaving" -// the memory. -static const char* const kOrtSessionOptionsMemoryOptimizerEnabler = "optimization.memory_optimizer_config"; - -// Specifies the config for detecting subgraphs for memory footprint reduction. -// The value should be a string contains int separated using commas. The default value is "0:0". -static const char* const kOrtSessionOptionsMemoryOptimizerProbeConfig = "optimization.enable_memory_probe_recompute_config"; -#endif - -// This setting if set should contain a comma separated list of optimizers names that should be disabled. -// Optimizers may take time to execute and affect model loading time. If you feel that a specific optimizer -// does not provider runtime benefits, but affects your model loading time you may disable it using this config -// entry. This option is not enabled in ORT_MINIMAL_BUILD build. -// A list of optimizes is available in onnxruntime/core/optimizer/graph_transformer_utils.cc -// -// Default is an empty string which means no optimizers are disabled. -static const char* const kOrtSessionOptionsDisableSpecifiedOptimizers = "optimization.disable_specified_optimizers"; - -// Enable or disable using device allocator for allocating initialized tensor memory. "1": enable; "0": disable. The default is "0". -// Using device allocators means the memory allocation is made using malloc/new. -static const char* const kOrtSessionOptionsUseDeviceAllocatorForInitializers = "session.use_device_allocator_for_initializers"; - -// Configure whether to allow the inter_op/intra_op threads spinning a number of times before blocking -// "0": thread will block if found no job to run -// "1": default, thread will spin a number of times before blocking -static const char* const kOrtSessionOptionsConfigAllowInterOpSpinning = "session.inter_op.allow_spinning"; -static const char* const kOrtSessionOptionsConfigAllowIntraOpSpinning = "session.intra_op.allow_spinning"; - -// Key for using model bytes directly for ORT format -// If a session is created using an input byte array contains the ORT format model data, -// By default we will copy the model bytes at the time of session creation to ensure the model bytes -// buffer is valid. -// Setting this option to "1" will disable copy the model bytes, and use the model bytes directly. The caller -// has to guarantee that the model bytes are valid until the ORT session using the model bytes is destroyed. -static const char* const kOrtSessionOptionsConfigUseORTModelBytesDirectly = "session.use_ort_model_bytes_directly"; - -/// -/// Key for using the ORT format model flatbuffer bytes directly for initializers. -/// This avoids copying the bytes and reduces peak memory usage during model loading and initialization. -/// Requires `session.use_ort_model_bytes_directly` to be true. -/// If set, the flatbuffer bytes provided when creating the InferenceSession MUST remain valid for the entire -/// duration of the InferenceSession. -/// -static const char* const kOrtSessionOptionsConfigUseORTModelBytesForInitializers = - "session.use_ort_model_bytes_for_initializers"; - -// This should only be specified when exporting an ORT format model for use on a different platform. -// If the ORT format model will be used on ARM platforms set to "1". For other platforms set to "0" -// Available since version 1.11. -static const char* const kOrtSessionOptionsQDQIsInt8Allowed = "session.qdqisint8allowed"; - -// x64 SSE4.1/AVX2/AVX512(with no VNNI) has overflow problem with quantizied matrix multiplication with U8S8. -// To avoid this we need to use slower U8U8 matrix multiplication instead. This option, if -// turned on, use slower U8U8 matrix multiplications. Only effective with AVX2 or AVX512 -// platforms. -static const char* const kOrtSessionOptionsAvx2PrecisionMode = "session.x64quantprecision"; - -// Specifies how minimal build graph optimizations are handled in a full build. -// These optimizations are at the extended level or higher. -// Possible values and their effects are: -// "save": Save runtime optimizations when saving an ORT format model. -// "apply": Only apply optimizations available in a minimal build. -// ""/: Apply optimizations available in a full build. -// Available since version 1.11. -static const char* const kOrtSessionOptionsConfigMinimalBuildOptimizations = - "optimization.minimal_build_optimizations"; - -// Note: The options specific to an EP should be specified prior to appending that EP to the session options object in -// order for them to take effect. - -// Specifies a list of stop op types. Nodes of a type in the stop op types and nodes downstream from them will not be -// run by the NNAPI EP. -// The value should be a ","-delimited list of op types. For example, "Add,Sub". -// If not specified, the default set of stop ops is used. To specify an empty stop ops types list and disable stop op -// exclusion, set the value to "". -static const char* const kOrtSessionOptionsConfigNnapiEpPartitioningStopOps = "ep.nnapi.partitioning_stop_ops"; - -// Enabling dynamic block-sizing for multithreading. -// With a positive value, thread pool will split a task of N iterations to blocks of size starting from: -// N / (num_of_threads * dynamic_block_base) -// As execution progresses, the size will decrease according to the diminishing residual of N, -// meaning the task will be distributed in smaller granularity for better parallelism. -// For some models, it helps to reduce the variance of E2E inference latency and boost performance. -// The feature will not function by default, specify any positive integer, e.g. "4", to enable it. -// Available since version 1.11. -static const char* const kOrtSessionOptionsConfigDynamicBlockBase = "session.dynamic_block_base"; - -// This option allows to decrease CPU usage between infrequent -// requests and forces any TP threads spinning stop immediately when the last of -// concurrent Run() call returns. -// Spinning is restarted on the next Run() call. -// Applies only to internal thread-pools -static const char* const kOrtSessionOptionsConfigForceSpinningStop = "session.force_spinning_stop"; - -// "1": all inconsistencies encountered during shape and type inference -// will result in failures. -// "0": in some cases warnings will be logged but processing will continue. The default. -// May be useful to expose bugs in models. -static const char* const kOrtSessionOptionsConfigStrictShapeTypeInference = "session.strict_shape_type_inference"; - -// "1": every model using a more recent opset than the latest released one will fail -// "0": the model may or may not work if onnxruntime cannot find an implementation, this option -// is used for development purpose. -static const char* const kOrtSessionOptionsConfigStrictAllowReleasedOpsetsOnly = "session.allow_released_opsets_only"; - -// The file saves configuration for partitioning node among logic streams -static const char* const kNodePartitionConfigFile = "session.node_partition_config_file"; - -// This Option allows setting affinities for intra op threads. -// Affinity string follows format: -// logical_processor_id,logical_processor_id;logical_processor_id,logical_processor_id -// Semicolon isolates configurations among threads, while comma split processors where ith thread expected to attach to. -// e.g.1,2,3;4,5 -// specifies affinities for two threads, with the 1st thread attach to the 1st, 2nd, and 3rd processor, and 2nd thread to the 4th and 5th. -// To ease the configuration, an "interval" is also allowed: -// e.g. 1-8;8-16;17-24 -// orders that the 1st thread runs on first eight processors, 2nd thread runs on next eight processors, and so forth. -// Note: -// 1. Once set, the number of thread affinities must equal to intra_op_num_threads - 1, since ort does not set affinity on the main thread which -// is started and managed by the calling app; -// 2. For windows, ort will infer the group id from a logical processor id, for example, assuming there are two groups with each has 64 logical processors, -// an id of 64 will be inferred as the last processor of the 1st group, while 65 will be interpreted as the 1st processor of the second group. -// Hence 64-65 is an invalid configuration, because a windows thread cannot be attached to processors across group boundary. -static const char* const kOrtSessionOptionsConfigIntraOpThreadAffinities = "session.intra_op_thread_affinities"; - -// This option will dump out the model to assist debugging any issues with layout transformation, -// and is primarily intended for developer usage. It is only relevant if an execution provider that requests -// NHWC layout is enabled such as NNAPI, XNNPACK or QNN. -// -// Default is off. Set to "1" to enable. -// -// If modified by layout transformation the model will be dumped after these steps: -// 1) insertion of the layout transformation Transpose nodes -// 2) after those are optimized using the transpose optimizer, -// 3) after the L1 transformers are applied to the updated graph. -// The model will be saved to filename post_layout_transform_step_.onnx. -static const char* const kDebugLayoutTransformation = "session.debug_layout_transformation"; - -// Graph nodes that are not supported by the execution providers (EPs) explicitly added to the session are -// assigned (i.e., "fallback") to the CPU EP by default. -// -// This option allows the user to disable the fallback of unsupported graph nodes to the CPU EP. -// If this option is set to "1", session creation will fail if the execution providers other than the CPU EP cannot -// fully support all of the nodes in the graph. -// -// It is invalid to set this option and explicitly add the CPU EP to the session. In this case, session creation -// will also fail with an error. -// -// Option values: -// - "0": CPU EP fallback is not disabled. [DEFAULT] -// - "1": CPU EP fallback is disabled. -static const char* const kOrtSessionOptionsDisableCPUEPFallback = "session.disable_cpu_ep_fallback"; - -// Use this config when serializing a large model after optimization to specify an external initializers file -static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersFileName = - "session.optimized_model_external_initializers_file_name"; - -// Use this config to control the minimum size of the initializer when externalizing it during serialization -static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersMinSizeInBytes = - "session.optimized_model_external_initializers_min_size_in_bytes"; - -// Enable EP context feature to dump the partitioned graph which includes the EP context into Onnx file. -// The dumped Onnx model with EP context can be used for future inference to avoid the EP graph partitioning/compile overhead. -// "0": disable. (default) -// "1": enable. -static const char* const kOrtSessionOptionEpContextEnable = "ep.context_enable"; - -// Specify the file path for the Onnx model which has EP context. -// Default to original_file_name_ctx.onnx if not specified -static const char* const kOrtSessionOptionEpContextFilePath = "ep.context_file_path"; - -// Flag to specify whether to dump the EP context into the Onnx model. -// "0": dump the EP context into separate file, keep the file name in the Onnx model. -// "1": dump the EP context into the Onnx model. (default). -static const char* const kOrtSessionOptionEpContextEmbedMode = "ep.context_embed_mode"; - -// Gemm fastmath mode provides fp32 gemm acceleration with bfloat16 based matmul. -// Option values: -// - "0": Gemm FastMath mode is not enabled. [DEFAULT] -// - "1": Gemm FastMath mode is enabled. -static const char* const kOrtSessionOptionsMlasGemmFastMathArm64Bfloat16 = "mlas.enable_gemm_fastmath_arm64_bfloat16"; diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_c_api.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_c_api.h deleted file mode 100644 index ed6d151..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_c_api.h +++ /dev/null @@ -1,731 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// This file contains the training c apis. - -#pragma once -#include -#include "onnxruntime_c_api.h" - -/** \page training_c_cpp_api Training C & C++ APIs - * - * Training C and C++ APIs are an extension of the \ref c_cpp_api "onnxruntime core C and C++ APIs" and should be used in conjunction with them. - * - * In order to train a model with onnxruntime, the following training artifacts must be generated: - * - The training onnx model - * - The checkpoint file - * - The optimizer onnx model - * - The eval onnx model model (optional) - * - * These training artifacts can be generated as part of an offline step using the python [utilities](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md) made available in the `onnxruntime-training` python package. - * - * After these artifacts have been generated, the C and C++ utilities listed in this documentation can be leveraged to perform training. - * - * If any problem is encountered, please create an [issue](https://github.com/microsoft/onnxruntime/issues/new) with your scenario and requirements, and we will be sure to respond and follow up on the request. - * - *

Training C API

- * - * ::OrtTrainingApi - Training C API functions. - * - * This C structure contains functions that enable users to perform training with onnxruntime. - * - * _Sample Code_: - * - * ```c - * #include - * - * OrtApi* g_ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION); - * OrtTrainingApi* g_ort_training_api = g_ort_api->GetTrainingApi(ORT_API_VERSION); - * - * OrtEnv* env = NULL; - * g_ort_api->CreateEnv(logging_level, logid, &env); - * OrtSessionOptions* session_options = NULL; - * g_ort_api->CreateSessionOptions(&session_options); - * - * OrtCheckpointState* state = NULL; - * g_ort_training_api->LoadCheckpoint(path_to_checkpoint, &state); - * - * OrtTrainingSession* training_session = NULL; - * g_ort_training_api->CreateTrainingSession(env, session_options, training_model_path, - * state, eval_model_path, optimizer_model_path, - * &training_session); - * // Training loop - * { - * g_ort_training_api->TrainStep(...); - * g_ort_training_api->OptimizerStep(...); - * g_ort_training_api->LazyResetGrad(...); - * } - * - * g_ort_training_api->ExportModelForInferencing(training_session, inference_model_path, ...); - * g_ort_training_api->SaveCheckpoint(state, path_to_checkpoint, false); - * - * g_ort_training_api->ReleaseTrainingSession(training_session); - * g_ort_training_api->ReleaseCheckpointState(state); - * ``` - * - * > **Note** - * > The ::OrtCheckpointState contains the entire training state that the ::OrtTrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::OrtCheckpointState instance must outlive the lifetime of the ::OrtTrainingSession instance. - * - *

Training C++ API

- * - * @ref TrainingCpp - Training C++ API classes and functions. - * - * These C++ classes and functions enable users to perform training with onnxruntime. - * - * _Sample Code_: - * - * ```cc - * #include - * - * Ort::Env env; - * Ort::SessionOptions session_options; - * - * auto state = Ort::CheckpointState::LoadCheckpoint(path_to_checkpoint); - * auto training_session = Ort::TrainingSession(env, session_options, state, training_model_path, - * eval_model_path, optimizer_model_path); - * - * // Training Loop - * { - * training_session.TrainStep(...); - * training_session.OptimizerStep(...); - * training_session.LazyResetGrad(...); - * } - * - * training_session->ExportModelForInferencing(inference_model_path, ...); - * Ort::CheckpointState::SaveCheckpoint(state, path_to_checkpoint, false); - * ``` - * > **Note** - * > The ::Ort::CheckpointState contains the entire training state that the ::Ort::TrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::Ort::CheckpointState instance must outlive the lifetime of the ::Ort::TrainingSession instance. - */ - -/** @defgroup TrainingC Ort Training C API - * @{ - */ -ORT_RUNTIME_CLASS(TrainingSession); // Type that enables performing training for the given user models. -ORT_RUNTIME_CLASS(CheckpointState); // Type that holds the training states for the training session. - -/** \brief Type of property to be added to or returned from the ::OrtCheckpointState. - */ -typedef enum OrtPropertyType { - OrtIntProperty = 0, - OrtFloatProperty = 1, - OrtStringProperty = 2, -} OrtPropertyType; - -/** \brief The Training C API that holds onnxruntime training function pointers - * - * All the Training C API functions are defined inside this structure as pointers to functions. - * Call OrtApi::GetTrainingApi to get a pointer to this struct. - * - * \nosubgrouping - */ -struct OrtTrainingApi { - /// \name Accessing The Training Session State - /// @{ - - /** \brief Load a checkpoint state from a file on disk into checkpoint_state. - * - * This function will parse a checkpoint file, pull relevant data and load the training - * state into the checkpoint_state. This checkpoint state can then be used to create the - * training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training - * session will resume training from the given checkpoint state. - * \note Note that the training session created with a checkpoint state uses this state to store the entire - * training state (including model parameters, its gradients, the optimizer states and the properties). - * As a result, it is required that the checkpoint state outlive the lifetime of the training session. - * \note Note that the checkpoint file can be either the complete checkpoint or the nominal checkpoint. - * - * \param[in] checkpoint_path Path to the checkpoint file - * \param[out] checkpoint_state Checkpoint state that contains the states of the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(LoadCheckpoint, _In_ const ORTCHAR_T* checkpoint_path, - _Outptr_ OrtCheckpointState** checkpoint_state); - - /** \brief Save the given state to a checkpoint file on disk. - * - * This function serializes the provided checkpoint state to a file on disk. - * This checkpoint can later be loaded by invoking OrtTrainingApi::LoadCheckpoint to resume - * training from this snapshot of the state. - * - * \param[in] checkpoint_state The checkpoint state to save. - * \param[in] checkpoint_path Path to the checkpoint file. - * \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SaveCheckpoint, _In_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* checkpoint_path, - const bool include_optimizer_state); - - /// @} - - /// \name Implementing The Training Loop - /// @{ - /** \brief Create a training session that can be used to begin or resume training. - * - * This function creates a training session based on the env and session options provided that can - * begin or resume training from a given checkpoint state for the given onnx models. - * The checkpoint state represents the parameters of the training session which will be moved - * to the device specified by the user through the session options (if necessary). - * The training session requires four training artifacts - * - The training onnx model - * - The evaluation onnx model (optional) - * - The optimizer onnx model - * - The checkpoint file - * - * These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md). - * - * \param[in] env Environment to be used for the training session. - * \param[in] options Session options that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_path Model to be used to perform training. - * \param[in] eval_model_path Model to be used to perform evaluation. - * \param[in] optimizer_model_path Model to be used to perform gradient descent. - * \param[out] out Created training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(CreateTrainingSession, _In_ const OrtEnv* env, _In_ const OrtSessionOptions* options, - _Inout_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* train_model_path, - _In_ const ORTCHAR_T* eval_model_path, _In_ const ORTCHAR_T* optimizer_model_path, - _Outptr_result_maybenull_ OrtTrainingSession** out); - - /** \brief Create a training session that can be used to begin or resume training. - * This api provides a way to load all the training artifacts from buffers instead of files. - * - * \param[in] env Environment to be used for the training session. - * \param[in] options Session options that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_data Buffer containing the model data to be used to perform training - * \param[in] train_data_length Length of the buffer containing train_model_data - * \param[in] eval_model_data Buffer containing the model data to be used to perform evaluation - * \param[in] eval_data_length Length of the buffer containing eval_model_data - * \param[in] optim_model_data Buffer containing the model data to be used to perform weight update - * \param[in] optim_data_length Length of the buffer containing optim_model_data - * \param[out] out Created training session. - * - */ - ORT_API2_STATUS(CreateTrainingSessionFromBuffer, _In_ const OrtEnv* env, - _In_ const OrtSessionOptions* options, _Inout_ OrtCheckpointState* checkpoint_state, - _In_ const void* train_model_data, size_t train_data_length, - _In_ const void* eval_model_data, size_t eval_data_length, - _In_ const void* optim_model_data, size_t optim_data_length, - _Outptr_result_maybenull_ OrtTrainingSession** out); - - /// @} - - /// \name Model IO Information - /// @{ - - /** \brief Retrieves the number of user outputs in the training model. - * - * This function returns the number of outputs of the training model so that the user can - * allocate space for the number of outputs when OrtTrainingApi::TrainStep is invoked. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user outputs in the training model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the number of user outputs in the eval model. - * - * This function returns the number of outputs of the eval model so that the user can - * allocate space for the number of outputs when OrtTrainingApi::EvalStep is invoked. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user outputs in the eval model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the names of user outputs in the training model. - * - * This function returns the names of outputs of the training model that can be associated with the OrtValue(s) - * returned by the OrtTrainingApi::TrainStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index Index of the output name requested. - * \param[in] allocator Allocator to use to allocate the memory for the name. - * \param[out] output Name of the training model output at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output); - - /** \brief Retrieves the names of user outputs in the eval model. - * - * This function returns the names of outputs of the eval model that can be associated with the OrtValue(s) returned - * by the OrtTrainingApi::EvalStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index Index of the output name requested. - * \param[in] allocator Allocator to use to allocate the memory for the name. - * \param[out] output Name of the eval model output at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output); - - /// @} - - /// \name Implementing The Training Loop - /// @{ - - /** \brief Reset the gradients of all trainable parameters to zero lazily. - * - * This function sets the internal state of the training session such that the gradients of the trainable - * parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are - * computed on the next invocation of the next OrtTrainingApi::TrainStep. - * - * \param[in] session The `this` pointer to the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(LazyResetGrad, _Inout_ OrtTrainingSession* session); - - /** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs - * - * This function performs a training step that computes the outputs of the training model and the gradients - * of the trainable parameters for the given inputs. The train step is performed based on the training model - * that was provided to the training session. - * The OrtTrainingApi::TrainStep is equivalent of running forward propagation and backward propagation in a single - * step. - * The gradients computed are stored inside the training session state so they can be later consumed - * by the OrtTrainingApi::OptimizerStep function. - * The gradients can be lazily reset by invoking the OrtTrainingApi::LazyResetGrad function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] run_options Run options for this training step. - * \param[in] inputs_len Number of user inputs to the training model. - * \param[in] inputs The user inputs to the training model. - * \param[in] outputs_len Number of user outputs expected from this training step. - * \param[out] outputs User outputs computed by train step. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainStep, _Inout_ OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options, - _In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs, - _In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs); - - /** \brief Computes the outputs for the eval model for the given inputs - * - * This function performs an eval step that computes the outputs of the eval model for the given inputs. - * The eval step is performed based on the eval model that was provided to the training session. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] run_options Run options for this eval step. - * \param[in] inputs_len Number of user inputs to the eval model. - * \param[in] inputs The user inputs to the eval model. - * \param[in] outputs_len Number of user outputs expected from this eval step. - * \param[out] outputs User outputs computed by eval step. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(EvalStep, _In_ const OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options, - _In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs, - _In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs); - - /** \brief Sets the learning rate for this training session. - * - * This function allows users to set the learning rate for the training session. The current - * learning rate is maintained by the training session and can be overwritten by invoking - * this function with the desired learning rate. This function should not be used when a valid - * learning rate scheduler is registered. It should be used either to set the learning rate - * derived from a custom learning rate scheduler or to set a constant learning rate to be used - * throughout the training session. - * \note Please note that this function does not set the initial learning rate that may be needed - * by the predefined learning rate schedulers. To set the initial learning rate for learning - * rate schedulers, please look at the function OrtTrainingApi::RegisterLinearLRScheduler. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] learning_rate Desired learning rate to be set. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SetLearningRate, _Inout_ OrtTrainingSession* sess, _In_ float learning_rate); - - /** \brief Gets the current learning rate for this training session. - * - * This function allows users to get the learning rate for the training session. The current - * learning rate is maintained by the training session, and users can query it for the purpose - * of implementing their own learning rate schedulers. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] learning_rate Learning rate currently in use by the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetLearningRate, _Inout_ OrtTrainingSession* sess, _Out_ float* learning_rate); - - /** \brief Performs the weight updates for the trainable parameters using the optimizer model. - * - * This function performs the weight update step that updates the trainable parameters such that they - * take a step in the direction of their gradients (gradient descent). The optimizer step is performed - * based on the optimizer model that was provided to the training session. - * The updated parameters are stored inside the training state so that they can be used by the next - * OrtTrainingApi::TrainStep function call. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] run_options Run options for this optimizer step. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(OptimizerStep, _Inout_ OrtTrainingSession* sess, - _In_opt_ const OrtRunOptions* run_options); - - /** \brief Registers a linear learning rate scheduler for the training session. - * - * Register a linear learning rate scheduler that decays the learning rate by linearly updated - * multiplicative factor from the initial learning rate set on the training session to 0. The decay - * is performed after the initial warm up phase where the learning rate is linearly incremented - * from 0 to the initial learning rate provided. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] warmup_step_count Warmup steps for LR warmup. - * \param[in] total_step_count Total step count. - * \param[in] initial_lr The initial learning rate to be used by the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(RegisterLinearLRScheduler, _Inout_ OrtTrainingSession* sess, _In_ const int64_t warmup_step_count, - _In_ const int64_t total_step_count, _In_ const float initial_lr); - - /** \brief Update the learning rate based on the registered learing rate scheduler. - * - * Takes a scheduler step that updates the learning rate that is being used by the training session. - * This function should typically be called before invoking the optimizer step for each round, - * or as determined necessary to update the learning rate being used by the training session. - * \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this - * function. - * - * \param[in] sess The `this` pointer to the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SchedulerStep, _Inout_ OrtTrainingSession* sess); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - /** \brief Retrieves the size of all the parameters. - * - * Calculates the total number of primitive (datatype of the parameters) elements of all the parameters in the - * training state. - * When trainable_only argument is true, the size is calculated for trainable params only. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Size of all parameter elements. - * \param[in] trainable_only Whether to skip non-trainable parameters - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetParametersSize, _Inout_ OrtTrainingSession* sess, _Out_ size_t* out, bool trainable_only); - - /** \brief Copy all parameters to a contiguous buffer held by the argument parameters_buffer - * - * The parameters_buffer has to be of the size given by GetParametersSize api call, - * with matching setting for the argument trainable_only. All the target parameters must be of the same - * datatype. The OrtValue must be pre-allocated onto - * the desired device. This is a complementary function to OrtTrainingApi::CopyBufferToParameters. - * Parameter ordering is preserved. - * User is responsible for allocating and freeing the resources used by the parameters_buffer. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] trainable_only Whether to skip non-trainable parameters - * \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy onto. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(CopyParametersToBuffer, _Inout_ OrtTrainingSession* sess, - _Inout_ OrtValue* parameters_buffer, bool trainable_only); - - /** \brief Copy parameter values from the given contiguous buffer held by parameters_buffer to the training state - * - * The parameters_buffer argument has to be of the size given by OrtTrainingApi::GetParametersSize api call, - * with matching setting for trainable_only argument. All the target parameters must be of the same - * datatype. This is a complementary function to OrtTrainingApi::CopyParametersToBuffer - * and can be used to load updated buffer values onto the training state. - * Parameter ordering is preserved. - * User is responsible for allocating and freeing the resources used by the parameters_buffer. - * In case the training session was created with a nominal checkpoint, invoking this function is required - * to load the updated parameters onto the checkpoint to complete it. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] trainable_only Whether to skip non-trainable parameters - * \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy from. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(CopyBufferToParameters, _Inout_ OrtTrainingSession* sess, - _Inout_ OrtValue* parameters_buffer, bool trainable_only); - - /// @} - - /// \name Release Training Resources - /// @{ - - /** \brief Frees up the memory used up by the training session. - * - * This function frees up any memory that was allocated in the training session. The training - * session can no longer be used after this call. - * - */ - ORT_CLASS_RELEASE(TrainingSession); - - /** \brief Frees up the memory used up by the checkpoint state. - * - * This function frees up any memory that was allocated in the checkpoint state. The checkpoint - * state can no longer be used after this call. - * \note Note that the checkpoint state must be released only after the training session has been released. - * - */ - ORT_CLASS_RELEASE(CheckpointState); - - /// @} - - /// \name Prepare For Inferencing - /// @{ - /** \brief Export a model that can be used for inferencing. - * - * If the training session was provided with an eval model, the training session can generate - * an inference model if it knows the inference graph outputs. The input inference graph outputs - * are used to prune the eval model so that the inference model's outputs align with the provided outputs. - * The exported model is saved at the path provided and can be used for inferencing with InferenceSession. - * \note Note that the function re-loads the eval model from the path provided to OrtTrainingApi::CreateTrainingSession - * and expects that this path still be valid. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] inference_model_path Path where the inference model should be serialized to. - * \param[in] graph_outputs_len Size of the graph output names array. - * \param[in] graph_output_names Names of the outputs that are needed in the inference model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(ExportModelForInferencing, _Inout_ OrtTrainingSession* sess, - _In_ const ORTCHAR_T* inference_model_path, size_t graph_outputs_len, - _In_reads_(graph_outputs_len) const char* const* graph_output_names); - - /// @} - - /// \name Training Utilities - /// @{ - /** \brief Sets the seed used for random number generation in Onnxruntime. - * - * Use this function to generate reproducible results. It should be noted that completely reproducible - * results are not guaranteed. - * - * \param[in] seed The seed to be set. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SetSeed, _In_ const int64_t seed); - - /// @} - - /// \name Model IO Information - /// @{ - /** \brief Retrieves the number of user inputs in the training model. - * - * This function returns the number of inputs of the training model so that the user can accordingly - * allocate the OrtValue(s) provided to the OrtTrainingApi::TrainStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user inputs in the training model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the number of user inputs in the eval model. - * - * This function returns the number of inputs of the eval model so that the user can accordingly - * allocate the OrtValue(s) provided to the OrtTrainingApi::EvalStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user inputs in the eval model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the name of the user input at given index in the training model. - * - * This function returns the names of inputs of the training model that can be associated with the - * OrtValue(s) provided to the OrtTrainingApi::TrainStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index The index of the training model input name requested. - * \param[in] allocator The allocator to use to allocate the memory for the requested name. - * \param[out] output Name of the user input for the training model at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelInputName, _In_ const OrtTrainingSession* sess, size_t index, - _In_ OrtAllocator* allocator, _Outptr_ char** output); - - /** \brief Retrieves the name of the user input at given index in the eval model. - * - * This function returns the names of inputs of the eval model that can be associated with the OrtValue(s) provided - * to the OrtTrainingApi::EvalStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index The index of the eval model input name requested. - * \param[in] allocator The allocator to use to allocate the memory for the requested name. - * \param[out] output Name of the user input for the eval model at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelInputName, _In_ const OrtTrainingSession* sess, size_t index, - _In_ OrtAllocator* allocator, _Outptr_ char** output); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Adds or updates the given property to/in the checkpoint state. - * - * Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint - * state by the user by calling this function with the corresponding property name and value. - * The given property name must be unique to be able to successfully add the property. - * - * \param[in] checkpoint_state The checkpoint state which should hold the property. - * \param[in] property_name Name of the property being added or updated. - * \param[in] property_type Type of the property associated with the given name. - * \param[in] property_value Property value associated with the given name. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(AddProperty, _Inout_ OrtCheckpointState* checkpoint_state, - _In_ const char* property_name, _In_ enum OrtPropertyType property_type, - _In_ void* property_value); - - /** \brief Gets the property value associated with the given name from the checkpoint state. - * - * Gets the property value from an existing entry in the checkpoint state. The property must - * exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] checkpoint_state The checkpoint state that is currently holding the property. - * \param[in] property_name Name of the property being retrieved. - * \param[in] allocator Allocator used to allocate the memory for the property_value. - * \param[out] property_type Type of the property associated with the given name. - * \param[out] property_value Property value associated with the given name. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetProperty, _In_ const OrtCheckpointState* checkpoint_state, - _In_ const char* property_name, _Inout_ OrtAllocator* allocator, - _Out_ enum OrtPropertyType* property_type, _Outptr_ void** property_value); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Load a checkpoint state from a buffer into checkpoint_state. - * - * This function will parse a checkpoint bytes buffer, pull relevant data and load the training - * state into the checkpoint_state. This checkpoint state can then be used to create the - * training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training - * session will resume training from the given checkpoint state. - * \note Note that the training session created with a checkpoint state uses this state to store the entire - * training state (including model parameters, its gradients, the optimizer states and the properties). - * As a result, it is required that the checkpoint state outlive the lifetime of the training session. - * - * \param[in] checkpoint_buffer Path to the checkpoint bytes buffer. - * \param[in] num_bytes Number of bytes in the checkpoint buffer. - * \param[out] checkpoint_state Checkpoint state that contains the states of the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(LoadCheckpointFromBuffer, _In_ const void* checkpoint_buffer, - _In_ const size_t num_bytes, _Outptr_ OrtCheckpointState** checkpoint_state); - - /** \brief Retrieves the type and shape information of the parameter associated with the given parameter name. - * - * This function retrieves the type and shape of the parameter associated with the given parameter name. - * The parameter must exist in the checkpoint state to be able to retrieve its type and shape information successfully. - * - * \param[in] checkpoint_state The checkpoint state. - * \param[in] parameter_name Name of the parameter being retrieved. - * \param[out] parameter_type_and_shape The type and shape of the parameter being retrieved. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetParameterTypeAndShape, _In_ const OrtCheckpointState* checkpoint_state, - _In_ const char* parameter_name, _Outptr_ OrtTensorTypeAndShapeInfo** parameter_type_and_shape); - - /** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name. - * - * This function updates a model parameter in the checkpoint state with the given parameter data. - * The training session must be already created with the checkpoint state that contains the parameter - * being updated. The given parameter is copied over to the registered device for the training session. - * The parameter must exist in the checkpoint state to be able to update it successfully. - * - * \param[in] checkpoint_state The checkpoint state. - * \param[in] parameter_name Name of the parameter being updated. - * \param[in] parameter The parameter data that should replace the existing parameter data. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(UpdateParameter, _Inout_ OrtCheckpointState* checkpoint_state, - _In_ const char* parameter_name, _In_ OrtValue* parameter); - - /** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name. - * - * This function retrieves the model parameter data from the checkpoint state for the given parameter name. - * The parameter is copied over and returned as an OrtValue. The training session must be already created - * with the checkpoint state that contains the parameter being retrieved. - * The parameter must exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] checkpoint_state The checkpoint state. - * \param[in] parameter_name Name of the parameter being retrieved. - * \param[in] allocator Allocator used to allocate the memory for the parameter. - * \param[out] parameter The parameter data that is retrieved from the checkpoint state. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetParameter, _In_ const OrtCheckpointState* checkpoint_state, - _In_ const char* parameter_name, _Inout_ OrtAllocator* allocator, - _Outptr_ OrtValue** parameter); - - /// @} -}; - -typedef struct OrtTrainingApi OrtTrainingApi; - -/// @} diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_cxx_api.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_cxx_api.h deleted file mode 100644 index e78c161..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_cxx_api.h +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "onnxruntime_training_c_api.h" -#include -#include - -namespace Ort::detail { - -#define ORT_DECLARE_TRAINING_RELEASE(NAME) \ - void OrtRelease(Ort##NAME* ptr); - -// These release methods must be forward declared before including onnxruntime_cxx_api.h -// otherwise class Base won't be aware of them -ORT_DECLARE_TRAINING_RELEASE(CheckpointState); -ORT_DECLARE_TRAINING_RELEASE(TrainingSession); - -} // namespace Ort::detail - -#include "onnxruntime_cxx_api.h" - -namespace Ort { - -/// -/// This function returns the C training api struct with the pointers to the ort training C functions. -/// If using C++, please use the class instances instead of invoking the C functions directly. -/// -/// OrtTrainingApi struct with ort training C function pointers. -inline const OrtTrainingApi& GetTrainingApi() { return *GetApi().GetTrainingApi(ORT_API_VERSION); } - -namespace detail { - -#define ORT_DEFINE_TRAINING_RELEASE(NAME) \ - inline void OrtRelease(Ort##NAME* ptr) { GetTrainingApi().Release##NAME(ptr); } - -ORT_DEFINE_TRAINING_RELEASE(CheckpointState); -ORT_DEFINE_TRAINING_RELEASE(TrainingSession); - -#undef ORT_DECLARE_TRAINING_RELEASE -#undef ORT_DEFINE_TRAINING_RELEASE - -} // namespace detail - -using Property = std::variant; - -/** - * \defgroup TrainingCpp Ort Training C++ API - * @{ - */ - -/** \brief Holds the state of the training session. - * - * This class holds the entire training session state that includes model parameters, their gradients, - * optimizer parameters, and user properties. The Ort::TrainingSession leverages the Ort::CheckpointState - * by accessing and updating the contained training state. - * \note Note that the training session created with a checkpoint state uses this state to store the entire - * training state (including model parameters, its gradients, the optimizer states and the properties). - * The Ort::TrainingSession does not hold a copy of the Ort::CheckpointState and as a result, it is required - * that the checkpoint state outlive the lifetime of the training session. - * \note Note that the checkpoint state can be either the complete checkpoint state or the nominal checkpoint - * state depending on the version provided while loading the checkpoint. - * - */ -class CheckpointState : public detail::Base { - private: - CheckpointState(OrtCheckpointState* checkpoint_state) { p_ = checkpoint_state; } - - public: - // Construct the checkpoint state by loading the checkpoint by calling LoadCheckpoint - CheckpointState() = delete; - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Load a checkpoint state from a file on disk into checkpoint_state. - * - * This function will parse a checkpoint file, pull relevant data and load the training - * state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the - * training session by instantiating Ort::TrainingSession. By doing so, the training session will resume - * training from the given checkpoint state. - * - * \param[in] path_to_checkpoint Path to the checkpoint file - * \return Ort::CheckpointState object which holds the state of the training session parameters. - * - */ - static CheckpointState LoadCheckpoint(const std::basic_string& path_to_checkpoint); - - /** \brief Load a checkpoint state from a buffer. - * - * This function will parse a checkpoint buffer, pull relevant data and load the training - * state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the - * training session by instantiating Ort::TrainingSession. By doing so, the training session will resume - * training from the given checkpoint state. - * - * \param[in] buffer Buffer containing the checkpoint data. - * \return Ort::CheckpointState object which holds the state of the training session parameters. - * - */ - static CheckpointState LoadCheckpointFromBuffer(const std::vector& buffer); - - /** \brief Save the given state to a checkpoint file on disk. - * - * This function serializes the provided checkpoint state to a file on disk. - * This checkpoint can later be loaded by invoking Ort::CheckpointState::LoadCheckpoint to resume - * training from this snapshot of the state. - * - * \param[in] checkpoint_state The checkpoint state to save. - * \param[in] path_to_checkpoint Path to the checkpoint file. - * \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not. - * - */ - static void SaveCheckpoint(const CheckpointState& checkpoint_state, - const std::basic_string& path_to_checkpoint, - const bool include_optimizer_state = false); - - /** \brief Adds or updates the given property to/in the checkpoint state. - * - * Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint - * state by the user by calling this function with the corresponding property name and value. - * The given property name must be unique to be able to successfully add the property. - * - * \param[in] property_name Name of the property being added or updated. - * \param[in] property_value Property value associated with the given name. - * - */ - void AddProperty(const std::string& property_name, const Property& property_value); - - /** \brief Gets the property value associated with the given name from the checkpoint state. - * - * Gets the property value from an existing entry in the checkpoint state. The property must - * exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] property_name Name of the property being retrieved. - * \return Property value associated with the given property name. - * - */ - Property GetProperty(const std::string& property_name); - - /** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name. - * - * This function updates a model parameter in the checkpoint state with the given parameter data. - * The training session must be already created with the checkpoint state that contains the parameter - * being updated. The given parameter is copied over to the registered device for the training session. - * The parameter must exist in the checkpoint state to be able to update it successfully. - * - * \param[in] parameter_name Name of the parameter being updated. - * \param[in] parameter The parameter data that should replace the existing parameter data. - * - */ - void UpdateParameter(const std::string& parameter_name, const Value& parameter); - - /** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name. - * - * This function retrieves the model parameter data from the checkpoint state for the given parameter name. - * The parameter is copied over to the provided OrtValue. The training session must be already created - * with the checkpoint state that contains the parameter being retrieved. - * The parameter must exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] parameter_name Name of the parameter being retrieved. - * \return The parameter data that is retrieved from the checkpoint state. - * - */ - Value GetParameter(const std::string& parameter_name); - - /// @} -}; - -/** \brief Trainer class that provides training, evaluation and optimizer methods for training an ONNX models. - * - * The training session requires four training artifacts - * - The training onnx model - * - The evaluation onnx model (optional) - * - The optimizer onnx model - * - The checkpoint file - * - * These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md). - * - */ -class TrainingSession : public detail::Base { - private: - size_t training_model_output_count_, eval_model_output_count_; - - public: - /// \name Constructing the Training Session - /// @{ - /** \brief Create a training session that can be used to begin or resume training. - * - * This constructor instantiates the training session based on the env and session options provided that can - * begin or resume training from a given checkpoint state for the given onnx models. - * The checkpoint state represents the parameters of the training session which will be moved - * to the device specified by the user through the session options (if necessary). - * - * \param[in] env Env to be used for the training session. - * \param[in] session_options SessionOptions that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_path Model to be used to perform training. - * \param[in] eval_model_path Model to be used to perform evaluation. - * \param[in] optimizer_model_path Model to be used to perform gradient descent. - * - */ - TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state, - const std::basic_string& train_model_path, - const std::optional>& eval_model_path = std::nullopt, - const std::optional>& optimizer_model_path = std::nullopt); - - /** \brief Create a training session that can be used to begin or resume training. - * This constructor allows the users to load the models from buffers instead of files. - * - * \param[in] env Env to be used for the training session. - * \param[in] session_options SessionOptions that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_data Buffer containing training model data. - * \param[in] eval_model_data Buffer containing evaluation model data. - * \param[in] optim_model_data Buffer containing optimizer model (used for performing weight/parameter update). - * - */ - TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state, - const std::vector& train_model_data, const std::vector& eval_model_data = {}, - const std::vector& optim_model_data = {}); - /// @} - - /// \name Implementing The Training Loop - /// @{ - /** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs - * - * This function performs a training step that computes the outputs of the training model and the gradients - * of the trainable parameters for the given inputs. The train step is performed based on the training model - * that was provided to the training session. - * The Ort::TrainingSession::TrainStep is equivalent of running forward propagation and backward propagation in a single - * step. - * The gradients computed are stored inside the training session state so they can be later consumed - * by the Ort::TrainingSession::OptimizerStep function. - * The gradients can be lazily reset by invoking the Ort::TrainingSession::LazyResetGrad function. - * - * \param[in] input_values The user inputs to the training model. - * \return A std::vector of Ort::Value objects that represents the output of the forward pass of the training model. - * - * - */ - std::vector TrainStep(const std::vector& input_values); - - /** \brief Reset the gradients of all trainable parameters to zero lazily. - * - * This function sets the internal state of the training session such that the gradients of the trainable - * parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are - * computed on the next invocation of the next Ort::TrainingSession::TrainStep. - * - */ - void LazyResetGrad(); - - /** \brief Computes the outputs for the eval model for the given inputs - * - * This function performs an eval step that computes the outputs of the eval model for the given inputs. - * The eval step is performed based on the eval model that was provided to the training session. - * - * \param[in] input_values The user inputs to the eval model. - * \return A std::vector of Ort::Value objects that represents the output of the eval pass. - * - */ - std::vector EvalStep(const std::vector& input_values); - - /** \brief Sets the learning rate for this training session. - * - * This function allows users to set the learning rate for the training session. The current - * learning rate is maintained by the training session and can be overwritten by invoking - * this function with the desired learning rate. This function should not be used when a valid - * learning rate scheduler is registered. It should be used either to set the learning rate - * derived from a custom learning rate scheduler or to set a constant learning rate to be used - * throughout the training session. - * \note Please note that this function does not set the initial learning rate that may be needed - * by the predefined learning rate schedulers. To set the initial learning rate for learning - * rate schedulers, please look at the function Ort::TrainingSession::RegisterLinearLRScheduler. - * - * \param[in] learning_rate Desired learning rate to be set. - * - */ - void SetLearningRate(float learning_rate); - - /** \brief Gets the current learning rate for this training session. - * - * This function allows users to get the learning rate for the training session. The current - * learning rate is maintained by the training session, and users can query it for the purpose - * of implementing their own learning rate schedulers. - * - * \return float representing the current learning rate. - * - */ - float GetLearningRate() const; - - /** \brief Registers a linear learning rate scheduler for the training session. - * - * Register a linear learning rate scheduler that decays the learning rate by linearly updated - * multiplicative factor from the initial learning rate set on the training session to 0. The decay - * is performed after the initial warm up phase where the learning rate is linearly incremented - * from 0 to the initial learning rate provided. - * - * \param[in] warmup_step_count Warmup steps for LR warmup. - * \param[in] total_step_count Total step count. - * \param[in] initial_lr The initial learning rate to be used by the training session. - * - */ - void RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count, - float initial_lr); - - /** \brief Update the learning rate based on the registered learing rate scheduler. - * - * Takes a scheduler step that updates the learning rate that is being used by the training session. - * This function should typically be called before invoking the optimizer step for each round, - * or as determined necessary to update the learning rate being used by the training session. - * \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this - * function. - * - */ - void SchedulerStep(); - - /** \brief Performs the weight updates for the trainable parameters using the optimizer model. - * - * This function performs the weight update step that updates the trainable parameters such that they - * take a step in the direction of their gradients (gradient descent). The optimizer step is performed - * based on the optimizer model that was provided to the training session. - * The updated parameters are stored inside the training state so that they can be used by the next - * Ort::TrainingSession::TrainStep function call. - * - */ - void OptimizerStep(); - - /// @} - - /// \name Prepare For Inferencing - /// @{ - - /** \brief Export a model that can be used for inferencing. - * - * If the training session was provided with an eval model, the training session can generate - * an inference model if it knows the inference graph outputs. The input inference graph outputs - * are used to prune the eval model so that the inference model's outputs align with the provided outputs. - * The exported model is saved at the path provided and can be used for inferencing with Ort::Session. - * \note Note that the function re-loads the eval model from the path provided to Ort::TrainingSession - * and expects that this path still be valid. - * - * \param[in] inference_model_path Path where the inference model should be serialized to. - * \param[in] graph_output_names Names of the outputs that are needed in the inference model. - * - */ - void ExportModelForInferencing(const std::basic_string& inference_model_path, - const std::vector& graph_output_names); - - /// @} - - /// \name Model IO Information - /// @{ - /** \brief Retrieves the names of the user inputs for the training and eval models. - * - * This function returns the names of inputs of the training or eval model that can be associated - * with the Ort::Value(s) provided to the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep - * function. - * - * \param[in] training Whether the training model input names are requested or eval model input names. - * \return Graph input names for either the training model or the eval model. - * - */ - std::vector InputNames(const bool training); - - /** \brief Retrieves the names of the user outputs for the training and eval models. - * - * This function returns the names of outputs of the training or eval model that can be associated - * with the Ort::Value(s) returned by the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep - * function. - * - * \param[in] training Whether the training model output names are requested or eval model output names. - * \return Graph output names for either the training model or the eval model. - * - */ - std::vector OutputNames(const bool training); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Returns a contiguous buffer that holds a copy of all training state parameters - * - * \param[in] only_trainable Whether to only copy trainable parameters or to copy all parameters. - * \return Contiguous buffer to the model parameters. - * - */ - Value ToBuffer(const bool only_trainable); - - /** \brief Loads the training session model parameters from a contiguous buffer - * - * In case the training session was created with a nominal checkpoint, invoking this function is required - * to load the updated parameters onto the checkpoint to complete it. - * - * \param[in] buffer Contiguous buffer to load the parameters from. - */ - void FromBuffer(Value& buffer); - - /// @} -}; - -/// \name Training Utilities -/// @{ -/** \brief This function sets the seed for generating random numbers. - * - * Use this function to generate reproducible results. It should be noted that completely - * reproducible results are not guaranteed. - * - * \param[in] seed Manual seed to use for random number generation. - */ -void SetSeed(const int64_t seed); -/// @} - -/// @} - -} // namespace Ort - -#include "onnxruntime_training_cxx_inline.h" diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_cxx_inline.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_cxx_inline.h deleted file mode 100644 index 397cba0..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/onnxruntime_training_cxx_inline.h +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "onnxruntime_training_c_api.h" -#include "onnxruntime_cxx_api.h" - -namespace Ort { - -inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options, - CheckpointState& checkpoint_state, - const std::basic_string& train_model_path, - const std::optional>& eval_model_path, - const std::optional>& optimizer_model_path) { - ThrowOnError(GetTrainingApi().CreateTrainingSession( - env, session_options, checkpoint_state, - train_model_path.c_str(), - eval_model_path.has_value() ? eval_model_path.value().c_str() : nullptr, - optimizer_model_path.has_value() ? optimizer_model_path.value().c_str() : nullptr, - &p_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_)); -} - -inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options, - CheckpointState& checkpoint_state, - const std::vector& train_model_data, - const std::vector& eval_model_data, - const std::vector& optim_model_data) { - ThrowOnError(GetTrainingApi().CreateTrainingSessionFromBuffer( - env, session_options, checkpoint_state, - train_model_data.data(), train_model_data.size(), - eval_model_data.data(), eval_model_data.size(), - optim_model_data.data(), optim_model_data.size(), - &p_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_)); -} - -inline std::vector TrainingSession::TrainStep(const std::vector& input_values) { - std::vector output_values; - output_values.reserve(training_model_output_count_); - for (size_t i = 0; i < training_model_output_count_; i++) output_values.emplace_back(nullptr); - auto ort_input_values = reinterpret_cast(input_values.data()); - auto ort_output_values = reinterpret_cast(output_values.data()); - RunOptions run_options; - ThrowOnError(GetTrainingApi().TrainStep( - p_, run_options, input_values.size(), ort_input_values, - training_model_output_count_, ort_output_values)); - - return output_values; -} - -inline void TrainingSession::LazyResetGrad() { - ThrowOnError(GetTrainingApi().LazyResetGrad(p_)); -} - -inline std::vector TrainingSession::EvalStep(const std::vector& input_values) { - std::vector output_values; - output_values.reserve(eval_model_output_count_); - for (size_t i = 0; i < eval_model_output_count_; i++) output_values.emplace_back(nullptr); - auto ort_input_values = reinterpret_cast(input_values.data()); - auto ort_output_values = reinterpret_cast(output_values.data()); - RunOptions run_options; - ThrowOnError(GetTrainingApi().EvalStep( - p_, run_options, input_values.size(), ort_input_values, - eval_model_output_count_, ort_output_values)); - - return output_values; -} - -inline void TrainingSession::SetLearningRate(float learning_rate) { - ThrowOnError(GetTrainingApi().SetLearningRate(p_, learning_rate)); -} - -inline float TrainingSession::GetLearningRate() const { - float learning_rate = 0; - ThrowOnError(GetTrainingApi().GetLearningRate(p_, &learning_rate)); - return learning_rate; -} - -inline void TrainingSession::RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count, - float initial_lr) { - ThrowOnError(GetTrainingApi().RegisterLinearLRScheduler(p_, warmup_step_count, total_step_count, - initial_lr)); -} - -inline void TrainingSession::SchedulerStep() { - ThrowOnError(GetTrainingApi().SchedulerStep(p_)); -} - -inline void TrainingSession::OptimizerStep() { - RunOptions run_options; - ThrowOnError(GetTrainingApi().OptimizerStep(p_, run_options)); -} - -inline std::vector TrainingSession::InputNames(const bool training) { - auto& input_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputCount - : GetTrainingApi().TrainingSessionGetEvalModelInputCount; - auto& input_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputName - : GetTrainingApi().TrainingSessionGetEvalModelInputName; - - size_t input_count = 0; - ThrowOnError(input_count_function(p_, &input_count)); - std::vector input_names(input_count); - AllocatorWithDefaultOptions allocator; - for (size_t index = 0; index < input_count; ++index) { - char* input_name; - ThrowOnError(input_name_function(p_, index, allocator, &input_name)); - input_names[index] = std::string(input_name); - allocator.Free(input_name); - } - - return input_names; -} - -inline std::vector TrainingSession::OutputNames(const bool training) { - auto& output_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputCount - : GetTrainingApi().TrainingSessionGetEvalModelOutputCount; - auto& output_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputName - : GetTrainingApi().TrainingSessionGetEvalModelOutputName; - - size_t output_count = 0; - ThrowOnError(output_count_function(p_, &output_count)); - std::vector output_names(output_count); - AllocatorWithDefaultOptions allocator; - for (size_t index = 0; index < output_count; ++index) { - char* output_name; - ThrowOnError(output_name_function(p_, index, allocator, &output_name)); - output_names[index] = std::string(output_name); - allocator.Free(output_name); - } - - return output_names; -} - -inline Value TrainingSession::ToBuffer(const bool only_trainable) { - size_t buffer_size = 0U; - ThrowOnError(GetTrainingApi().GetParametersSize(p_, &buffer_size, only_trainable)); - - std::array buffer_shape{static_cast(buffer_size)}; - - AllocatorWithDefaultOptions allocator; - Value buffer = Value::CreateTensor(allocator, buffer_shape.data(), 1U, - ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); - - ThrowOnError(GetTrainingApi().CopyParametersToBuffer(p_, buffer, only_trainable)); - - return buffer; -} - -inline void TrainingSession::FromBuffer(Value& buffer) { - if (!buffer.IsTensor()) { - ThrowStatus(Status("Incorrect buffer received. Expected a tensor buffer.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - } - - auto tensor_info = buffer.GetTensorTypeAndShapeInfo(); - auto buffer_shape = tensor_info.GetShape(); - - if (buffer_shape.size() != 1U) { - ThrowStatus(Status("Incorrect buffer received. Expected a contiguous tensor buffer.", - OrtErrorCode::ORT_INVALID_ARGUMENT)); - } - - auto buffer_size = buffer_shape.front(); - - size_t session_buffer_size = 0U; - ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size, false)); - - if (buffer_size == static_cast(session_buffer_size)) { - ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, false)); - return; - } - - size_t session_buffer_size_trainable_only = 0U; - ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size_trainable_only, true)); - - if (buffer_size == static_cast(session_buffer_size_trainable_only)) { - ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, true)); - return; - } else { - ThrowStatus(Status("Incorrect buffer size received.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - } -} - -inline CheckpointState CheckpointState::LoadCheckpoint(const std::basic_string& path_to_checkpoint) { - OrtCheckpointState* checkpoint_state; - ThrowOnError(GetTrainingApi().LoadCheckpoint(path_to_checkpoint.c_str(), &checkpoint_state)); - return CheckpointState(checkpoint_state); -} - -inline CheckpointState CheckpointState::LoadCheckpointFromBuffer(const std::vector& buffer) { - OrtCheckpointState* checkpoint_state; - ThrowOnError(GetTrainingApi().LoadCheckpointFromBuffer(buffer.data(), buffer.size(), &checkpoint_state)); - return CheckpointState(checkpoint_state); -} - -inline void CheckpointState::SaveCheckpoint(const CheckpointState& checkpoint_states, - const std::basic_string& path_to_checkpoint, - const bool include_optimizer_state) { - ThrowOnError(GetTrainingApi().SaveCheckpoint(checkpoint_states, path_to_checkpoint.c_str(), - include_optimizer_state)); -} - -inline void TrainingSession::ExportModelForInferencing(const std::basic_string& inference_model_path, - const std::vector& graph_output_names) { - std::vector output_names; - output_names.reserve(graph_output_names.size()); - for (const auto& output_name : graph_output_names) { - output_names.push_back(output_name.c_str()); - } - ThrowOnError(GetTrainingApi().ExportModelForInferencing( - p_, inference_model_path.c_str(), graph_output_names.size(), output_names.data())); -} - -inline void SetSeed(const int64_t seed) { - ThrowOnError(GetTrainingApi().SetSeed(seed)); -} - -inline void CheckpointState::AddProperty(const std::string& property_name, const Property& property_value) { - if (std::holds_alternative(property_value)) { - int64_t value = std::get(property_value); - void* value_p = &value; - ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtIntProperty, value_p)); - } else if (std::holds_alternative(property_value)) { - float value = std::get(property_value); - void* value_p = &value; - ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtFloatProperty, value_p)); - } else if (std::holds_alternative(property_value)) { - std::string value = std::get(property_value); - auto buffer = std::make_unique(value.length() + 1); - memcpy(buffer.get(), value.c_str(), value.length()); - // AddProperty takes a char* and calls PropertyBag::AddProperty which takes a std::string. The data will be - // copied at that point so buffer can free the local allocation once the call is made. - ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtStringProperty, - buffer.get())); - } else { - ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - } -} - -inline Property CheckpointState::GetProperty(const std::string& property_name) { - void* property_value = nullptr; - OrtPropertyType property_type; - - AllocatorWithDefaultOptions allocator; - ThrowOnError(GetTrainingApi().GetProperty(p_, property_name.c_str(), allocator, &property_type, &property_value)); - - Property property; - - switch (property_type) { - case OrtPropertyType::OrtIntProperty: { - auto value_p = reinterpret_cast(property_value); - property = *value_p; - allocator.Free(property_value); - break; - } - case OrtPropertyType::OrtFloatProperty: { - auto value_p = reinterpret_cast(property_value); - property = *value_p; - allocator.Free(property_value); - break; - } - case OrtPropertyType::OrtStringProperty: { - auto value_p = reinterpret_cast(property_value); - property = std::string(value_p); - allocator.Free(property_value); - break; - } - default: { - ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - break; - } - } - - return property; -} - -inline void CheckpointState::UpdateParameter(const std::string& parameter_name, const Value& parameter) { - ThrowOnError(GetTrainingApi().UpdateParameter(p_, parameter_name.c_str(), parameter)); -} - -inline Value CheckpointState::GetParameter(const std::string& parameter_name) { - AllocatorWithDefaultOptions allocator; - OrtValue* parameter; - ThrowOnError(GetTrainingApi().GetParameter(p_, parameter_name.c_str(), allocator, ¶meter)); - - return Value{parameter}; -} - -} // namespace Ort diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/provider_options.h b/inference_core/ort_core/onnxruntime-pkg-aarch64/include/provider_options.h deleted file mode 100644 index aab13e8..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/include/provider_options.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include -#include -#include - -namespace onnxruntime { - -// data types for execution provider options - -using ProviderOptions = std::unordered_map; -using ProviderOptionsVector = std::vector; -using ProviderOptionsMap = std::unordered_map; - -} // namespace onnxruntime diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/lib/libonnxruntime.so b/inference_core/ort_core/onnxruntime-pkg-aarch64/lib/libonnxruntime.so deleted file mode 120000 index cc5e169..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-aarch64/lib/libonnxruntime.so +++ /dev/null @@ -1 +0,0 @@ -libonnxruntime.so.1.18.1 \ No newline at end of file diff --git a/inference_core/ort_core/onnxruntime-pkg-aarch64/lib/libonnxruntime.so.1.18.1 b/inference_core/ort_core/onnxruntime-pkg-aarch64/lib/libonnxruntime.so.1.18.1 deleted file mode 100755 index d9befc9..0000000 Binary files a/inference_core/ort_core/onnxruntime-pkg-aarch64/lib/libonnxruntime.so.1.18.1 and /dev/null differ diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/GIT_COMMIT_ID b/inference_core/ort_core/onnxruntime-pkg-x86/GIT_COMMIT_ID deleted file mode 100644 index a1dfcbc..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/GIT_COMMIT_ID +++ /dev/null @@ -1 +0,0 @@ -387127404e6c1d84b3468c387d864877ed1c67fe diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/LICENSE b/inference_core/ort_core/onnxruntime-pkg-x86/LICENSE deleted file mode 100644 index 48bc6bb..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) Microsoft Corporation - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/Privacy.md b/inference_core/ort_core/onnxruntime-pkg-x86/Privacy.md deleted file mode 100644 index fcc8468..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/Privacy.md +++ /dev/null @@ -1,21 +0,0 @@ -# Privacy - -## Data Collection -The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft's privacy statement. Our privacy statement is located at https://go.microsoft.com/fwlink/?LinkID=824704. You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. - -*** - -### Private Builds -No data collection is performed when using your private builds built from source code. - -### Official Builds -ONNX Runtime does not maintain any independent telemetry collection mechanisms outside of what is provided by the platforms it supports. However, where applicable, ONNX Runtime will take advantage of platform-supported telemetry systems to collect trace events with the goal of improving product quality. - -Currently telemetry is only implemented for Windows builds and is turned **ON** by default in the official builds distributed in their respective package management repositories ([see here](../README.md#binaries)). This may be expanded to cover other platforms in the future. Data collection is implemented via 'Platform Telemetry' per vendor platform providers (see [telemetry.h](../onnxruntime/core/platform/telemetry.h)). - -#### Technical Details -The Windows provider uses the [TraceLogging](https://docs.microsoft.com/en-us/windows/win32/tracelogging/trace-logging-about) API for its implementation. This enables ONNX Runtime trace events to be collected by the operating system, and based on user consent, this data may be periodically sent to Microsoft servers following GDPR and privacy regulations for anonymity and data access controls. - -Windows ML and onnxruntime C APIs allow Trace Logging to be turned on/off (see [API pages](../README.md#api-documentation) for details). -For information on how to enable and disable telemetry, see [C API: Telemetry](./C_API.md#telemetry). -There are equivalent APIs in the C#, Python, and Java language bindings as well. diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/README.md b/inference_core/ort_core/onnxruntime-pkg-x86/README.md deleted file mode 100644 index 24c3e19..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/README.md +++ /dev/null @@ -1,61 +0,0 @@ -

- -**ONNX Runtime is a cross-platform inference and training machine-learning accelerator**. - -**ONNX Runtime inference** can enable faster customer experiences and lower costs, supporting models from deep learning frameworks such as PyTorch and TensorFlow/Keras as well as classical machine learning libraries such as scikit-learn, LightGBM, XGBoost, etc. ONNX Runtime is compatible with different hardware, drivers, and operating systems, and provides optimal performance by leveraging hardware accelerators where applicable alongside graph optimizations and transforms. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-inferencing) - -**ONNX Runtime training** can accelerate the model training time on multi-node NVIDIA GPUs for transformer models with a one-line addition for existing PyTorch training scripts. [Learn more →](https://www.onnxruntime.ai/docs/#onnx-runtime-for-training) - -## Get Started & Resources - -* **General Information**: [onnxruntime.ai](https://onnxruntime.ai) - -* **Usage documentation and tutorials**: [onnxruntime.ai/docs](https://onnxruntime.ai/docs) - -* **YouTube video tutorials**: [youtube.com/@ONNXRuntime](https://www.youtube.com/@ONNXRuntime) - -* [**Upcoming Release Roadmap**](https://github.com/microsoft/onnxruntime/wiki/Upcoming-Release-Roadmap) - -* **Companion sample repositories**: - - ONNX Runtime Inferencing: [microsoft/onnxruntime-inference-examples](https://github.com/microsoft/onnxruntime-inference-examples) - - ONNX Runtime Training: [microsoft/onnxruntime-training-examples](https://github.com/microsoft/onnxruntime-training-examples) - -## Builtin Pipeline Status - -|System|Inference|Training| -|---|---|---| -|Windows|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20CPU%20CI%20Pipeline?label=Windows+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=9)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20GPU%20CI%20Pipeline?label=Windows+GPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=10)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Windows%20GPU%20TensorRT%20CI%20Pipeline?label=Windows+GPU+TensorRT)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=47)|| -|Linux|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20CPU%20CI%20Pipeline?label=Linux+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=11)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20CPU%20Minimal%20Build%20E2E%20CI%20Pipeline?label=Linux+CPU+Minimal+Build)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=64)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20GPU%20CI%20Pipeline?label=Linux+GPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=12)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20GPU%20TensorRT%20CI%20Pipeline?label=Linux+GPU+TensorRT)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=45)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Linux%20OpenVINO%20CI%20Pipeline?label=Linux+OpenVINO)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=55)|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining-linux-ci-pipeline?label=Linux+CPU+Training)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=86)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining-linux-gpu-ci-pipeline?label=Linux+GPU+Training)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=84)
[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/orttraining/orttraining-ortmodule-distributed?label=Training+Distributed)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=148)| -|Mac|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/MacOS%20CI%20Pipeline?label=MacOS+CPU)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=13)|| -|Android|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Android%20CI%20Pipeline?label=Android)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=53)|| -|iOS|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/iOS%20CI%20Pipeline?label=iOS)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=134)|| -|Web|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/ONNX%20Runtime%20Web%20CI%20Pipeline?label=Web)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=161)|| -|Other|[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/onnxruntime-binary-size-checks-ci-pipeline?repoName=microsoft%2Fonnxruntime&label=Binary+Size+Check)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=187&repoName=microsoft%2Fonnxruntime)|| - -## Third-party Pipeline Status - -|System|Inference|Training| -|---|---|---| -|Linux|[![Build Status](https://github.com/Ascend/onnxruntime/actions/workflows/build-and-test.yaml/badge.svg)](https://github.com/Ascend/onnxruntime/actions/workflows/build-and-test.yaml)|| - -## Data/Telemetry - -Windows distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. - -## Contributions and Feedback - -We welcome contributions! Please see the [contribution guidelines](CONTRIBUTING.md). - -For feature requests or bug reports, please file a [GitHub Issue](https://github.com/Microsoft/onnxruntime/issues). - -For general discussion or questions, please use [GitHub Discussions](https://github.com/microsoft/onnxruntime/discussions). - -## Code of Conduct - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) -or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - -## License - -This project is licensed under the [MIT License](LICENSE). diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/ThirdPartyNotices.txt b/inference_core/ort_core/onnxruntime-pkg-x86/ThirdPartyNotices.txt deleted file mode 100644 index dcbdbdf..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/ThirdPartyNotices.txt +++ /dev/null @@ -1,6508 +0,0 @@ -THIRD PARTY SOFTWARE NOTICES AND INFORMATION - -Do Not Translate or Localize - -This software incorporates material from third parties. Microsoft makes certain -open source code available at http://3rdpartysource.microsoft.com, or you may -send a check or money order for US $5.00, including the product name, the open -source component name, and version number, to: - -Source Code Compliance Team -Microsoft Corporation -One Microsoft Way -Redmond, WA 98052 -USA - -Notwithstanding any other terms, you may reverse engineer this software to the -extent required to debug changes to any libraries licensed under the GNU Lesser -General Public License. - -_____ - -Intel Math Kernel Library (Intel MKL) - -Intel Simplified Software License (Version April 2018) - -Copyright (c) 2018 Intel Corporation. - -Use and Redistribution. You may use and redistribute the software (the “Software”), without modification, -provided the following conditions are met: - -* Redistributions must reproduce the above copyright notice and the following terms of use in the Software -and in the documentation and/or other materials provided with the distribution. - -* Neither the name of Intel nor the names of its suppliers may be used to endorse or promote products -derived from this Software without specific prior written permission. - -* No reverse engineering, decompilation, or disassembly of this Software is permitted. - -Limited patent license. Intel grants you a world-wide, royalty-free, non-exclusive license under patents it now -or hereafter owns or controls to make, have made, use, import, offer to sell and sell (“Utilize”) this Software, -but solely to the extent that any such patent is necessary to Utilize the Software alone. The patent license -shall not apply to any combinations which include this software. No hardware per se is licensed hereunder. - -Third party and other Intel programs. “Third Party Programs” are the files listed in the “third-party-programs.txt” -text file that is included with the Software and may include Intel programs under separate license terms. -Third Party Programs, even if included with the distribution of the Materials, are governed by -separate license terms and those license terms solely govern your use of those programs. - -DISCLAIMER. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT ARE DISCLAIMED. THIS SOFTWARE IS -NOT INTENDED FOR USE IN SYSTEMS OR APPLICATIONS WHERE FAILURE OF THE SOFTWARE -MAY CAUSE PERSONAL INJURY OR DEATH AND YOU AGREE THAT YOU ARE FULLY RESPONSIBLE FOR ANY -CLAIMS, COSTS, DAMAGES, EXPENSES, AND ATTORNEYS’ FEES ARISING OUT OF ANY SUCH USE, -EVEN IF ANY CLAIM ALLEGES THAT INTEL WAS NEGLIGENT REGARDING THE DESIGN OR MANUFACTURE OF -THE MATERIALS. - -LIMITATION OF LIABILITY. IN NO EVENT WILL INTEL BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -OF SUCH DAMAGE. YOU AGREE TO INDEMNIFY AND HOLD INTEL HARMLESS AGAINST ANY CLAIMS -AND EXPENSES RESULTING FROM YOUR USE OR UNAUTHORIZED USE OF THE SOFTWARE. - -No support. Intel may make changes to the Software, at any time without notice, and is not obligated to -support, update or provide training for the Software. - -Termination. Intel may terminate your right to use the Software in the event of your breach of this Agreement -and you fail to cure the breach within a reasonable period of time. - -Feedback. Should you provide Intel with comments, modifications, corrections, enhancements or other input -(“Feedback”) related to the Software Intel will be free to use, disclose, reproduce, license or otherwise -distribute or exploit the Feedback in its sole discretion without any obligations or restrictions of any kind, -including without limitation, intellectual property rights or licensing obligations. - -Compliance with laws. You agree to comply with all relevant laws and regulations governing your use, -transfer, import or export (or prohibition thereof) of the Software. - -Governing law. All disputes will be governed by the laws of the United States of America and the State of -Delaware without reference to conflict of law principles and subject to the exclusive jurisdiction of the state or -federal courts sitting in the State of Delaware, and each party agrees that it submits to the personal -jurisdiction and venue of those courts and waives any objections. The United Nations Convention on -Contracts for the International Sale of Goods (1980) is specifically excluded and will not apply to the -Software. - -*Other names and brands may be claimed as the property of others. - -_____ - -protocolbuffers/protobuf - -Copyright 2008 Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -_____ - -madler/zlib - -The deflate format used by zlib was defined by Phil Katz. The deflate and -zlib specifications were written by L. Peter Deutsch. Thanks to all the -people who reported problems and suggested various improvements in zlib; they -are too numerous to cite here. - -Copyright notice: - - (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - -If you use the zlib library in a product, we would appreciate *not* receiving -lengthy legal documents to sign. The sources are provided for free but without -warranty of any kind. The library has been entirely written by Jean-loup -Gailly and Mark Adler; it does not include third-party code. - -If you redistribute modified sources, we would appreciate that you include in -the file ChangeLog history information documenting your changes. Please read -the FAQ for more information on the distribution of modified source versions. - -_____ - -pybind/pybind11 - -Copyright (c) 2016 Wenzel Jakob , All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Please also refer to the file CONTRIBUTING.md, which clarifies licensing of -external contributions to this project including patches, pull requests, etc. - -_____ - -onnx -Open Neural Network Exchange - -Copyright (c) Facebook, Inc. and Microsoft Corporation. All rights reserved. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -Eigen - -MPL v2.0 -Mozilla Public License Version 2.0 - - -================================== - -1. Definitions - --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions - --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities - -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination - --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. - -_____ - -intel/dnnl - -Copyright 2016-2018 Intel Corporation - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -sub-components: - -xbyak - -Copyright (c) 2007 MITSUNARI Shigeo. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. -Neither the name of the copyright owner nor the names of its contributors may -be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -Microsoft GSL - -Copyright (c) 2015 Microsoft Corporation. All rights reserved. - -This code is licensed under the MIT License (MIT). - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -_____ - -Tensorflow - -Copyright 2018 The TensorFlow Authors. All rights reserved. - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, -and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by -the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all -other entities that control, are controlled by, or are under common -control with that entity. For the purposes of this definition, -"control" means (i) the power, direct or indirect, to cause the -direction or management of such entity, whether by contract or -otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity -exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, -including but not limited to software source code, documentation -source, and configuration files. - -"Object" form shall mean any form resulting from mechanical -transformation or translation of a Source form, including but -not limited to compiled object code, generated documentation, -and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or -Object form, made available under the License, as indicated by a -copyright notice that is included in or attached to the work -(an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object -form, that is based on (or derived from) the Work and for which the -editorial revisions, annotations, elaborations, or other modifications -represent, as a whole, an original work of authorship. For the purposes -of this License, Derivative Works shall not include works that remain -separable from, or merely link (or bind by name) to the interfaces of, -the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including -the original version of the Work and any modifications or additions -to that Work or Derivative Works thereof, that is intentionally -submitted to Licensor for inclusion in the Work by the copyright owner -or by an individual or Legal Entity authorized to submit on behalf of -the copyright owner. For the purposes of this definition, "submitted" -means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, -and issue tracking systems that are managed by, or on behalf of, the -Licensor for the purpose of discussing and improving the Work, but -excluding communication that is conspicuously marked or otherwise -designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity -on behalf of whom a Contribution has been received by Licensor and -subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of -this License, each Contributor hereby grants to You a perpetual, -worldwide, non-exclusive, no-charge, royalty-free, irrevocable -copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the -Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of -this License, each Contributor hereby grants to You a perpetual, -worldwide, non-exclusive, no-charge, royalty-free, irrevocable -(except as stated in this section) patent license to make, have made, -use, offer to sell, sell, import, and otherwise transfer the Work, -where such license applies only to those patent claims licensable -by such Contributor that are necessarily infringed by their -Contribution(s) alone or by combination of their Contribution(s) -with the Work to which such Contribution(s) was submitted. If You -institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work -or a Contribution incorporated within the Work constitutes direct -or contributory patent infringement, then any patent licenses -granted to You under this License for that Work shall terminate -as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the -Work or Derivative Works thereof in any medium, with or without -modifications, and in Source or Object form, provided that You -meet the following conditions: - -(a) You must give any other recipients of the Work or -Derivative Works a copy of this License; and - -(b) You must cause any modified files to carry prominent notices -stating that You changed the files; and - -(c) You must retain, in the Source form of any Derivative Works -that You distribute, all copyright, patent, trademark, and -attribution notices from the Source form of the Work, -excluding those notices that do not pertain to any part of -the Derivative Works; and - -(d) If the Work includes a "NOTICE" text file as part of its -distribution, then any Derivative Works that You distribute must -include a readable copy of the attribution notices contained -within such NOTICE file, excluding those notices that do not -pertain to any part of the Derivative Works, in at least one -of the following places: within a NOTICE text file distributed -as part of the Derivative Works; within the Source form or -documentation, if provided along with the Derivative Works; or, -within a display generated by the Derivative Works, if and -wherever such third-party notices normally appear. The contents -of the NOTICE file are for informational purposes only and -do not modify the License. You may add Your own attribution -notices within Derivative Works that You distribute, alongside -or as an addendum to the NOTICE text from the Work, provided -that such additional attribution notices cannot be construed -as modifying the License. - -You may add Your own copyright statement to Your modifications and -may provide additional or different license terms and conditions -for use, reproduction, or distribution of Your modifications, or -for any such Derivative Works as a whole, provided Your use, -reproduction, and distribution of the Work otherwise complies with -the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, -any Contribution intentionally submitted for inclusion in the Work -by You to the Licensor shall be under the terms and conditions of -this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify -the terms of any separate license agreement you may have executed -with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade -names, trademarks, service marks, or product names of the Licensor, -except as required for reasonable and customary use in describing the -origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or -agreed to in writing, Licensor provides the Work (and each -Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -implied, including, without limitation, any warranties or conditions -of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A -PARTICULAR PURPOSE. You are solely responsible for determining the -appropriateness of using or redistributing the Work and assume any -risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, -whether in tort (including negligence), contract, or otherwise, -unless required by applicable law (such as deliberate and grossly -negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, -incidental, or consequential damages of any character arising as a -result of this License or out of the use or inability to use the -Work (including but not limited to damages for loss of goodwill, -work stoppage, computer failure or malfunction, or any and all -other commercial damages or losses), even if such Contributor -has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing -the Work or Derivative Works thereof, You may choose to offer, -and charge a fee for, acceptance of support, warranty, indemnity, -or other liability obligations and/or rights consistent with this -License. However, in accepting such obligations, You may act only -on Your own behalf and on Your sole responsibility, not on behalf -of any other Contributor, and only if You agree to indemnify, -defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason -of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017, The TensorFlow Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -Microsoft Cognitive Toolkit (CNTK) - -Copyright (c) Microsoft Corporation. All rights reserved. - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT -LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -NumPy License - -Copyright (c) 2005, NumPy Developers - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name of the NumPy Developers nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -Pytorch / Caffe2 - -From PyTorch: - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) -Copyright (c) 2006 Idiap Research Institute (Samy Bengio) -Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) - -From Caffe2: - -Copyright (c) 2016-present, Facebook Inc. All rights reserved. - -All contributions by Facebook: -Copyright (c) 2016 Facebook Inc. - -All contributions by Google: -Copyright (c) 2015 Google Inc. -All rights reserved. - -All contributions by Yangqing Jia: -Copyright (c) 2015 Yangqing Jia -All rights reserved. - -All contributions from Caffe: -Copyright(c) 2013, 2014, 2015, the respective contributors -All rights reserved. - -All other contributions: -Copyright(c) 2015, 2016 the respective contributors -All rights reserved. - -Caffe2 uses a copyright model similar to Caffe: each contributor holds -copyright over their contributions to Caffe2. The project versioning records -all such contribution and copyright details. If a contributor wants to further -mark their specific copyright on a particular contribution, they should -indicate their copyright solely in the commit message of the change when it is -committed. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America - and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -_____ - -Caffe - -COPYRIGHT - -All contributions by the University of California: -Copyright (c) 2014-2017 The Regents of the University of California (Regents) -All rights reserved. - -All other contributions: -Copyright (c) 2014-2017, the respective contributors -All rights reserved. - -Caffe uses a shared copyright model: each contributor holds copyright over -their contributions to Caffe. The project versioning records all such -contribution and copyright details. If a contributor wants to further mark -their specific copyright on a particular contribution, they should indicate -their copyright solely in the commit message of the change when it is -committed. - -LICENSE - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -CONTRIBUTION AGREEMENT - -By contributing to the BVLC/caffe repository through pull-request, comment, -or otherwise, the contributor releases their content to the -license and copyright terms herein. - -_____ - -The LLVM Compiler Infrastructure - -============================================================================== -LLVM Release License -============================================================================== -University of Illinois/NCSA -Open Source License - -Copyright (c) 2003-2017 University of Illinois at Urbana-Champaign. -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== -Copyrights and Licenses for Third Party Software Distributed with LLVM: -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- -Google Test llvm/utils/unittest/googletest -OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} -pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} -ARM contributions llvm/lib/Target/ARM/LICENSE.TXT -md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h - -_____ - -google/benchmark - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -CONTRIBUTORS - -# People who have agreed to one of the CLAs and can contribute patches. -# The AUTHORS file lists the copyright holders; this file -# lists people. For example, Google employees are listed here -# but not in AUTHORS, because Google holds the copyright. -# -# Names should be added to this file only after verifying that -# the individual or the individual's organization has agreed to -# the appropriate Contributor License Agreement, found here: -# -# https://developers.google.com/open-source/cla/individual -# https://developers.google.com/open-source/cla/corporate -# -# The agreement for individuals can be filled out on the web. -# -# When adding J Random Contributor's name to this file, -# either J's name or J's organization's name should be -# added to the AUTHORS file, depending on whether the -# individual or corporate CLA was used. -# -# Names should be added to this file as: -# Name -# -# Please keep the list sorted. - -Albert Pretorius -Arne Beer -Billy Robert O'Neal III -Chris Kennelly -Christopher Seymour -David Coeurjolly -Deniz Evrenci -Dominic Hamon -Dominik Czarnota -Eric Fiselier -Eugene Zhuk -Evgeny Safronov -Federico Ficarelli -Felix Homann -Ismael Jimenez Martinez -Jern-Kuan Leong -JianXiong Zhou -Joao Paulo Magalhaes -John Millikin -Jussi Knuuttila -Kai Wolf -Kishan Kumar -Kaito Udagawa -Lei Xu -Matt Clarkson -Maxim Vafin -Nick Hutchinson -Oleksandr Sochka -Pascal Leroy -Paul Redmond -Pierre Phaneuf -Radoslav Yovchev -Raul Marin -Ray Glover -Robert Guo -Roman Lebedev -Shuo Chen -Tobias Ulvgård -Tom Madams -Yixuan Qiu -Yusuke Suzuki -Zbigniew Skowron - -AUTHORS - -# This is the official list of benchmark authors for copyright purposes. -# This file is distinct from the CONTRIBUTORS files. -# See the latter for an explanation. -# -# Names should be added to this file as: -# Name or Organization -# The email address is not required for organizations. -# -# Please keep the list sorted. - -Albert Pretorius -Arne Beer -Carto -Christopher Seymour -David Coeurjolly -Deniz Evrenci -Dirac Research -Dominik Czarnota -Eric Fiselier -Eugene Zhuk -Evgeny Safronov -Federico Ficarelli -Felix Homann -Google Inc. -International Business Machines Corporation -Ismael Jimenez Martinez -Jern-Kuan Leong -JianXiong Zhou -Joao Paulo Magalhaes -Jussi Knuuttila -Kaito Udagawa -Kishan Kumar -Lei Xu -Matt Clarkson -Maxim Vafin -MongoDB Inc. -Nick Hutchinson -Oleksandr Sochka -Paul Redmond -Radoslav Yovchev -Roman Lebedev -Shuo Chen -Steinar H. Gunderson -Stripe, Inc. -Yixuan Qiu -Yusuke Suzuki -Zbigniew Skowron - -_____ - -HalideIR - -Copyright (c) 2016 HalideIR contributors -Copyright (c) 2012-2014 MIT CSAIL, Google Inc., and other contributors -HalideIR is derived from the Halide project. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -Distributed Machine Learning Common Codebase - -Copyright (c) 2015 by Contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -_____ - -DLPack: Open In Memory Tensor Structure - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 by Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -HowardHinnant/date - -The source code in this project is released using the MIT License. There is no -global license for the project because each file is licensed individually with -different author names and/or dates. - -If you contribute to this project, please add your name to the license of each -file you modify. If you have already contributed to this project and forgot to -add your name to the license, please feel free to submit a new P/R to add your -name to the license in each file you modified. - -For convenience, here is a copy of the MIT license found in each file except -without author names or dates: - -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -TVM Open Deep Learning Compiler Stack - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -CONTRIBUTORS - -TVM Contributors -================ -TVM adopts the Apache style model and governs by merit. We believe that it is important to create an inclusive community where everyone can use, -contribute to, and influence the direction of the project. We actively invite contributors who have earned the merit to be part of the development community. - -See the [community structure document](http://docs.tvm.ai/contribute/community.html) for the explanation of community structure and contribution guidelines. - -## Committers -- [Tianqi Chen](https://github.com/tqchen) (PMC) -- [Thierry Moreau](http://homes.cs.washington.edu/~moreau/) -- [Ziheng Jiang](https://github.com/ZihengJiang) -- [Haichen Shen](http://homes.cs.washington.edu/~haichen/) -- [Yizhi Liu](https://github.com/yzhliu) - -## Code Owners -- [Aditya Atluri](https://github.com/adityaatluri) ROCM -- [Leyuan Wang](https://github.com/Laurawly) TOPI -- [Yuwei Hu](https://github.com/Huyuwei) TOPI -- [Zhixun Tan](https://github.com/phisiart) OpenGL/WebGL backend -- [Nick Hynes](https://github.com/bfnes) SGX and secured computing -- [Lianmin Zheng](https://github.com/merrymercy) AutoTVM - -## Reviewers -- [Zhi Chen](https://github.com/zhiics) -- [Xiaoqiang Dan](https://github.com/xqdan) -- [Liangfu Chen](https://github.com/liangfu) -- [Masahiro Masuda](https://github.com/masahi) -- [Kazutaka Morita](https://github.com/kazum) -- [Tatsuya Nishiyama](https://github.com/nishi-t) -- [Pariksheet Pinjari](https://github.com/PariksheetPinjari909) -- [Jared Roesch](https://github.com/jroesch) -- [Siva](https://github.com/srkreddy1238) -- [Siju Samuel](https://github.com/siju-samuel) -- [Alex Weaver](https://github.com/alex-weaver) -- [Yao Wang](https://github.com/kevinthesun) -- [Jian Weng](https://github.com/were) -- [Eddie Yan](https://github.com/eqy) -- [Joshua Z. Zhang](https://github.com/zhreshold) - -## List of Contributors -- [Full List of Contributors](https://github.com/dmlc/tvm/graphs/contributors) - - To contributors: please add your name to the list. -- [Qiao Zhang](https://github.com/zhangqiaorjc) -- [Haolong Zhang](https://github.com/haolongzhangm) -- [Cody Hao Yu](https://github.com/comaniac) -- [Chris Nuernberger](https://github.com/cnuernber) - -_____ - -FreeBSD: getopt.c file - -Copyright (c) 1987, 1993, 1994 -The Regents of the University of California. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - -3. Neither the name of the University nor the names of its contributors -may be used to endorse or promote products derived from this software -without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. -_____ - - -google/googletest - -Copyright 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -G3log : Asynchronous logger with Dynamic Sinks - -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or -distribute this software, either in source code form or as a compiled -binary, for any purpose, commercial or non-commercial, and by any -means. - -In jurisdictions that recognize copyright laws, the author or authors -of this software dedicate any and all copyright interest in the -software to the public domain. We make this dedication for the benefit -of the public at large and to the detriment of our heirs and -successors. We intend this dedication to be an overt act of -relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to -_____ - -Scikit-learn - -Copyright (c) 2007–2018 The scikit-learn developers. -All rights reserved. - - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - a. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - b. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - c. Neither the name of the Scikit-learn Developers nor the names of - its contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. - -_____ - -google/nsync - -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -google/re2 - -Copyright (c) 2009 The RE2 Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -_____ -onnx/onnx-tensorrt - -MIT License - -Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -Copyright (c) 2018 Open Neural Network Exchange - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ -nvidia/cutlass - -Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: BSD-3-Clause - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ -Boost - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - - -_____ - -JDAI-CV/DNNLibrary - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [2019] [JD.com Inc. JD AI] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -google/flatbuffers - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2014 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -google/glog - -Copyright (c) 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -A function gettimeofday in utilities.cc is based on - -http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd - -The license of this code is: - -Copyright (c) 2003-2008, Jouni Malinen and contributors -All Rights Reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name(s) of the above-listed copyright holder(s) nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -abseil-cpp -https://github.com/abseil/abseil-cpp - - Apache License - Version 2.0, January 2004 - https://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -microsoft/wil - -MIT License - -Copyright (c) Microsoft Corporation. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE - -_____ - -nlohmann/json - -MIT License - -Copyright (c) 2013-2019 Niels Lohmann - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -dcleblanc/SafeInt - -MIT License - -Copyright (c) 2018 Microsoft - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ -Open MPI - -3-Clause BSD License - -Most files in this release are marked with the copyrights of the -organizations who have edited them. The copyrights below are in no -particular order and generally reflect members of the Open MPI core -team who have contributed code to this release. The copyrights for -code used under license from other parties are included in the -corresponding files. - -Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana - University Research and Technology - Corporation. All rights reserved. -Copyright (c) 2004-2017 The University of Tennessee and The University - of Tennessee Research Foundation. All rights - reserved. -Copyright (c) 2004-2010 High Performance Computing Center Stuttgart, - University of Stuttgart. All rights reserved. -Copyright (c) 2004-2008 The Regents of the University of California. - All rights reserved. -Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights - reserved. -Copyright (c) 2006-2017 Cisco Systems, Inc. All rights reserved. -Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved. -Copyright (c) 2006-2017 Sandia National Laboratories. All rights reserved. -Copyright (c) 2006-2010 Sun Microsystems, Inc. All rights reserved. - Use is subject to license terms. -Copyright (c) 2006-2017 The University of Houston. All rights reserved. -Copyright (c) 2006-2009 Myricom, Inc. All rights reserved. -Copyright (c) 2007-2017 UT-Battelle, LLC. All rights reserved. -Copyright (c) 2007-2017 IBM Corporation. All rights reserved. -Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing - Centre, Federal Republic of Germany -Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany -Copyright (c) 2007 Evergrid, Inc. All rights reserved. -Copyright (c) 2008 Chelsio, Inc. All rights reserved. -Copyright (c) 2008-2009 Institut National de Recherche en - Informatique. All rights reserved. -Copyright (c) 2007 Lawrence Livermore National Security, LLC. - All rights reserved. -Copyright (c) 2007-2017 Mellanox Technologies. All rights reserved. -Copyright (c) 2006-2010 QLogic Corporation. All rights reserved. -Copyright (c) 2008-2017 Oak Ridge National Labs. All rights reserved. -Copyright (c) 2006-2012 Oracle and/or its affiliates. All rights reserved. -Copyright (c) 2009-2015 Bull SAS. All rights reserved. -Copyright (c) 2010 ARM ltd. All rights reserved. -Copyright (c) 2016 ARM, Inc. All rights reserved. -Copyright (c) 2010-2011 Alex Brick . All rights reserved. -Copyright (c) 2012 The University of Wisconsin-La Crosse. All rights - reserved. -Copyright (c) 2013-2016 Intel, Inc. All rights reserved. -Copyright (c) 2011-2017 NVIDIA Corporation. All rights reserved. -Copyright (c) 2016 Broadcom Limited. All rights reserved. -Copyright (c) 2011-2017 Fujitsu Limited. All rights reserved. -Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP. All - rights reserved. -Copyright (c) 2013-2017 Research Organization for Information Science (RIST). - All rights reserved. -Copyright (c) 2017-2018 Amazon.com, Inc. or its affiliates. All Rights - reserved. -Copyright (c) 2018 DataDirect Networks. All rights reserved. -Copyright (c) 2018-2019 Triad National Security, LLC. All rights reserved. - -$COPYRIGHT$ - -Additional copyrights may follow - -$HEADER$ - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -- Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -- Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer listed - in this license in the documentation and/or other materials - provided with the distribution. - -- Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -The copyright holders provide no reassurances that the source code -provided does not infringe any patent, copyright, or any other -intellectual property rights of third parties. The copyright holders -disclaim any liability to any recipient for claims brought against -recipient by any third party for infringement of that parties -intellectual property rights. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -The Android Open Source Project - -Copyright (C) 2017 The Android Open Source Project -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ------- - -libprotobuf-mutator - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- - - openucx/ucx - https://github.com/openucx/ucx - - Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. - Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. - Copyright (C) 2014-2015 The University of Houston System. All rights reserved. - Copyright (C) 2015 The University of Tennessee and The University - of Tennessee Research Foundation. All rights reserved. - Copyright (C) 2016-2020 ARM Ltd. All rights reserved. - Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. - Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. - Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. - Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. - Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. - Copyright (C) 2016-2020 Stony Brook University. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - ----- - - From PyTorch: - - Copyright (c) 2016- Facebook, Inc (Adam Paszke) - Copyright (c) 2014- Facebook, Inc (Soumith Chintala) - Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) - Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) - Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) - Copyright (c) 2011-2013 NYU (Clement Farabet) - Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) - Copyright (c) 2006 Idiap Research Institute (Samy Bengio) - Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) - - From Caffe2: - - Copyright (c) 2016-present, Facebook Inc. All rights reserved. - - All contributions by Facebook: - Copyright (c) 2016 Facebook Inc. - - All contributions by Google: - Copyright (c) 2015 Google Inc. - All rights reserved. - - All contributions by Yangqing Jia: - Copyright (c) 2015 Yangqing Jia - All rights reserved. - - All contributions from Caffe: - Copyright(c) 2013, 2014, 2015, the respective contributors - All rights reserved. - - All other contributions: - Copyright(c) 2015, 2016 the respective contributors - All rights reserved. - - Caffe2 uses a copyright model similar to Caffe: each contributor holds - copyright over their contributions to Caffe2. The project versioning records - all such contribution and copyright details. If a contributor wants to further - mark their specific copyright on a particular contribution, they should - indicate their copyright solely in the commit message of the change when it is - committed. - - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America - and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -_____ - - mpi4py - https://github.com/mpi4py/mpi4py/ - - ======================= - LICENSE: MPI for Python - ======================= - - :Author: Lisandro Dalcin - :Contact: dalcinl@gmail.com - - - Copyright (c) 2019, Lisandro Dalcin. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -_____ -huggingface/transformers - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -_____ -msgpack/msgpack-python - -Copyright (C) 2008-2011 INADA Naoki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -_____ -lanpa/tensorboardX - -MIT License - -Copyright (c) 2017 Tzu-Wei Huang - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -_____ -tensorflow/tensorboard - -Copyright 2017 The TensorFlow Authors. All rights reserved. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017, The TensorFlow Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -_____ - -cerberus - -Cerberus is a lightweight and extensible data validation library for Python. - -ISC License - -Copyright (c) 2012-2016 Nicola Iarocci. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. - -_____ - -MurmurHash3 - -MIT license - -https://github.com/aappleby/smhasher - -SMHasher is a test suite designed to test the distribution, collision, and -performance properties of non-cryptographic hash functions. -This is the home for the MurmurHash family of hash functions along with the -SMHasher test suite used to verify them. -SMHasher is released under the MIT license. -All MurmurHash versions are public domain software, and the author disclaims all copyright to their code. - -_____ - -gtest-ios-framework - -https://github.com/mestevens/gtest-ios-framework - -Copyright (c) 2013 Matthew Stevens - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -DLPack - -https://github.com/dmlc/dlpack - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 by Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -emsdk - -MIT/Expat license - -https://github.com/emscripten-core/emsdk - -Copyright (c) 2018 Emscripten authors (see AUTHORS in Emscripten) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ----------------------------------------------------------------------------- - -This is the MIT/Expat Licence. For more information see: - -1. http://www.opensource.org/licenses/mit-license.php - -2. http://en.wikipedia.org/wiki/MIT_License - -_____ - -coremltools - -BSD-3-Clause License - -https://github.com/apple/coremltools - -Copyright (c) 2020, Apple Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -© 2021 GitHub, Inc. - -_____ - -react-native - -MIT License - -https://github.com/facebook/react-native - -Copyright (c) Facebook, Inc. and its affiliates. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -pytorch/cpuinfo - -BSD 2-Clause "Simplified" License - -https://github.com/pytorch/cpuinfo - -Copyright (c) 2019 Google LLC -Copyright (c) 2017-2018 Facebook Inc. -Copyright (C) 2012-2017 Georgia Institute of Technology -Copyright (C) 2010-2012 Marat Dukhan - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -SQLite Is Public Domain - -All of the code and documentation in SQLite has been dedicated to the public -domain by the authors. All code authors, and representatives of the companies -they work for, have signed affidavits dedicating their contributions to the -public domain and originals of those signed affidavits are stored in a firesafe -at the main offices of Hwaci. Anyone is free to copy, modify, publish, use, -compile, sell, or distribute the original SQLite code, either in source code -form or as a compiled binary, for any purpose, commercial or non-commercial, -and by any means. - -The previous paragraph applies to the deliverable code and documentation in -SQLite - those parts of the SQLite library that you actually bundle and ship -with a larger application. Some scripts used as part of the build process (for -example the "configure" scripts generated by autoconf) might fall under other -open-source licenses. Nothing from these build scripts ever reaches the final -deliverable SQLite library, however, and so the licenses associated with those -scripts should not be a factor in assessing your rights to copy and use the -SQLite library. - -All of the deliverable code in SQLite has been written from scratch. No code -has been taken from other projects or from the open internet. Every line of -code can be traced back to its original author, and all of those authors have -public domain dedications on file. So the SQLite code base is clean and is -uncontaminated with licensed code from other projects. - -_____ - -google/XNNPACK - -BSD License - -For XNNPACK software - -Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. -Copyright 2019 Google LLC - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -google/sentencepiece, https://github.com/google/sentencepiece -(included when statically linked with onnxruntime-extensions) - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -dlfcn-win32/dlfcn-win32 is licensed under the MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -_____ - -The Python Imaging Library (PIL) is - - Copyright © 1997-2011 by Secret Labs AB - Copyright © 1995-2011 by Fredrik Lundh - -Pillow is the friendly PIL fork. It is - - Copyright © 2010-2023 by Alex Clark and contributors - -Like PIL, Pillow is licensed under the open source HPND License: - -By obtaining, using, and/or copying this software and/or its associated -documentation, you agree that you have read, understood, and will comply -with the following terms and conditions: - -Permission to use, copy, modify, and distribute this software and its -associated documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appears in all copies, and that -both that copyright notice and this permission notice appear in supporting -documentation, and that the name of Secret Labs AB or the author not be -used in advertising or publicity pertaining to distribution of the software -without specific, written prior permission. - -SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS -SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. -IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, -INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. - -_____ - -openssl/openssl, https://github.com/openssl/openssl - - Apache License - Version 2.0, January 2004 - https://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -_____ - -Tencent/rapidjson, https://github.com/Tencent/rapidjson - -Tencent is pleased to support the open source community by making RapidJSON available. - -Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. - -If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. -A copy of the MIT License is included in this file. - -Other dependencies and licenses: - -Open Source Software Licensed Under the BSD License: --------------------------------------------------------------------- - -The msinttypes r29 -Copyright (c) 2006-2013 Alexander Chemeris -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* Neither the name of copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Open Source Software Licensed Under the JSON License: --------------------------------------------------------------------- - -json.org -Copyright (c) 2002 JSON.org -All Rights Reserved. - -JSON_checker -Copyright (c) 2002 JSON.org -All Rights Reserved. - - -Terms of the JSON License: ---------------------------------------------------- - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -The Software shall be used for Good, not Evil. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -Terms of the MIT License: --------------------------------------------------------------------- - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -_____ - -boostorg/boost, https://github.com/boostorg/boost - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -_____ - -libb64/libb64, https://github.com/libb64/libb64 - -Copyright-Only Dedication (based on United States law) or Public Domain Certification - -The person or persons who have associated work with this document (the "Dedicator" or "Certifier") hereby either (a) certifies that, to the best of his knowledge, the work of authorship identified is in the public domain of the country from which the work is published, or (b) hereby dedicates whatever copyright the dedicators holds in the work of authorship identified below (the "Work") to the public domain. A certifier, moreover, dedicates any copyright interest he may have in the associated work, and for these purposes, is described as a "dedicator" below. - -A certifier has taken reasonable steps to verify the copyright status of this work. Certifier recognizes that his good faith efforts may not shield him from liability if in fact the work certified is not in the public domain. - -Dedicator makes this dedication for the benefit of the public at large and to the detriment of the Dedicator's heirs and successors. Dedicator intends this dedication to be an overt act of relinquishment in perpetuity of all present and future rights under copyright law, whether vested or contingent, in the Work. Dedicator understands that such relinquishment of all rights includes the relinquishment of all rights to enforce (by lawsuit or otherwise) those copyrights in the Work. - -Dedicator recognizes that, once placed in the public domain, the Work may be freely reproduced, distributed, transmitted, used, modified, built upon, or otherwise exploited by anyone for any purpose, commercial or non-commercial, and in any way, including by methods that have not yet been invented or conceived. - -_____ - -posix pthread library, https://sourceforge.net/projects/pthreads4w - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -_____ - -Triton Inference Server & Client, https://github.com/triton-inference-server - -Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of NVIDIA CORPORATION nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -microsoft/mimalloc, https://github.com/microsoft/mimalloc - -MIT License - -Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -TensorFlow.js - -https://github.com/tensorflow/tfjs - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -—— - -curl/curl - -https://github.com/curl - -COPYRIGHT AND PERMISSION NOTICE - -Copyright (C) Daniel Stenberg, , and many -contributors, see the THANKS file. - -All rights reserved. - -Permission to use, copy, modify, and distribute this software for any purpose -with or without fee is hereby granted, provided that the above copyright -notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN -NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. - -Except as contained in this notice, the name of a copyright holder shall not -be used in advertising or otherwise to promote the sale, use or other dealings -in this Software without prior written authorization of the copyright holder. - -_____ - -Intel neural-compressor - -https://github.com/intel/neural-compressor - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - ============================================================================ - - Copyright 2016-2019 Intel Corporation - Copyright 2018 YANDEX LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - This distribution includes third party software ("third party programs"). - This third party software, even if included with the distribution of - the Intel software, may be governed by separate license terms, including - without limitation, third party license terms, other Intel software license - terms, and open source software license terms. These separate license terms - govern your use of the third party programs as set forth in the - "THIRD-PARTY-PROGRAMS" file. - -_____ - -FlashAttention, https://github.com/Dao-AILab/flash-attention - -BSD 3-Clause License - -Copyright (c) 2022, the respective contributors, as shown by the AUTHORS file. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -_____ - -composable_kernel - -https://github.com/ROCmSoftwarePlatform/composable_kernel - -Copyright (c) 2018- , Advanced Micro Devices, Inc. (Chao Liu, Jing Zhang) -Copyright (c) 2019- , Advanced Micro Devices, Inc. (Letao Qin, Qianfeng Zhang, Liang Huang, Shaojie Wang) -Copyright (c) 2022- , Advanced Micro Devices, Inc. (Anthony Chang, Chunyu Lai, Illia Silin, Adam Osewski, Poyen Chen, Jehandad Khan) -Copyright (c) 2019-2021, Advanced Micro Devices, Inc. (Hanwen Chang) -Copyright (c) 2019-2020, Advanced Micro Devices, Inc. (Tejash Shah) -Copyright (c) 2020 , Advanced Micro Devices, Inc. (Xiaoyan Zhou) -Copyright (c) 2021-2022, Advanced Micro Devices, Inc. (Jianfeng Yan) - -SPDX-License-Identifier: MIT -Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - -neural-speed - -https://github.com/intel/neural-speed - - Apache License - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - ============================================================================ - - Copyright 2016-2019 Intel Corporation - Copyright 2018 YANDEX LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - This distribution includes third party software ("third party programs"). - This third party software, even if included with the distribution of - the Intel software, may be governed by separate license terms, including - without limitation, third party license terms, other Intel software license - terms, and open source software license terms. These separate license terms - govern your use of the third party programs as set forth in the - "THIRD-PARTY-PROGRAMS" file. diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/VERSION_NUMBER b/inference_core/ort_core/onnxruntime-pkg-x86/VERSION_NUMBER deleted file mode 100644 index ec6d649..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/VERSION_NUMBER +++ /dev/null @@ -1 +0,0 @@ -1.18.1 diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/cpu_provider_factory.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/cpu_provider_factory.h deleted file mode 100644 index 2926786..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/cpu_provider_factory.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "onnxruntime_c_api.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \param use_arena zero: false. non-zero: true. - */ -ORT_EXPORT -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena) -ORT_ALL_ARGS_NONNULL; - -#ifdef __cplusplus -} -#endif diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_c_api.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_c_api.h deleted file mode 100644 index 52f496c..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_c_api.h +++ /dev/null @@ -1,4822 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// See docs\c_cxx\README.md on generating the Doxygen documentation from this file - -/** \mainpage ONNX Runtime - * - * ONNX Runtime is a high-performance inference and training graph execution engine for deep learning models. - * - * ONNX Runtime's C, C++ APIs offer an easy to use interface to onboard and execute onnx models. - * - \subpage c_cpp_api "Core C, C++ APIs" - * - \subpage training_c_cpp_api "Training C, C++ APIs for on-device training" - * - * \page c_cpp_api Core C, C++ APIs - *

C

- * - * ::OrtApi - Click here to go to the structure with all C API functions. - * - *

C++

- * - * ::Ort - Click here to go to the namespace holding all of the C++ wrapper classes - * - * It is a set of header only wrapper classes around the C API. The goal is to turn the C style return value error codes into C++ exceptions, and to - * automate memory management through standard C++ RAII principles. - * - * \addtogroup Global - * ONNX Runtime C API - * @{ - */ - -#pragma once -#include -#include -#include -#include - -/** \brief The API version defined in this header - * - * This value is used by some API functions to behave as this version of the header expects. - */ -#define ORT_API_VERSION 18 - -#ifdef __cplusplus -extern "C" { -#endif - -//! @} -// SAL2 Definitions -#ifndef _WIN32 -#define _In_ -#define _In_z_ -#define _In_opt_ -#define _In_opt_z_ -#define _Out_ -#define _Outptr_ -#define _Out_opt_ -#define _Inout_ -#define _Inout_opt_ -#define _Frees_ptr_opt_ -#define _Ret_maybenull_ -#define _Ret_notnull_ -#define _Check_return_ -#define _Outptr_result_maybenull_ -#define _In_reads_(X) -#define _Inout_updates_(X) -#define _Out_writes_(X) -#define _Inout_updates_all_(X) -#define _Out_writes_bytes_all_(X) -#define _Out_writes_all_(X) -#define _Success_(X) -#define _Outptr_result_buffer_maybenull_(X) -#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull)) -#else -#include -#define ORT_ALL_ARGS_NONNULL -#endif - -#ifdef _WIN32 -// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort. -// dllexport is not used, we use a .def file. -#ifdef ORT_DLL_IMPORT -#define ORT_EXPORT __declspec(dllimport) -#else -#define ORT_EXPORT -#endif -#define ORT_API_CALL _stdcall -#define ORT_MUST_USE_RESULT -#define ORTCHAR_T wchar_t -#else -// To make symbols visible on macOS/iOS -#ifdef __APPLE__ -#define ORT_EXPORT __attribute__((visibility("default"))) -#else -#define ORT_EXPORT -#endif -#define ORT_API_CALL -#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) -#define ORTCHAR_T char -#endif - -/// ORTCHAR_T, ORT_TSTR are reserved specifically for path handling. -/// All other strings are UTF-8 encoded, use char and std::string -#ifndef ORT_TSTR -#ifdef _WIN32 -#define ORT_TSTR(X) L##X -// When X is a macro, L##X is not defined. In this case, we need to use ORT_TSTR_ON_MACRO. -#define ORT_TSTR_ON_MACRO(X) L"" X -#else -#define ORT_TSTR(X) X -#define ORT_TSTR_ON_MACRO(X) X -#endif -#endif - -// On Windows, ORT_FILE is a wchar_t version of the __FILE__ macro. -// Otherwise, ORT_FILE is equivalent to __FILE__. -#ifndef ORT_FILE -#define ORT_FILE_INTERNAL(x) ORT_TSTR(x) -#define ORT_FILE ORT_FILE_INTERNAL(__FILE__) -#endif - -// Any pointer marked with _In_ or _Out_, cannot be NULL. - -// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation -// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. -// for ReleaseXXX(...) functions, they can accept NULL pointer. - -#ifdef __cplusplus -// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept. -// Such complex condition is needed because compilers set __cplusplus value differently. -#ifndef __has_feature -#define __has_feature(x) 0 -#endif -#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept))) -#define NO_EXCEPTION noexcept -#else -#define NO_EXCEPTION throw() -#endif -#else -#define NO_EXCEPTION -#endif - -// __VA_ARGS__ on Windows and Linux are different -#define ORT_API(RETURN_TYPE, NAME, ...) RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION - -#define ORT_API_STATUS(NAME, ...) \ - _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) \ - NO_EXCEPTION ORT_MUST_USE_RESULT - -// XXX: Unfortunately, SAL annotations are known to not work with function pointers -#define ORT_API2_STATUS(NAME, ...) \ - _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT - -// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT -#define ORT_API_STATUS_IMPL(NAME, ...) \ - _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION - -#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input) - -#ifdef __DOXYGEN__ -#undef ORT_API_STATUS -#define ORT_API_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) -#undef ORT_API2_STATUS -#define ORT_API2_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) -#undef ORT_CLASS_RELEASE -#define ORT_CLASS_RELEASE(X) void Release##X(Ort##X* input) -#undef NO_EXCEPTION -#define NO_EXCEPTION -#endif -/** \addtogroup Global - * ONNX Runtime C API - * @{ - */ - -/** Copied from TensorProto::DataType - * Currently, Ort doesn't support complex64, complex128 - */ -typedef enum ONNXTensorElementDataType { - ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string - ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, - ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16, // Non-IEEE floating-point format based on IEEE754 single-precision - // float 8 types were introduced in onnx 1.14, see https://onnx.ai/onnx/technical/float8.html - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ // Non-IEEE floating-point format based on IEEE754 single-precision -} ONNXTensorElementDataType; - -// Synced with onnx TypeProto oneof -typedef enum ONNXType { - ONNX_TYPE_UNKNOWN, - ONNX_TYPE_TENSOR, - ONNX_TYPE_SEQUENCE, - ONNX_TYPE_MAP, - ONNX_TYPE_OPAQUE, - ONNX_TYPE_SPARSETENSOR, - ONNX_TYPE_OPTIONAL -} ONNXType; - -// These types are synced with internal -// SparseFormatFlags -typedef enum OrtSparseFormat { - ORT_SPARSE_UNDEFINED = 0, - ORT_SPARSE_COO = 0x1, - ORT_SPARSE_CSRC = 0x2, - ORT_SPARSE_BLOCK_SPARSE = 0x4 -} OrtSparseFormat; - -// Enum allows to query sparse tensor indices -enum OrtSparseIndicesFormat { - ORT_SPARSE_COO_INDICES, - ORT_SPARSE_CSR_INNER_INDICES, - ORT_SPARSE_CSR_OUTER_INDICES, - ORT_SPARSE_BLOCK_SPARSE_INDICES -}; - -/** \brief Logging severity levels - * - * In typical API usage, specifying a logging severity level specifies the minimum severity of log messages to show. - */ -typedef enum OrtLoggingLevel { - ORT_LOGGING_LEVEL_VERBOSE, ///< Verbose informational messages (least severe). - ORT_LOGGING_LEVEL_INFO, ///< Informational messages. - ORT_LOGGING_LEVEL_WARNING, ///< Warning messages. - ORT_LOGGING_LEVEL_ERROR, ///< Error messages. - ORT_LOGGING_LEVEL_FATAL, ///< Fatal error messages (most severe). -} OrtLoggingLevel; - -typedef enum OrtErrorCode { - ORT_OK, - ORT_FAIL, - ORT_INVALID_ARGUMENT, - ORT_NO_SUCHFILE, - ORT_NO_MODEL, - ORT_ENGINE_ERROR, - ORT_RUNTIME_EXCEPTION, - ORT_INVALID_PROTOBUF, - ORT_MODEL_LOADED, - ORT_NOT_IMPLEMENTED, - ORT_INVALID_GRAPH, - ORT_EP_FAIL, -} OrtErrorCode; - -typedef enum OrtOpAttrType { - ORT_OP_ATTR_UNDEFINED = 0, - ORT_OP_ATTR_INT, - ORT_OP_ATTR_INTS, - ORT_OP_ATTR_FLOAT, - ORT_OP_ATTR_FLOATS, - ORT_OP_ATTR_STRING, - ORT_OP_ATTR_STRINGS, -} OrtOpAttrType; - -//! @} -#define ORT_RUNTIME_CLASS(X) \ - struct Ort##X; \ - typedef struct Ort##X Ort##X - -/** \addtogroup Global - * ONNX Runtime C API - * @{ - */ -// The actual types defined have an Ort prefix -ORT_RUNTIME_CLASS(Env); -ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success -ORT_RUNTIME_CLASS(MemoryInfo); -ORT_RUNTIME_CLASS(IoBinding); -ORT_RUNTIME_CLASS(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) -ORT_RUNTIME_CLASS(Value); -ORT_RUNTIME_CLASS(RunOptions); -ORT_RUNTIME_CLASS(TypeInfo); -ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); -ORT_RUNTIME_CLASS(MapTypeInfo); -ORT_RUNTIME_CLASS(SequenceTypeInfo); -ORT_RUNTIME_CLASS(OptionalTypeInfo); -ORT_RUNTIME_CLASS(SessionOptions); -ORT_RUNTIME_CLASS(CustomOpDomain); -ORT_RUNTIME_CLASS(ModelMetadata); -ORT_RUNTIME_CLASS(ThreadPoolParams); -ORT_RUNTIME_CLASS(ThreadingOptions); -ORT_RUNTIME_CLASS(ArenaCfg); -ORT_RUNTIME_CLASS(PrepackedWeightsContainer); -ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2); -ORT_RUNTIME_CLASS(CUDAProviderOptionsV2); -ORT_RUNTIME_CLASS(CANNProviderOptions); -ORT_RUNTIME_CLASS(DnnlProviderOptions); -ORT_RUNTIME_CLASS(Op); -ORT_RUNTIME_CLASS(OpAttr); -ORT_RUNTIME_CLASS(Logger); -ORT_RUNTIME_CLASS(ShapeInferContext); - -#ifdef _WIN32 -typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; -#else -typedef OrtStatus* OrtStatusPtr; -#endif - -/** \brief Memory allocation interface - * - * Structure of function pointers that defines a memory allocator. This can be created and filled in by the user for custom allocators. - * - * When an allocator is passed to any function, be sure that the allocator object is not destroyed until the last allocated object using it is freed. - */ -typedef struct OrtAllocator { - uint32_t version; ///< Must be initialized to ORT_API_VERSION - void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); ///< Returns a pointer to an allocated block of `size` bytes - void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); ///< Free a block of memory previously allocated with OrtAllocator::Alloc - const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); ///< Return a pointer to an ::OrtMemoryInfo that describes this allocator - /** - * @brief Optional allocation function to use for memory allocations made during session initialization. - * Use this function if you want to separate allocations made by ORT during Run() calls from - * those made during session initialization. This allows for separate memory management strategies for these allocations. - */ - void*(ORT_API_CALL* Reserve)(struct OrtAllocator* this_, size_t size); ///< Returns a pointer to an allocated block of `size` bytes -} OrtAllocator; - -typedef void(ORT_API_CALL* OrtLoggingFunction)( - void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, - const char* message); - -/** \brief Graph optimization level - * - * Refer to https://www.onnxruntime.ai/docs/performance/graph-optimizations.html#graph-optimization-levels - * for an in-depth understanding of the Graph Optimization Levels. - */ -typedef enum GraphOptimizationLevel { - ORT_DISABLE_ALL = 0, - ORT_ENABLE_BASIC = 1, - ORT_ENABLE_EXTENDED = 2, - ORT_ENABLE_ALL = 99 -} GraphOptimizationLevel; - -typedef enum ExecutionMode { - ORT_SEQUENTIAL = 0, - ORT_PARALLEL = 1, -} ExecutionMode; - -/** \brief Language projection identifiers - * /see OrtApi::SetLanguageProjection - */ -typedef enum OrtLanguageProjection { - ORT_PROJECTION_C = 0, - ORT_PROJECTION_CPLUSPLUS = 1, - ORT_PROJECTION_CSHARP = 2, - ORT_PROJECTION_PYTHON = 3, - ORT_PROJECTION_JAVA = 4, - ORT_PROJECTION_WINML = 5, - ORT_PROJECTION_NODEJS = 6, -} OrtLanguageProjection; - -struct OrtKernelInfo; -typedef struct OrtKernelInfo OrtKernelInfo; -struct OrtKernelContext; -typedef struct OrtKernelContext OrtKernelContext; -struct OrtCustomOp; -typedef struct OrtCustomOp OrtCustomOp; - -typedef enum OrtAllocatorType { - OrtInvalidAllocator = -1, - OrtDeviceAllocator = 0, - OrtArenaAllocator = 1 -} OrtAllocatorType; - -/** \brief Memory types for allocated memory, execution provider specific types should be extended in each provider. - */ -// Whenever this struct is updated, please also update the MakeKey function in onnxruntime / core / framework / execution_provider.cc -typedef enum OrtMemType { - OrtMemTypeCPUInput = -2, ///< Any CPU memory used by non-CPU execution provider - OrtMemTypeCPUOutput = -1, ///< CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeCPU = OrtMemTypeCPUOutput, ///< Temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeDefault = 0, ///< The default allocator for execution provider -} OrtMemType; - -/** \brief This mimics OrtDevice type constants so they can be returned in the API - */ -typedef enum OrtMemoryInfoDeviceType { - OrtMemoryInfoDeviceType_CPU = 0, - OrtMemoryInfoDeviceType_GPU = 1, - OrtMemoryInfoDeviceType_FPGA = 2 -} OrtMemoryInfoDeviceType; - -/** \brief Algorithm to use for cuDNN Convolution Op - */ -typedef enum OrtCudnnConvAlgoSearch { - OrtCudnnConvAlgoSearchExhaustive, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx - OrtCudnnConvAlgoSearchHeuristic, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7 - OrtCudnnConvAlgoSearchDefault, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM -} OrtCudnnConvAlgoSearch; - -/** \brief CUDA Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_CUDA - */ -typedef struct OrtCUDAProviderOptions { -#ifdef __cplusplus - OrtCUDAProviderOptions() - : device_id{}, - cudnn_conv_algo_search{OrtCudnnConvAlgoSearchExhaustive}, - gpu_mem_limit{SIZE_MAX}, - arena_extend_strategy{}, - do_copy_in_default_stream{1}, - has_user_compute_stream{}, - user_compute_stream{}, - default_memory_arena_cfg{}, - tunable_op_enable{false}, - tunable_op_tuning_enable{false}, - tunable_op_max_tuning_duration_ms{} {} -#endif - - /** \brief CUDA device Id - * Defaults to 0. - */ - int device_id; - - /** \brief CUDA Convolution algorithm search configuration. - * See enum OrtCudnnConvAlgoSearch for more details. - * Defaults to OrtCudnnConvAlgoSearchExhaustive. - */ - OrtCudnnConvAlgoSearch cudnn_conv_algo_search; - - /** \brief CUDA memory limit (To use all possible memory pass in maximum size_t) - * Defaults to SIZE_MAX. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - size_t gpu_mem_limit; - - /** \brief Strategy used to grow the memory arena - * 0 = kNextPowerOfTwo
- * 1 = kSameAsRequested
- * Defaults to 0. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - int arena_extend_strategy; - - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP - * 0 = Use separate streams for copying and compute. - * 1 = Use the same stream for copying and compute. - * Defaults to 1. - * WARNING: Setting this to 0 may result in data races for some models. - * Please see issue #4829 for more details. - */ - int do_copy_in_default_stream; - - /** \brief Flag indicating if there is a user provided compute stream - * Defaults to 0. - */ - int has_user_compute_stream; - - /** \brief User provided compute stream. - * If provided, please set `has_user_compute_stream` to 1. - */ - void* user_compute_stream; - - /** \brief CUDA memory arena configuration parameters - */ - OrtArenaCfg* default_memory_arena_cfg; - - /** \brief Enable TunableOp for using. - * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_CUDA_TUNABLE_OP_ENABLE. - */ - int tunable_op_enable; - - /** \brief Enable TunableOp for tuning. - * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_CUDA_TUNABLE_OP_TUNING_ENABLE. - */ - int tunable_op_tuning_enable; - - /** \brief Max tuning duration time limit for each instance of TunableOp. - * Defaults to 0 to disable the limit. - */ - int tunable_op_max_tuning_duration_ms; - -} OrtCUDAProviderOptions; - -/** \brief ROCM Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_ROCM - */ -typedef struct OrtROCMProviderOptions { -#ifdef __cplusplus - OrtROCMProviderOptions() - : device_id{}, - miopen_conv_exhaustive_search{0}, - gpu_mem_limit{SIZE_MAX}, - arena_extend_strategy{}, - do_copy_in_default_stream{1}, - has_user_compute_stream{}, - user_compute_stream{}, - default_memory_arena_cfg{}, - enable_hip_graph{false}, - tunable_op_enable{false}, - tunable_op_tuning_enable{false}, - tunable_op_max_tuning_duration_ms{} {} -#endif - - /** \brief ROCM device Id - * Defaults to 0. - */ - int device_id; - - /** \brief ROCM MIOpen Convolution algorithm exaustive search option. - * Defaults to 0 (false). - */ - int miopen_conv_exhaustive_search; - - /** \brief ROCM memory limit (To use all possible memory pass in maximum size_t) - * Defaults to SIZE_MAX. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - size_t gpu_mem_limit; - - /** \brief Strategy used to grow the memory arena - * 0 = kNextPowerOfTwo
- * 1 = kSameAsRequested
- * Defaults to 0. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - int arena_extend_strategy; - - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the ROCM EP - * 0 = Use separate streams for copying and compute. - * 1 = Use the same stream for copying and compute. - * Defaults to 1. - * WARNING: Setting this to 0 may result in data races for some models. - * Please see issue #4829 for more details. - */ - int do_copy_in_default_stream; - - /** \brief Flag indicating if there is a user provided compute stream - * Defaults to 0. - */ - int has_user_compute_stream; - - /** \brief User provided compute stream. - * If provided, please set `has_user_compute_stream` to 1. - */ - void* user_compute_stream; - - /** \brief ROCM memory arena configuration parameters - */ - OrtArenaCfg* default_memory_arena_cfg; - - int enable_hip_graph; - - /** \brief Enable TunableOp for using. - * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_ROCM_TUNABLE_OP_ENABLE. - */ - int tunable_op_enable; - - /** \brief Enable TunableOp for tuning. - * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. - * This option can be overriden by environment variable ORT_ROCM_TUNABLE_OP_TUNING_ENABLE. - */ - int tunable_op_tuning_enable; - - /** \brief Max tuning duration time limit for each instance of TunableOp. - * Defaults to 0 to disable the limit. - */ - int tunable_op_max_tuning_duration_ms; - -} OrtROCMProviderOptions; - -/** \brief TensorRT Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - */ -typedef struct OrtTensorRTProviderOptions { - int device_id; ///< CUDA device id (0 = default device) - int has_user_compute_stream; // indicator of user specified CUDA compute stream. - void* user_compute_stream; // user specified CUDA compute stream. - int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability - int trt_min_subgraph_size; // minimum size of TensorRT subgraphs - size_t trt_max_workspace_size; // maximum workspace size for TensorRT. - int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true - int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true - const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name. - int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true - int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true - int trt_dla_core; // DLA core number. Default 0 - int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true - int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true - const char* trt_engine_cache_path; // specify engine cache path - int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true - const char* trt_engine_decryption_lib_path; // specify engine decryption library path - int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true - // This is the legacy struct and don't add new fields here. - // For new field that can be represented by string, please add it in include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h - // For non-string field, need to create a new separate api to handle it. -} OrtTensorRTProviderOptions; - -/** \brief MIGraphX Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX - */ -typedef struct OrtMIGraphXProviderOptions { - int device_id; // hip device id. - int migraphx_fp16_enable; // MIGraphX FP16 precision. Default 0 = false, nonzero = true - int migraphx_int8_enable; // MIGraphX INT8 precision. Default 0 = false, nonzero = true - int migraphx_use_native_calibration_table; // MIGraphx INT8 cal table. Default 0 = false, noznero = true - const char* migraphx_int8_calibration_table_name; // MIGraphx INT8 calibration table name -} OrtMIGraphXProviderOptions; - -/** \brief OpenVINO Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO - */ -typedef struct OrtOpenVINOProviderOptions { -#ifdef __cplusplus - OrtOpenVINOProviderOptions() : device_type{}, - enable_npu_fast_compile{}, - device_id{}, - num_of_threads{}, - cache_dir{}, - context{}, - enable_opencl_throttling{}, - enable_dynamic_shapes{} {} -#endif - /** \brief Device type string - * - * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" - */ - const char* device_type; - unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled - const char* device_id; - size_t num_of_threads; ///< 0 = Use default number of threads - const char* cache_dir; // path is set to empty by default - void* context; - unsigned char enable_opencl_throttling; ///< 0 = disabled, nonzero = enabled - unsigned char enable_dynamic_shapes; ///< 0 = disabled, nonzero = enabled -} OrtOpenVINOProviderOptions; - -struct OrtApi; -typedef struct OrtApi OrtApi; - -struct OrtTrainingApi; -typedef struct OrtTrainingApi OrtTrainingApi; - -/** \brief The helper interface to get the right version of OrtApi - * - * Get a pointer to this structure through ::OrtGetApiBase - */ -struct OrtApiBase { - /** \brief Get a pointer to the requested version of the ::OrtApi - * - * \param[in] version Must be ::ORT_API_VERSION - * \return The ::OrtApi for the version requested, nullptr will be returned if this version is unsupported, for example when using a runtime - * older than the version created with this header file. - * - * One can call GetVersionString() to get the version of the Onnxruntime library for logging - * and error reporting purposes. - */ - const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; - - /** \brief Returns a null terminated string of the version of the Onnxruntime library (eg: "1.8.1") - * - * \return UTF-8 encoded version string. Do not deallocate the returned buffer. - */ - const char*(ORT_API_CALL* GetVersionString)(void)NO_EXCEPTION; -}; - -typedef struct OrtApiBase OrtApiBase; - -/** \brief The Onnxruntime library's entry point to access the C API - * - * Call this to get the a pointer to an ::OrtApiBase - */ -ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION; - -/** \brief Thread work loop function - * - * Onnxruntime will provide the working loop on custom thread creation - * Argument is an onnxruntime built-in type which will be provided when thread pool calls OrtCustomCreateThreadFn - */ -typedef void (*OrtThreadWorkerFn)(void* ort_worker_fn_param); - -typedef const struct OrtCustomHandleType { - char __place_holder; -}* OrtCustomThreadHandle; - -/** \brief Ort custom thread creation function - * - * The function should return a thread handle to be used in onnxruntime thread pools - * Onnxruntime will throw exception on return value of nullptr or 0, indicating that the function failed to create a thread - */ -typedef OrtCustomThreadHandle (*OrtCustomCreateThreadFn)(void* ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void* ort_worker_fn_param); - -/** \brief Custom thread join function - * - * Onnxruntime thread pool destructor will call the function to join a custom thread. - * Argument ort_custom_thread_handle is the value returned by OrtCustomCreateThreadFn - */ -typedef void (*OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle); - -typedef OrtStatus*(ORT_API_CALL* RegisterCustomOpsFn)(OrtSessionOptions* options, const OrtApiBase* api); - -/** \brief Callback function for RunAsync - * - * \param[in] user_data User specific data that passed back to the callback - * \param[out] outputs On succeed, outputs host inference results, on error, the value will be nullptr - * \param[out] num_outputs Number of outputs, on error, the value will be zero - * \param[out] status On error, status will provide details - */ -typedef void (*RunAsyncCallbackFn)(void* user_data, OrtValue** outputs, size_t num_outputs, OrtStatusPtr status); - -/** \brief The C API - * - * All C API functions are defined inside this structure as pointers to functions. - * Call OrtApiBase::GetApi to get a pointer to it - * - * \nosubgrouping - */ -struct OrtApi { - /// \name OrtStatus - /// @{ - - /** - * \brief Create an OrtStatus from a null terminated string - * - * \param[in] code - * \param[in] msg A null-terminated string. Its contents will be copied. - * \return A new OrtStatus object, must be destroyed with OrtApi::ReleaseStatus - */ - OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Get OrtErrorCode from OrtStatus - * - * \param[in] status - * \return OrtErrorCode that \p status was created with - */ - OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Get error string from OrtStatus - * - * \param[in] status - * \return The error message inside the `status`. Do not free the returned value. - */ - const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an OrtEnv - * - * \note Invoking this function will return the same instance of the environment as that returned by a previous call - * to another env creation function; all arguments to this function will be ignored. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnv, OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); - - /** \brief Create an OrtEnv - * - * \note Invoking this function will return the same instance of the environment as that returned by a previous call - * to another env creation function; all arguments to this function will be ignored. If you want to provide your - * own logging function, consider setting it using the SetUserLoggingFunction API instead. - * \param[in] logging_function A pointer to a logging function. - * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `logging_function`. This parameter is optional. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithCustomLogger, _In_ OrtLoggingFunction logging_function, _In_opt_ void* logger_param, - _In_ OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); - - /** \brief Enable Telemetry - * - * \note Telemetry events are on by default since they are lightweight - * \param[in] env - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env); - /** \brief Disable Telemetry - * - * \see OrtApi::EnableTelemetryEvents - * \param[in] env - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Create an OrtSession from a model file - * - * \param[in] env - * \param[in] model_path - * \param[in] options - * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - // TODO: document the path separator convention? '/' vs '\' - // TODO: should specify the access characteristics of model_path. Is this read only during the - // execution of CreateSession, or does the OrtSession retain a handle to the file/directory - // and continue to access throughout the OrtSession lifetime? - // What sort of access is needed to model_path : read or read/write? - ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); - - /** \brief Create an OrtSession from memory - * - * \param[in] env - * \param[in] model_data - * \param[in] model_data_length - * \param[in] options - * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); - - /** \brief Run the model in an ::OrtSession - * - * Will not return until the model run has completed. Multiple threads might be used to run the model based on - * the options in the ::OrtSession and settings used when creating the ::OrtEnv - * - * \param[in] session - * \param[in] run_options If nullptr, will use a default ::OrtRunOptions - * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names - * \param[in] inputs Array of ::OrtValue%s of the input values - * \param[in] input_len Number of elements in the input_names and inputs arrays - * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names - * \param[in] output_names_len Number of elements in the output_names and outputs array - * \param[out] outputs Array of ::OrtValue%s that the outputs are stored in. This can also be - * an array of nullptr values, in this case ::OrtValue objects will be allocated and pointers - * to them will be set into the `outputs` array. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(Run, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, - _In_reads_(input_len) const char* const* input_names, - _In_reads_(input_len) const OrtValue* const* inputs, size_t input_len, - _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, - _Inout_updates_all_(output_names_len) OrtValue** outputs); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Create an ::OrtSessionOptions object - * - * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these - * functions to enable them in the session:
- * OrtSessionOptionsAppendExecutionProvider_CPU
- * OrtSessionOptionsAppendExecutionProvider_CUDA
- * OrtSessionOptionsAppendExecutionProvider_(remaining providers...)
- * The order they are called indicates the preference order as well. In other words call this method - * on your most preferred execution provider first followed by the less preferred ones. - * If none are called Ort will use its internal CPU execution provider. - * - * \param[out] options The newly created OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options); - - /** \brief Set filepath to save optimized model after graph level transformations - * - * \param[in] options - * \param[in] optimized_model_filepath - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, - _In_ const ORTCHAR_T* optimized_model_filepath); - - /** \brief Create a copy of an existing ::OrtSessionOptions - * - * \param[in] in_options OrtSessionOptions to copy - * \param[out] out_options Returned newly created ::OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options, - _Outptr_ OrtSessionOptions** out_options); - - /** \brief Set execution mode - * - * Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model - * has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance. - * See [docs/ONNX_Runtime_Perf_Tuning.md] for more details. - * - * \param[in] options - * \param[in] execution_mode - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode); - - /** \brief Enable profiling for a session - * - * \param[in] options - * \param[in] profile_file_prefix - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix); - - /** \brief Disable profiling for a session - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options); - - /** \brief Enable the memory pattern optimization - * - * The idea is if the input shapes are the same, we could trace the internal memory allocation - * and generate a memory pattern for future request. So next time we could just do one allocation - * with a big chunk for all the internal memory allocation. - * \note Memory pattern optimization is only available when Sequential Execution mode is enabled (see OrtApi::SetSessionExecutionMode) - * - * \see OrtApi::DisableMemPattern - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options); - - /** \brief Disable the memory pattern optimization - * - * \see OrtApi::EnableMemPattern - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options); - - /** \brief Enable the memory arena on CPU - * - * Arena may pre-allocate memory for future usage. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options); - - /** \brief Disable the memory arena on CPU - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options); - - /** \brief Set session log id - * - * \param[in] options - * \param[in] logid The log identifier. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid); - - /** \brief Set session log verbosity level - * - * Applies to session load, initialization, etc - * - * \param[in] options - * \param[in] session_log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level); - - /** \brief Set session log severity level - * - * \param[in] options - * \param[in] session_log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level); - - /** \brief Set the optimization level to apply when loading a graph - * - * Please see https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html for an in-depth explanation - * \param[in,out] options The session options object - * \param[in] graph_optimization_level The optimization level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, - GraphOptimizationLevel graph_optimization_level); - - /** \brief Sets the number of threads used to parallelize the execution within nodes - * - * When running a single node operation, ex. add, this sets the maximum number of threads to use. - * - * \note If built with OpenMP, this has no effect on the number of threads used. In this case - * use the OpenMP env variables to configure the number of intra op num threads. - * - * \param[in] options - * \param[in] intra_op_num_threads Number of threads to use
- * A value of 0 will use the default number of threads
- * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads); - - /** \brief Sets the number of threads used to parallelize the execution of the graph - * - * If nodes can be run in parallel, this sets the maximum number of threads to use to run them in parallel. - * - * \note If sequential execution is enabled this value is ignored, it acts as if it was set to 1. - * - * \param[in] options - * \param[in] inter_op_num_threads Number of threads to use
- * A value of 0 will use the default number of threads
- * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads); - - /// @} - /// \name OrtCustomOpDomain - /// @{ - - /** \brief Create a custom op domain - * - * \param[in] domain - * \param[out] out Newly created domain. Must be freed with OrtApi::ReleaseCustomOpDomain - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out); - - /** \brief Add a custom op to a custom op domain - * - * \note The OrtCustomOp* pointer must remain valid until the ::OrtCustomOpDomain using it is released - * - * \param[in] custom_op_domain - * \param[in] op - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Add custom op domain to a session options - * - * \note The OrtCustomOpDomain* must not be deleted until all sessions using it are released - * - * \param[in] options - * \param[in] custom_op_domain - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain); - - /** \deprecated Use OrtApi::RegisterCustomOpsLibrary_V2. - * - * Registers custom ops from a shared library. - * - * Loads a shared library (dll on windows, so on linux, etc) named 'library_path' and looks for this entry point: - * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); - * It then passes in the provided session options to this function along with the api base. - * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in - * session options are destroyed, or if an error occurs and it is non null. - * - * \param[in] options - * \param[in] library_path - * \param[out] library_handle OS specific handle to the loaded library (Use FreeLibrary on Windows, dlclose on Linux, etc.. to unload) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path, _Outptr_ void** library_handle); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Get input count for a session - * - * This number must also match the number of inputs passed to OrtApi::Run - * - * \see OrtApi::SessionGetInputTypeInfo, OrtApi::SessionGetInputName, OrtApi::Session - * - * \param[in] session - * \param[out] out Number of inputs - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get output count for a session - * - * This number must also match the number of outputs returned by OrtApi::Run - * - * \see OrtApi::SessionGetOutputTypeInfo, OrtApi::SessionGetOutputName, OrtApi::Session - * - * \param[in] session - * \param[out] out Number of outputs - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get overridable initializer count - * - * \see OrtApi::SessionGetOverridableInitializerTypeInfo, OrtApi::SessionGetOverridableInitializerName - * - * \param[in] session - * \param[in] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get input type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get output type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get overridable initializer type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get input name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get output name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get overridable initializer name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* session, size_t index, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /// @} - /// \name OrtRunOptions - /// @{ - - /** \brief Create an OrtRunOptions - * - * \param[out] out Returned newly created ::OrtRunOptions. Must be freed with OrtApi::ReleaseRunOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out); - - /** \brief Set per-run log verbosity level - * - * \see OrtApi::RunOptionsGetRunLogVerbosityLevel - * - * \param[in] options - * \param[in] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int log_verbosity_level); - - /** \brief Set per-run log severity level - * - * \see OrtApi::RunOptionsGetRunLogSeverityLevel - * - * \param[in] options - * \param[in] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - */ - ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int log_severity_level); - - /** \brief Set per-run tag - * - * This is used in a per-run log identifier. - * - * \see OrtApi::RunOptionsGetRunTag - * - * \param[in] options - * \param[in] run_tag The run tag. - */ - ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions* options, _In_ const char* run_tag); - - /** \brief Get per-run log verbosity level - * - * \see OrtApi::RunOptionsSetRunLogVerbosityLevel - * - * \param[in] options - * \param[out] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, - _Out_ int* log_verbosity_level); - - /** \brief Get per-run log severity level - * - * \see OrtApi::RunOptionsSetRunLogSeverityLevel - * - * \param[in] options - * \param[out] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - */ - ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* log_severity_level); - - /** \brief Get per-run tag - * - * This is used in a per-run log identifier. - * - * \see OrtApi::RunOptionsSetRunTag - * - * \param[in] options - * \param[out] run_tag The run tag. - * Do not free this value, it is owned by `options`. It will be invalidated if the run tag - * changes (i.e., with OrtApi::RunOptionsSetRunTag) or `options` is freed. - */ - ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions* options, _Out_ const char** run_tag); - - /** \brief Set terminate flag - * - * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options); - - /** \brief Clears the terminate flag - * - * Used so the OrtRunOptions instance can be used in a new OrtApi::Run call without it instantly terminating - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Create a tensor - * - * Create a tensor using a supplied ::OrtAllocator - * - * \param[in] allocator - * \param[in] shape Pointer to the tensor shape dimensions. - * \param[in] shape_len The number of tensor shape dimensions. - * \param[in] type - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** \brief Create a tensor backed by a user supplied buffer - * - * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. - * p_data is owned by caller. ReleaseValue won't release p_data. - * - * \param[in] info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param[in] p_data Pointer to the data buffer. - * \param[in] p_data_len The number of bytes in the data buffer. - * \param[in] shape Pointer to the tensor shape dimensions. - * \param[in] shape_len The number of tensor shape dimensions. - * \param[in] type The data type. - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, - size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type, - _Outptr_ OrtValue** out); - - /** \brief Return if an ::OrtValue is a tensor type - * - * \param[in] value A tensor type (string tensors are not supported) - * \param[out] out Set to 1 iff ::OrtValue is a tensor, 0 otherwise - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out); - - /** \brief Get a pointer to the raw data inside a tensor - * - * Used to read/write/modify the internal tensor data directly. - * \note The returned pointer is valid until the \p value is destroyed. - * - * \param[in] value A tensor type (string tensors are not supported) - * \param[out] out Filled in with a pointer to the internal storage - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorMutableData, _In_ OrtValue* value, _Outptr_ void** out); - - /** \brief Set all strings at once in a string tensor - * - * \param[in,out] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[in] s An array of strings. Each string in this array must be null terminated. - * \param[in] s_len Count of strings in s (Must match the size of \p value's tensor shape) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len); - - /** \brief Get total byte length for all strings in a string tensor - * - * Typically used with OrtApi::GetStringTensorContent - * - * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[out] len Total byte length of all strings (does not include trailing nulls) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); - - /** \brief Get all strings from a string tensor - * - * An example of the results:
- * Given \p value is a string tensor with the strings { "This" "is" "a" "test" }
- * \p s must have a size of 11 bytes
- * \p offsets must have 4 elements
- * After the call, these values will be filled in:
- * \p s will contain "Thisisatest"
- * \p offsets will contain { 0, 4, 6, 7 }
- * The length of the last string is just s_len - offsets[last] - * - * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[in] s Buffer to sequentially write all tensor strings to. Each string is NOT null-terminated. - * \param[in] s_len Number of bytes of buffer pointed to by \p s (Get it from OrtApi::GetStringTensorDataLength) - * \param[out] offsets Array of start offsets into the strings written to \p s - * \param[in] offsets_len Number of elements in offsets - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s, - size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len); - - /// @} - /// \name OrtTypeInfo - /// @{ - - /** \brief Get ::OrtTensorTypeAndShapeInfo from an ::OrtTypeInfo - * - * \param[in] type_info - * \param[out] out Do not free this value, it will be valid until type_info is freed. - * If type_info does not represent tensor, this value will be set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out); - - /** \brief Get ::ONNXType from ::OrtTypeInfo - * - * \param[in] type_info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ enum ONNXType* out); - - /// @} - /// \name OrtTensorTypeAndShapeInfo - /// @{ - - /** \brief Create an ::OrtTensorTypeAndShapeInfo object - * - * \param[out] out Returns newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Set element type in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] type - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo* info, enum ONNXTensorElementDataType type); - - /** \brief Set shape information in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] dim_values Array with `dim_count` elements. Can contain negative values. - * \param[in] dim_count Number of elements in `dim_values` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); - - /** \brief Get element type in ::OrtTensorTypeAndShapeInfo - * - * \see OrtApi::SetTensorElementType - * - * \param[in] info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo* info, - _Out_ enum ONNXTensorElementDataType* out); - - /** \brief Get dimension count in ::OrtTensorTypeAndShapeInfo - * - * \see OrtApi::GetDimensions - * - * \param[in] info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); - - /** \brief Get dimensions in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[out] dim_values Array with `dim_values_length` elements. On return, filled with the dimensions stored in the ::OrtTensorTypeAndShapeInfo - * \param[in] dim_values_length Number of elements in `dim_values`. Use OrtApi::GetDimensionsCount to get this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, - size_t dim_values_length); - - /** \brief Get symbolic dimension names in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] dim_params Array with `dim_params_length` elements. On return filled with pointers to null terminated strings of the dimension names - * \param[in] dim_params_length Number of elements in `dim_params`. Use OrtApi::GetDimensionsCount to get this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, - _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length); - - /** \brief Get total number of elements in a tensor shape from an ::OrtTensorTypeAndShapeInfo - * - * Return the number of elements specified by the tensor shape (all dimensions multiplied by each other). - * For 0 dimensions, 1 is returned. If any dimension is less than 0, the result is always -1. - * - * Examples:
- * [] = 1
- * [1,3,4] = 12
- * [2,0,4] = 0
- * [-1,3,4] = -1
- * - * \param[in] info - * \param[out] out Number of elements - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Get type and shape information from a tensor ::OrtValue - * - * \param[in] value Must be a tensor (not a map/sequence/etc) or will return failure - * \param[out] out Newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Get type information of an OrtValue - * - * \param[in] value - * \param[out] out Newly created ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out); - - /** \brief Get ONNXType of an ::OrtValue - * - * \param[in] value - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out); - - /// @} - /// \name OrtMemoryInfo - /// @{ - - /** \brief Create an ::OrtMemoryInfo - * - * \param[in] name - * \param[in] type - * \param[in] id - * \param[in] mem_type - * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtAPi::ReleaseMemoryInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name, enum OrtAllocatorType type, int id, - enum OrtMemType mem_type, _Outptr_ OrtMemoryInfo** out); - - /** \brief Create an ::OrtMemoryInfo for CPU memory - * - * Special case version of OrtApi::CreateMemoryInfo for CPU based memory. Same as using OrtApi::CreateMemoryInfo with name = "Cpu" and id = 0. - * - * \param[in] type - * \param[in] mem_type - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type, - _Outptr_ OrtMemoryInfo** out); - - /** \brief Compare ::OrtMemoryInfo objects for equality - * - * Compares all settings of each ::OrtMemoryInfo for equality - * - * \param[in] info1 - * \param[in] info2 - * \param[out] out Set to 0 if equal, -1 if not equal - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out); - - /** \brief Get name from ::OrtMemoryInfo - * - * \param[in] ptr - * \param[out] out Writes null terminated string to this pointer. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtMemoryInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out); - - /** \brief Get the id from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out); - - /** \brief Get the ::OrtMemType from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out); - - /** \brief Get the ::OrtAllocatorType from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out); - - /// @} - /// \name OrtAllocator - /// @{ - - /// \brief Calls OrtAllocator::Alloc function - ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ort_allocator, size_t size, _Outptr_ void** out); - /// \brief Calls OrtAllocator::Free function - ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ort_allocator, void* p); - /// \brief Calls OrtAllocator::Info function - ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ort_allocator, _Outptr_ const struct OrtMemoryInfo** out); - - /** \brief Get the default allocator - * - * The default allocator is a CPU based, non-arena. Always returns the same pointer to the same default allocator. - * - * \param[out] out Returned value should NOT be freed - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Override session symbolic dimensions - * - * Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable - * optimizations that can take advantage of fixed values (such as memory planning, etc) - * - * \param[in] options - * \param[in] dim_denotation - * \param[in] dim_value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation, - _In_ int64_t dim_value); - - /// @} - /// \name OrtValue - /// @{ - - /* Internal information (not seen in Doxygen) - * - * APIs to support non-tensor types - map and sequence. - * Currently only the following types are supported - * Note: the following types should be kept in sync with data_types.h - * Map types - * ========= - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * - * Sequence types - * ============== - * std::vector - * std::vector - * std::vector - * std::vector - * std::vector> - * std::vector - */ - - /** \brief Get non tensor data from an ::OrtValue - * - * If `value` is of type ONNX_TYPE_MAP, you need to retrieve the keys and values - * separately. Use index=0 to retrieve keys and index=1 to retrieve values. - * If `value` is of type ONNX_TYPE_SEQUENCE, use index to retrieve the index'th element - * of the sequence. - * - * \param[in] value - * \param[in] index See above for usage based on `value` type - * \param[in] allocator Allocator used to allocate ::OrtValue - * \param[out] out Created ::OrtValue that holds the element requested. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator, - _Outptr_ OrtValue** out); - - /** \brief Get non tensor value count from an ::OrtValue - * - * If `value` is of type ONNX_TYPE_MAP 2 will always be returned. For ONNX_TYPE_SEQUENCE - * the number of elements in the sequence will be returned - * - * \param[in] value - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out); - - /** \brief Create a map or sequence ::OrtValue - * - * To construct a map (ONNX_TYPE_MAP), use num_values = 2 and `in` should be an array of 2 ::OrtValue%s - * representing keys and values.
- * - * To construct a sequence (ONNX_TYPE_SEQUENCE), use num_values = N where N is the number of the elements in the - * sequence. 'in' should be an array of N ::OrtValue%s. - * - * \param[in] in See above for details - * \param[in] num_values - * \param[in] value_type Must be either ONNX_TYPE_MAP or ONNX_TYPE_SEQUENCE - * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values, - enum ONNXType value_type, _Outptr_ OrtValue** out); - - /** \brief Create an opaque (custom user defined type) ::OrtValue - * - * Constructs an ::OrtValue that contains a value of non-standard type created for - * experiments or while awaiting standardization. ::OrtValue in this case would contain - * an internal representation of the Opaque type. Opaque types are distinguished from - * each other by two strings 1) domain and 2) type name. The combination of the two - * must be unique, so the type representation is properly identified internally. The combination - * must be properly registered from within ORT at both compile/run time or by another API. - * - * To construct the ::OrtValue pass domain and type names, also a pointer to a data container - * the type of which must be known to both ORT and the client program. That data container may or may - * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for - * verification purposes. - * - * \param[in] domain_name Null terminated string of the domain name - * \param[in] type_name Null terminated string of the type name - * \param[in] data_container User pointer Data to populate ::OrtValue - * \param[in] data_container_size Size in bytes of what `data_container` points to - * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name, - _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out); - - /** \brief Get internal data from an opaque (custom user defined type) ::OrtValue - * - * Copies internal data from an opaque value into a user provided buffer - * - * \see OrtApi::CreateOpaqueValue - * - * \param[in] domain_name Null terminated string of the domain name - * \param[in] type_name Null terminated string of the type name - * \param[in] in The opaque ::OrtValue - * \param[out] data_container Buffer to copy data into - * \param[out] data_container_size Size in bytes of the buffer pointed to by data_container. Must match the size of the internal buffer. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in, - _Out_ void* data_container, size_t data_container_size); - - /// @} - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Get a float stored as an attribute in the graph node - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ float* out); - - /** \brief Fetch a 64-bit int stored as an attribute in the graph node - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ int64_t* out); - - /** \brief Fetch a string stored as an attribute in the graph node - * - * If `out` is nullptr, the value of `size` is set to the true size of the string - * attribute, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual string attribute's size, - * the value of `size` is set to the true size of the string attribute, the provided memory - * is filled with the attribute's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string attribute's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string attribute - * and a failure status is returned.) - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * \param[in,out] size See above comments for details - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out, - _Inout_ size_t* size); - - /// @} - /// \name OrtKernelContext - /// Custom operator APIs. - /// @{ - - /** \brief Used for custom operators, get the input count of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); - - /** \brief Used for custom operators, get the output count of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); - - /** \brief Used for custom operators, get an input of a kernel - * - * The function attempts fetches the input of the kernel. If the input is optional - * and not present, the function returns success and out is set to nullptr. - * - * \param[in] context ::OrtKernelContext instance - * \param[in] input index. See KernelContext_GetInputCount for boundaries check. - * \param[in, out] returns a ptr to OrtValue if the input is present - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index, - _Out_ const OrtValue** out); - - /** \brief Used for custom operators, get an output of a kernel - * - * The function attempts fetches the output of the kernel. If the output is optional - * and not present, the function returns success and out is set to nullptr. - * - * \param[in] context ::OrtKernelContext instance - * \param[in] output index. See KernelContext_GetOutputCount for boundaries check. - * \param[in, out] returns a ptr to OrtValue if the output is present - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index, - _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out); - - /// @} - /// \name OrtEnv - /// @{ - ORT_CLASS_RELEASE(Env); - /// @} - /// \name OrtStatus - /// @{ - ORT_CLASS_RELEASE(Status); - /// @} - /// \name OrtMemoryInfo - /// @{ - ORT_CLASS_RELEASE(MemoryInfo); - /// @} - /// \name OrtSession - /// @{ - ORT_CLASS_RELEASE(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) - /// @} - /// \name OrtValue - /// @{ - ORT_CLASS_RELEASE(Value); - /// @} - /// \name OrtRunOptions - /// @{ - ORT_CLASS_RELEASE(RunOptions); - /// @} - /// \name OrtTypeInfo - /// @{ - ORT_CLASS_RELEASE(TypeInfo); - /// @} - /// \name OrtTensorTypeAndShapeInfo - /// @{ - ORT_CLASS_RELEASE(TensorTypeAndShapeInfo); - /// @} - /// \name OrtSessionOptions - /// @{ - ORT_CLASS_RELEASE(SessionOptions); - /// @} - /// \name OrtCustomOpDomain - /// @{ - ORT_CLASS_RELEASE(CustomOpDomain); - - /// @} - /// \name OrtTypeInfo - /// @{ - - /** \brief Get denotation from type information - * - * Augments ::OrtTypeInfo to return denotations on the type. - * - * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor. - * - * \param[in] type_info - * \param[out] denotation Pointer to the null terminated denotation string is written to this pointer. This pointer is valid until the object is destroyed or the name is changed, do not free. - * \param[out] len Length in bytes of the string returned in `denotation` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ const char** const denotation, - _Out_ size_t* len); - - /** \brief Get detailed map information from an ::OrtTypeInfo - * - * This augments ::OrtTypeInfo to return an ::OrtMapTypeInfo when the type is a map. - * The OrtMapTypeInfo has additional information about the map's key type and value type. - * - * This is used by WinML to support model reflection APIs. - * - * \param[out] type_info - * \param[out] out A pointer to the ::OrtMapTypeInfo. Do not free this value. If type_info - * does not contain a map, this value will be set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtMapTypeInfo** out); - - /** \brief Cast ::OrtTypeInfo to an ::OrtSequenceTypeInfo - * - * This api augments ::OrtTypeInfo to return an ::OrtSequenceTypeInfo when the type is a sequence. - * The ::OrtSequenceTypeInfo has additional information about the sequence's element type. - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] type_info - * \param[out] out A pointer to the OrtSequenceTypeInfo. Do not free this value. If type_info - * doesn not contain a sequence, this value will be set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out); - - /// @} - /// \name OrtMapTypeInfo - /// @{ - - /** \brief Get key type from an ::OrtMapTypeInfo - * - * Key types are restricted to being scalar types. - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] map_type_info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out); - - /** \brief Get the value type from an ::OrtMapTypeInfo - * - * \param[in] map_type_info - * \param[out] type_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info); - - /// @} - /// \name OrtSequenceTypeInfo - /// @{ - - /** \brief Get element type from an ::OrtSequenceTypeInfo - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] sequence_type_info - * \param[out] type_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info, - _Outptr_ OrtTypeInfo** type_info); - - /// @} - /// \name OrtMapTypeInfo - /// @{ - ORT_CLASS_RELEASE(MapTypeInfo); - /// @} - /// \name OrtSequenceTypeInfo - /// @{ - ORT_CLASS_RELEASE(SequenceTypeInfo); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief End profiling and return filename of the profile data - * - * Profiling is turned on through OrtApi::EnableProfiling - * - * \param[in] session - * \param[in] allocator - * \param[out] out Null terminated string of the filename, allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* session, _Inout_ OrtAllocator* allocator, _Outptr_ char** out); - - /** \brief Get ::OrtModelMetadata from an ::OrtSession - * - * \param[in] session - * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* session, _Outptr_ OrtModelMetadata** out); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** \brief Get `producer name` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get `graph name` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get `domain` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator, - _Outptr_ char** value); - - /** \brief Get `description` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Return data for a key in the custom metadata map in an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[in] key Null terminated string - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * `value` will be set to nullptr if the given key is not found in the custom metadata map. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value); - - /** \brief Get version number from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[out] value Set to the version number - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value); - - ORT_CLASS_RELEASE(ModelMetadata); - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an OrtEnv - * - * Create an environment with global threadpools that will be shared across sessions. - * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use - * its own thread pools. - * - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[in] tp_options - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel log_severity_level, _In_ const char* logid, - _In_ const OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Use global thread pool on a session - * - * Disable using per session thread pool and use the shared global threadpool. - * This should be used in conjunction with OrtApi::CreateEnvWithGlobalThreadPools. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Create an ::OrtThreadingOptions - * - * \param[out] out Newly created ::OrtThreadingOptions. Must be freed with OrtApi::ReleaseThreadingOptions - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out); - - ORT_CLASS_RELEASE(ThreadingOptions); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] keys Array of null terminated strings (array count = num_keys) allocated using `allocator`. - * The strings and the pointer array must be freed using `allocator` - * `keys` will be set to nullptr if the custom metadata map is empty. - * \param[out] num_keys Set to the number of elements in the `keys` array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** - * - * Override symbolic dimensions (by specific name strings) with actual values - * if known at session initialization time to enable optimizations that can - * take advantage of fixed values (such as memory planning, etc) - * - */ - ORT_API2_STATUS(AddFreeDimensionOverrideByName, - _Inout_ OrtSessionOptions* options, _In_ const char* dim_name, - _In_ int64_t dim_value); - - /// @} - /// \name Misc - /// @{ - - /** \brief Get the names of all available providers - * - * \note The providers in the list are not guaranteed to be usable. They may fail to load due to missing system dependencies. - * For example, if the CUDA/cuDNN libraries are not installed, the CUDA provider will report an error when it is added to the session options. - * - * \param[out] out_ptr Set to a pointer to an array of null terminated strings of the available providers. The entries and the - * array itself must be freed using OrtApi::ReleaseAvailableProviders - * \param[out] provider_length Set to the number of entries in the `out_ptr` array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr, _Out_ int* provider_length); - - /** \brief Release data from OrtApi::GetAvailableProviders. This API will never fail - * so you can rely on it in a noexcept code. - * - * \param[in] ptr The `out_ptr` result from OrtApi::GetAvailableProviders. - * \param[in] providers_length The `provider_length` result from OrtApi::GetAvailableProviders - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr, - _In_ int providers_length); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Get the length of a single string in a string tensor - * - * \param[in] value A string tensor - * \param[in] index Index of the string in the tensor - * \param[out] out Set to number of bytes of the string element - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out); - - /** \brief Get a single string from a string tensor - * - * \param[in] value A string tensor - * \param[in] s_len Number of bytes in the `s` buffer. Must match the value returned by OrtApi::GetStringTensorElementLength. - * \param[in] index Index of the string in the tensor - * \param[out] s The string element contents in UTF-8 encoding. The string is NOT null-terminated. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s); - - /** \brief Set a single string in a string tensor - * - * \param[in] value A string tensor - * \param[in] s A null terminated UTF-8 encoded string - * \param[in] index Index of the string in the tensor to set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Set a session configuration entry as a pair of strings - * - * If a configuration with same key exists, this will overwrite the configuration with the given config_value. - * - * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h - * - * \param[in] options - * \param[in] config_key A null terminated string representation of the config key - * \param[in] config_value A null terminated string representation of the config value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options, - _In_z_ const char* config_key, _In_z_ const char* config_value); - - /// @} - /// \name OrtAllocator - /// @{ - - /** \brief Create an allocator for an ::OrtSession following an ::OrtMemoryInfo - * - * \param[in] session - * \param[in] mem_info valid ::OrtMemoryInfo instance - * \param[out] out Newly created ::OrtAllocator. Must be freed with OrtApi::ReleaseAllocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* session, _In_ const OrtMemoryInfo* mem_info, - _Outptr_ OrtAllocator** out); - - /** \brief Release an ::OrtAllocator obtained from OrtApi::CreateAllocator - */ - ORT_CLASS_RELEASE(Allocator); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Run a model using Io Bindings for the inputs & outputs - * - * \see OrtApi::Run - * - * \param[in] session - * \param[in] run_options - * \param[in] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* session, _In_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr); - - /** \brief Create an ::OrtIoBinding instance - * - * An IoBinding object allows one to bind pre-allocated ::OrtValue%s to input names. - * Thus if you want to use a raw on device buffer as input or output you can avoid - * extra copy during runtime. - * - * \param[in] session - * \param[out] out Newly created ::OrtIoBinding. Must be freed with OrtApi::ReleaseIoBinding - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* session, _Outptr_ OrtIoBinding** out); - - /// @} - /// \name OrtIoBinding - /// @{ - - /** \brief Release an ::OrtIoBinding obtained from OrtApi::CreateIoBinding - */ - ORT_CLASS_RELEASE(IoBinding); - - /** \brief Bind an ::OrtValue to an ::OrtIoBinding input - * - * When using OrtApi::RunWithBinding this value is used for the named input - * - * \param[in] binding_ptr - * \param[in] name Name for the model input - * \param[in] val_ptr ::OrtValue of Tensor type. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); - - /** \brief Bind an ::OrtValue to an ::OrtIoBinding output - * - * When using OrtApi::RunWithBinding this value is used for the named output - * - * \param[in] binding_ptr - * \param[in] name Null terminated string of the model output name - * \param[in] val_ptr ::OrtValue of Tensor type. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); - - /** \brief Bind an ::OrtIoBinding output to a device - * - * Binds the ::OrtValue to a device which is specified by ::OrtMemoryInfo. - * You can either create an instance of ::OrtMemoryInfo with a device id or obtain one from the allocator that you have created/are using - * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocate and bind a chunk of - * memory within ::OrtValue ahead of time. - * - * \see OrtApi::RunWithBinding - * - * \param[in] binding_ptr - * \param[in] name Null terminated string of the device name - * \param[in] mem_info_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* mem_info_ptr); - - /** \brief Get the names of an ::OrtIoBinding's outputs - * - * Returns the names of the outputs in the order they were bound. This is useful after running the model - * with bound outputs because the returned names are in order in which output ::OrtValue are returned. This is useful if - * the order of outputs and their names is not known. - * - * \param[in] binding_ptr - * \param[in] allocator Allocator used to allocate continuous buffers for output strings and lengths. - * \param[out] buffer Returns an array of non-null terminated UTF-8 strings. The number of strings stored is returned in the count parameter. - * This buffer is allocated using `allocator` and must be freed using it. - * \param[out] lengths Returns an array of `count` lengths of the strings returned in `buffer` - * This buffer is allocated using `allocator` and must be freed using it. - * \param[out] count Number of strings returned. If `binding_ptr` has no bound outputs, zero is returned, - * no memory allocation is performed and buffer and lengths are set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, - _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count); - - /** \brief Get the output ::OrtValue objects from an ::OrtIoBinding - * - * Returns an array of pointers to individually allocated ::OrtValue%s that contain results of a model execution with OrtApi::RunWithBinding - * The array contains the same number of ::OrtValue%s and they are in the same order as they were bound with OrtApi::BindOutput - * or OrtApi::BindOutputToDevice. - * - * The returned ::OrtValue%s must be released using OrtApi::ReleaseValue after they are no longer needed. - * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after - * all the ::OrtValue%s contained therein are individually released. - * - * \param[in] binding_ptr - * \param[in] allocator Allocator used to allocate output array - * \param[out] output Set to the allocated array of allocated ::OrtValue outputs. Set to nullptr if there are 0 outputs. - * \param[out] output_count Set to number of ::OrtValue%s returned - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, - _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count); - - /** \brief Clears any previously set Inputs for an ::OrtIoBinding - */ - void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Clears any previously set Outputs for an ::OrtIoBinding - */ - void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Direct memory access to a specified tensor element - * - * For example, given a tensor with shape of [3,224,224], a pointer to the element at location [2,150,128] can be retrieved - * - * This function only works for numeric type tensors (No strings, etc). - * This is a no-copy method whose returned pointer is valid until the passed in ::OrtValue is free'd. - * - * \param[in] value - * \param[in] location_values Pointer to an array of index values that specify an element's location relative to its shape - * \param[in] location_values_count Number of elements in location_values. Must match the number of elements in the tensor's shape. - * \param[out] out Set to a pointer to the element specified - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out); - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an allocator and register it with the ::OrtEnv - * - * Enables sharing the allocator between multiple sessions that use the same env instance. - * Lifetime of the created allocator will be valid for the duration of the environment. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * - * See https://onnxruntime.ai/docs/get-started/with-c.html for details. - * - * \param[in] env ::OrtEnv instance - * \param[in] mem_info - * \param[in] arena_cfg Pass nullptr for defaults - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, - _In_ const OrtArenaCfg* arena_cfg); - - /** \brief Set language projection - * - * Set the language projection for collecting telemetry data when Env is created. - * - * The default is ORT_PROJECTION_C, which means it will classify the language not in the list to C also. - * - * \param[in] ort_env - * \param[in] projection - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Return the time that profiling was started - * - * \note The timer precision varies per platform. On Windows and MacOS, the precision will be ~100ns - * - * \param[in] session - * \param[out] out nanoseconds of profiling's start time - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* session, _Outptr_ uint64_t* out); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Set global intra-op thread count - * - * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools - * - * \param[in] tp_options - * \param[in] intra_op_num_threads Number of threads, special values:
- * 0 = Use default thread count
- * 1 = The invoking thread will be used; no threads will be created in the thread pool. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads); - - /** \brief Set global inter-op thread count - * - * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools - * - * \param[in] tp_options - * \param[in] inter_op_num_threads Number of threads, special values:
- * 0 = Use default thread count
- * 1 = The invoking thread will be used; no threads will be created in the thread pool. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads); - - /** \brief Set global spin control options - * - * This will configure the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. - * Allow spinning of thread pools when their queues are empty. This will set the value for both - * inter_op and intra_op threadpools. - * - * \param[in] tp_options - * \param[in] allow_spinning Valid values are 0 or 1.
- * 0 = It won't spin (recommended if CPU usage is high)
- * 1 = Threadpool will spin to wait for queue to become non-empty - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Add a pre-allocated initializer to a session - * - * If a model contains an initializer with a name that is same as the name passed to this call, - * ORT will use this initializer instance instead of deserializing one from the model file. This - * is useful when you want to share the same initializer across sessions. - * - * \param[in] options - * \param[in] name Null terminated string of the initializer name - * \param[in] val ::OrtValue containing the initializer. Its lifetime and the underlying initializer buffer must be - * managed by the user (created using the OrtApi::CreateTensorWithDataAsOrtValue) and it must outlive the session object - * to which it is added. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name, - _In_ const OrtValue* val); - - /// @} - /// \name OrtEnv - /// @{ - - /** - * Create a custom environment with global threadpools and logger that will be shared across sessions. - * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use - * its own thread pools. - * - * \param[in] logging_function A pointer to a logging function. - * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `logging_function`. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[in] tp_options - * \param[out] out Newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel log_severity_level, - _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append CUDA provider to session options - * - * If CUDA is not available (due to a non CUDA enabled build, or if CUDA is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] cuda_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA, - _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options); - - /** \brief Append ROCM execution provider to the session options - * - * If ROCM is not available (due to a non ROCM enabled build, or if ROCM is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] rocm_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_ROCM, - _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* rocm_options); - - /** \brief Append OpenVINO execution provider to the session options - * - * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. - * - * \param[in] options - * \param[in] provider_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO, - _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Set threading flush-to-zero and denormal-as-zero - * - * Sets global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. - * Flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool. - * \note This option is not needed if the models used have no denormals. Having no denormals is recommended as this option may hurt model accuracy. - * - * \param[in] tp_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options); - - /// @} - /// \name OrtArenaCfg - /// @{ - - /** \deprecated Use OrtApi::CreateArenaCfgV2 - * - * This will create the configuration of an arena that can eventually be used to define an arena based allocator's behavior - * - * \param[in] max_mem Use 0 to allow ORT to choose the default - * \param[in] arena_extend_strategy Use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested - * \param[in] initial_chunk_size_bytes Use -1 to allow ORT to choose the default - * \param[in] max_dead_bytes_per_chunk Use -1 to allow ORT to choose the default - * \param[in] out A pointer to an OrtArenaCfg instance - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, - int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out); - - ORT_CLASS_RELEASE(ArenaCfg); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** - * Use this to obtain the description of the graph present in the model - * (doc_string field of the GraphProto message within the ModelProto message). - * If it doesn't exist, an empty string will be returned. - * - * \param[in] model_metadata An instance of ::OrtModelMetadata - * \param[in] allocator Allocator used to allocate the string that will be returned back - * \param[out] value Set to a null terminated string allocated using `allocator`. The caller is responsible for freeing it using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append TensorRT provider to session options - * - * If TensorRT is not available (due to a non TensorRT enabled build, or if TensorRT is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] tensorrt_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT, - _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptions* tensorrt_options); - - /// @} - /// \name Misc - /// @{ - - /** \brief Set current GPU device ID - * - * Set the current device id of the GPU execution provider (CUDA/tensorrt/rocm). The device id should be less - * than the total number of devices available. This is only useful when multiple-GPUs are installed and it is - * required to restrict execution to a single GPU. - * - * \param[in] device_id - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetCurrentGpuDeviceId, _In_ int device_id); - - /** \brief Get current GPU device ID - * - * Get the current device id of the GPU execution provider (CUDA/tensorrt/rocm). - * - * \see OrtApi::SetCurrentGpuDeviceId - * - * \param[out] device_id - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetCurrentGpuDeviceId, _In_ int* device_id); - - /// @} - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Fetch an array of int64_t values stored as an attribute in the graph node - * - * - * If `out` is nullptr, the value of `size` is set to the true size of the attribute - * array's size, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual attribute array's size, - * the value of `size` is set to the true size of the attribute array's size, - * the provided memory is filled with the attribute's contents, - * and a success status is returned. - * - * If the `size` parameter is less than the actual attribute array's size and `out` - * is not nullptr, the value of `size` is set to the true size of the attribute array's size - * and a failure status is returned.) - * - * \param[in] info instance - * \param[in] name name of the attribute to be parsed - * \param[out] out pointer to memory where the attribute's contents are to be stored - * \param[in, out] size actual size of attribute array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttributeArray_float, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ float* out, _Inout_ size_t* size); - - /** \brief Fetch an array of int64_t values stored as an attribute in the graph node - * - * If `out` is nullptr, the value of `size` is set to the true size of the attribute - * array's size, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual attribute array's size, - * the value of `size` is set to the true size of the attribute array's size, - * the provided memory is filled with the attribute's contents, - * and a success status is returned. - * - * If the `size` parameter is less than the actual attribute array's size and `out` - * is not nullptr, the value of `size` is set to the true size of the attribute array's size - * and a failure status is returned.) - * - * \param[in] info instance - * \param[in] name name of the attribute to be parsed - * \param[out] out pointer to memory where the attribute's contents are to be stored - * \param[in, out] size actual size of attribute array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttributeArray_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ int64_t* out, _Inout_ size_t* size); - - /// @} - /// \name OrtArenaCfg - /// @{ - - /** \brief Create an ::OrtArenaCfg - * - * Create the configuration of an arena that can eventually be used to define an arena based allocator's behavior. - * - * Supported keys are (See https://onnxruntime.ai/docs/get-started/with-c.html for details on what the - * following parameters mean and how to choose these values.): - * "max_mem": Maximum memory that can be allocated by the arena based allocator. - * Use 0 for ORT to pick the best value. Default is 0. - * "arena_extend_strategy": 0 = kNextPowerOfTwo, 1 = kSameAsRequested. - * Use -1 to allow ORT to choose the default. - * "initial_chunk_size_bytes": (Possible) Size of the first allocation in the arena. - * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. - * Ultimately, the first allocation size is determined by the allocation memory request. - * "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after - * crossing which the current chunk is chunked into 2. - * "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. - * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. - * "max_power_of_two_extend_bytes": The maximum enxtend size if arena strategy is `kNextPowerOfTwo`. - * It is not an allocation limit, it is only a limit for extention when requested byte is less than the limit. - * When requested bytes is more than the limit, allocator will still return as requested. - * Use -1 to allow ORT to choose the default 1GB for max_power_of_two_extend_bytes. - * Ultimately, the allocation size is determined by the allocation memory request. - * Further allocation sizes are governed by the arena extend strategy. - * - * \param[in] arena_config_keys Keys to configure the arena - * \param[in] arena_config_values Values to configure the arena - * \param[in] num_keys Number of keys in `arena_config_keys` and `arena_config_values` - * \param[out] out Newly created ::OrtArenaCfg. Must be freed with OrtApi::ReleaseArenaCfg - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateArenaCfgV2, _In_reads_(num_keys) const char* const* arena_config_keys, - _In_reads_(num_keys) const size_t* arena_config_values, _In_ size_t num_keys, - _Outptr_ OrtArenaCfg** out); - - /// @} - /// \name OrtRunOptions - /// @{ - - /** \brief Set a single run configuration entry as a pair of strings - * - * If a configuration with same key exists, this will overwrite the configuration with the given config_value - * - * The config_key and the format of config_value are defined in onnxruntime_run_options_config_keys.h - * - * \param[in] options - * \param[in] config_key A null terminated string representation of the config key - * \param[in] config_value A null terminated string representation of the config value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddRunConfigEntry, _Inout_ OrtRunOptions* options, - _In_z_ const char* config_key, _In_z_ const char* config_value); - - /// @} - /// \name OrtPrepackedWeightsContainer - /// @{ - - /** \brief Create an ::OrtPrepackedWeightsContainer - * - * This container will hold pre-packed buffers of shared initializers for sharing between sessions - * (i.e.) if there are shared initializers that can be shared between sessions, the pre-packed buffers - * of these (if any) may possibly be shared to provide memory footprint savings. Pass this container - * to sessions that you would like to share pre-packed buffers of shared initializers at session - * creation time. - * - * \param[out] out Newly created ::OrtPrepackedWeightsContainer. Must be freed with OrtApi::ReleasePrepackedWeightsContainer - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreatePrepackedWeightsContainer, _Outptr_ OrtPrepackedWeightsContainer** out); - - /** \brief Release OrtPrepackedWeightsContainer instance - * - * \note instance must not be released until the sessions using it are released - */ - ORT_CLASS_RELEASE(PrepackedWeightsContainer); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Create session with prepacked weights container - * - * Same functionality offered by OrtApi::CreateSession except that a container that contains - * pre-packed weights' buffers is written into/read from by the created session. - * This is useful when used in conjunction with OrtApi::AddInitializer which injects - * shared initializer info into sessions. Wherever possible, the pre-packed versions of these - * shared initializers are cached in this container so that multiple sessions can just re-use - * these instead of duplicating these in memory. - * - * \param[in] env OrtEnv instance instance - * \param[in] model_path Null terminated string of the path (wchar on Windows, char otherwise) - * \param[in] options - * \param[in] prepacked_weights_container - * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, - _Outptr_ OrtSession** out); - - /** \brief Create session from memory with prepacked weights container - * - * Same functionality offered by OrtApi::CreateSessionFromArray except that a container that contains - * pre-packed weights' buffers is written into/read from by the created session. - * This is useful when used in conjunction with OrtApi::AddInitializer which injects - * shared initializer info into sessions. Wherever possible, the pre-packed versions of these - * shared initializers are cached in this container so that multiple sessions can just re-use - * these instead of duplicating these in memory. - * - * \param[in] env - * \param[in] model_data Array of bytes holding the model - * \param[in] model_data_length Number of bytes in `model_data_model` - * \param[in] options - * \param[in] prepacked_weights_container - * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ const OrtEnv* env, - _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, - _Outptr_ OrtSession** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append TensorRT execution provider to the session options - * - * If TensorRT is not available (due to a non TensorRT enabled build), this function will return failure. - * - * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, it takes an - * ::OrtTensorRTProviderOptions which is publicly defined. This takes an opaque ::OrtTensorRTProviderOptionsV2 - * which must be created with OrtApi::CreateTensorRTProviderOptions. - * - * For OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, the user needs to instantiate ::OrtTensorRTProviderOptions - * as well as allocate/release buffers for some members of ::OrtTensorRTProviderOptions. - * Here, OrtApi::CreateTensorRTProviderOptions and Ortapi::ReleaseTensorRTProviderOptions will do the memory management for you. - * - * \param[in] options - * \param[in] tensorrt_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT_V2, - _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options); - - /// @} - /// \name OrtTensorRTProviderOptionsV2 - /// @{ - - /** \brief Create an OrtTensorRTProviderOptionsV2 - * - * \param[out] out Newly created ::OrtTensorRTProviderOptionsV2. Must be released with OrtApi::ReleaseTensorRTProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out); - - /** \brief Set options in a TensorRT Execution Provider. - * - * Please refer to https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#cc - * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtTensorRTProviderOptionsV2 - * and value should be its related range. Recreates the options and only sets the supplied values. - * - * For example, key="trt_max_workspace_size" and value="2147483648" - * - * \param[in] tensorrt_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UpdateTensorRTProviderOptions, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Get serialized TensorRT provider options string. - * - * For example, "trt_max_workspace_size=2147483648;trt_max_partition_iterations=10;trt_int8_enable=1;......" - * - * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with OrtApi::CreateAllocator or OrtApi::GetAllocatorWithDefaultOptions - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtTensorRTProviderOptionsV2 - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - */ - void(ORT_API_CALL* ReleaseTensorRTProviderOptions)(_Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* input); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Enable custom operators - * - * See onnxruntime-extensions: https://github.com/microsoft/onnxruntime-extensions.git - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableOrtCustomOps, _Inout_ OrtSessionOptions* options); - - /// @} - /// \name OrtAllocator - /// @{ - - /** \brief Register a custom allocator - * - * Enables sharing between multiple sessions that use the same env instance. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * - * The behavior of this is exactly the same as OrtApi::CreateAndRegisterAllocator except - * instead of ORT creating an allocator based on provided info, in this case - * ORT uses the user-provided custom allocator. - * See https://onnxruntime.ai/docs/get-started/with-c.html for details. - * - * \param[in] env - * \param[in] allocator User provided allocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RegisterAllocator, _Inout_ OrtEnv* env, _In_ OrtAllocator* allocator); - - /** \brief Unregister a custom allocator - * - * It is an error if you provide an ::OrtMemoryInfo not corresponding to any - * registered allocators for sharing. - * - * \param[in] env - * \param[in] mem_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UnregisterAllocator, _Inout_ OrtEnv* env, - _In_ const OrtMemoryInfo* mem_info); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Sets *out to 1 iff an ::OrtValue is a SparseTensor, and 0 otherwise - * - * \param[in] value existing ::OrtValue - * \param[out] out unless an error occurs, contains 1 iff the value contains an instance - * of sparse tensor or 0 otherwise. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(IsSparseTensor, _In_ const OrtValue* value, _Out_ int* out); - - /** \brief Create an ::OrtValue with a sparse tensor that is empty. - * - * Use FillSparseTensor() functions to populate sparse tensor with non-zero values and - * format specific indices data. - * Use ReleaseValue to destroy the sparse tensor, this will also release the buffer inside the output value - * if any was allocated. - * \param[in,out] allocator allocator to use when performing an allocation. Allocation will be performed - * by FillSparseTensor() APIs. The lifespan of the allocator instance must eclipse the lifespan - * this sparse tensor instance as the same allocator will be used to free memory. - * \param[in] dense_shape shape of the original dense tensor - * \param[in] dense_shape_len number of shape dimensions being passed - * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx - * \param[out] out Should be freed by calling ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* dense_shape, - size_t dense_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and COO indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape pointer to values shape array - * \param[in] values_shape_len length of the values_shape - * \param[in] values pointer to an array of values. For strings, pass const char**. - * \param[in] indices_data pointer to a location of COO indices - * \param[in] indices_num number of COO indices - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* indices_data, size_t indices_num); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and CSR indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape pointer to values shape array - * \param[in] values_shape_len length of the values_shape - * \param[in] values - pointer to an array of values. For strings, pass const char**. - * \param[in] inner_indices_data pointer to a location of CSR inner indices - * \param[in] inner_indices_num number of CSR inner indices - * \param[in] outer_indices_data pointer to a location of CSR outer indices - * \param[in] outer_indices_num number of CSR outer indices - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* inner_indices_data, size_t inner_indices_num, - _In_ const int64_t* outer_indices_data, size_t outer_indices_num); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and BlockSparse indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape - * \param[in] values_shape_len - * \param[in] values structure with values information - * \param[in] indices_shape_data pointer to a location of indices shape - * \param[in] indices_shape_len length of the block sparse indices shape - * \param[in] indices_data pointer to a location of indices data. Shape will determine the length of the indices data. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* indices_shape_data, size_t indices_shape_len, - _In_ const int32_t* indices_data); - - /** - * Create an ::OrtValue with a sparse tensor. This is the first step. - * Next, use UseIndices() functions to supply sparse tensor with - * format specific indices data and set its sparse format to a specific enum value. - * This will not perform memory allocations. It will - * use supplied user buffer which should outlive the created sparse tensor. - * Use OrtApi::ReleaseValue to destroy the sparse tensor. It would not release the supplied values buffer. - * This function can not be used to map strings from the user allocated memory. Strings must always be copied - * and have UTF-8 encoding. Therefore, use OrtApi::CreateSparseTensorAsOrtValue above and then fill it with data - * using appropriate Make*() function. - * - * \param[in] info memory info where sparse values reside. - * \param[in,out] p_data pointer to a user allocated buffer with values. To create a full sparse tensor with no non-zero - * values, pass nullptr - * \param[in] dense_shape shape of the original dense tensor - * \param[in] dense_shape_len number of shape dimensions being passed - * \param[in] values_shape shape of the values data. To create a fully sparse tensor with no non-zero values, - * pass {0} shape. - * \param[in] values_shape_len number of values shape dimensions - * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx - * \param[out] out Should be freed by calling ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSparseTensorWithValuesAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, - _In_ const int64_t* dense_shape, size_t dense_shape_len, - _In_ const int64_t* values_shape, size_t values_shape_len, - ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** - * This assigns Coo format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_COO. This will not allocate any additional memory for data. The life span of - * indices_data buffer should eclipse the life span of this ::OrtValue. - * - * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in,out] indices_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] indices_num number of COO indices. Should either be 0 for fully sparse tensors, be equal - * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue for 1-D {nnz} indices or - * be twice as number of nnz values for a 2-D indices {nnz, 2} - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* indices_data, size_t indices_num); - - /** - * The assigns CSR format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_CSRC. This will not allocate any additional memory for data. The life spans of - * inner_data and outer_data buffers should eclipse the life span of this ::OrtValue. - * - * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in,out] inner_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] inner_num number of inner CSR indices. Should either be 0 for fully sparse tensors or be equal - * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue. - * \param[in,out] outer_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] outer_num number of CSR outer indices. Should either be 0 for fully sparse tensors or - * equal to rows + 1 of the dense shape. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseCsrIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* inner_data, size_t inner_num, - _Inout_ int64_t* outer_data, size_t outer_num); - - /** - * The assigns BlockSparse format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_BLOCK_SPARSE. This will not allocate any additional memory for data. The life span of - * indices_data buffer must eclipse the lifespan of this ::OrtValue. - * - * \param[in,out] ort_value OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in] indices_shape pointer to indices shape. Use {0} for fully sparse tensors - * \param[in] indices_shape_len length of the indices shape - * \param[in,out] indices_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseBlockSparseIndices, _Inout_ OrtValue* ort_value, const int64_t* indices_shape, size_t indices_shape_len, _Inout_ int32_t* indices_data); - - /** \brief Returns sparse tensor format enum iff a given ort value contains an instance of sparse tensor. - * - * \param[in] ort_value ::OrtValue that contains an instance of sparse tensor - * \param[out] out pointer to out parameter - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorFormat, _In_ const OrtValue* ort_value, _Out_ enum OrtSparseFormat* out); - - /** \brief Returns data type and shape of sparse tensor values (nnz) iff ::OrtValue contains a SparseTensor. - * - * \param[in] ort_value An ::OrtValue that contains a fully constructed sparse tensor - * \param[out] out Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorValuesTypeAndShape, _In_ const OrtValue* ort_value, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Returns numeric data for sparse tensor values (nnz). For string values use GetStringTensor*(). - * - * \param[in] ort_value an instance of ::OrtValue containing sparse tensor - * \param[out] out returns a pointer to values data. Do not attempt to free this ptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorValues, _In_ const OrtValue* ort_value, _Outptr_ const void** out); - - /** \brief Returns data type, shape for the type of indices specified by indices_format. - * - * \param[in] ort_value ::OrtValue containing sparse tensor. - * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse - * tensor does not contain. - * \param[out] out an instance of ::OrtTensorTypeAndShapeInfo. Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorIndicesTypeShape, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Returns indices data for the type of the indices specified by indices_format - * - * \param[in] ort_value ::OrtValue containing sparse tensor. - * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse tensor does not contain. - * \param[out] num_indices Pointer to where the number of indices entries is returned - * \param[out] indices Returned pointer to the indices data. Do not free the returned pointer as it refers to internal data owned by the ::OrtValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorIndices, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Out_ size_t* num_indices, _Outptr_ const void** indices); - /// @} - /// \name OrtSessionOptions - /// @{ - - /** - * \brief Sets out to 1 iff an optional type OrtValue has an element, 0 otherwise (OrtValue is None) - * Use this API to find if the optional type OrtValue is None or not. - * If the optional type OrtValue is not None, use the OrtValue just like any other OrtValue. - * For example, if you get an OrtValue that corresponds to Optional(tensor) and - * if HasValue() returns true, use it as tensor and so on. - - * \param[in] value Input OrtValue. - * \param[out] out indicating if the input OrtValue contains data (1) or if it is a None (0) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(HasValue, _In_ const OrtValue* value, _Out_ int* out); - - /// @} - /// \name OrtKernelContext - /// Custom operator APIs. - /// @{ - - /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel - * \see ::OrtCustomOp - * \param[in] context OrtKernelContext instance - * \param[out] out Returns pointer to a GPU compute stream that can be used to launch the custom GPU kernel. - * If retrieving the GPU compute stream is not relevant (GPU not enabled in the build, kernel partitioned to - * some other EP), then a nullptr is returned as the output param. - * Do not free or mutate the returned pointer as it refers to internal data owned by the underlying session. - * Only use it for custom kernel launching. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetGPUComputeStream, _In_ const OrtKernelContext* context, _Outptr_ void** out); - - /// @} - /// \name GetTensorMemoryInfo - /// @{ - /** \brief Returns a pointer to the ::OrtMemoryInfo of a Tensor - * \param[in] value ::OrtValue containing tensor. - * \param[out] mem_info ::OrtMemoryInfo of the tensor. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorMemoryInfo, _In_ const OrtValue* value, _Out_ const OrtMemoryInfo** mem_info); - - /// @} - /// \name GetExecutionProviderApi - /// @{ - /** \brief Get a pointer to the requested version of the Execution Provider specific - * API extensions to the OrtApi - * \param[in] provider_name The name of the execution provider name. Currently only the following - * values are supported: "DML". - * \param[in] version Must be ::ORT_API_VERSION. - * \param[out] provider_api A void pointer containing a reference to the execution provider versioned api structure. - * For example, the provider_api pointer can be cast to the OrtDmlApi* when the provider_name is "DML". - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetExecutionProviderApi, _In_ const char* provider_name, _In_ uint32_t version, _Outptr_ const void** provider_api); - - /// @} - - /// \name SessionOptions - /// @{ - /** \brief Set custom thread creation function - * - * \param[in] options Session options - * \param[in] ort_custom_create_thread_fn Custom thread creation function - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomCreateThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /** \brief Set creation options for custom thread - * - * \param[in] options Session options - * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomThreadCreationOptions, _Inout_ OrtSessionOptions* options, _In_ void* ort_custom_thread_creation_options); - - /** \brief Set custom thread join function - * - * \param[in] options Session options - * \param[in] ort_custom_join_thread_fn Custom join thread function, must not be nullptr when ort_custom_create_thread_fn is set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomJoinThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); - /// @} - - /// \name OrtThreadingOptions - /// @{ - /** \brief Set custom thread creation function for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_create_thread_fn Custom thread creation function - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomCreateThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /** \brief Set custom thread creation options for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomThreadCreationOptions, _Inout_ OrtThreadingOptions* tp_options, _In_ void* ort_custom_thread_creation_options); - - /** \brief Set custom thread join function for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_join_thread_fn Custom thread join function, must not be nullptr when global ort_custom_create_thread_fn is set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomJoinThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); - /// @} - - /** \brief Synchronize bound inputs. The call may be necessary for some providers, such as cuda, - * in case the system that allocated bound memory operated on a different stream. However, the - * operation is provider specific and could be a no-op. - * - * \param[inout] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SynchronizeBoundInputs, _Inout_ OrtIoBinding* binding_ptr); - - /** \brief Synchronize bound outputs. The call may be necessary for some providers, such as cuda, - * in case the system that allocated bound memory operated on a different stream. However, the - * operation is provider specific and could be a no-op. - * - * \param[inout] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr); - - /// \name OrtSessionOptions - /// @{ - - /** \brief Append CUDA execution provider to the session options - * - * If CUDA is not available (due to a non CUDA enabled build), this function will return failure. - * - * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_CUDA, it takes an - * ::OrtCUDAProviderOptions which is publicly defined. This takes an opaque ::OrtCUDAProviderOptionsV2 - * which must be created with OrtApi::CreateCUDAProviderOptions. - * - * For OrtApi::SessionOptionsAppendExecutionProvider_CUDA, the user needs to instantiate ::OrtCUDAProviderOptions - * as well as allocate/release buffers for some members of ::OrtCUDAProviderOptions. - * Here, OrtApi::CreateCUDAProviderOptions and Ortapi::ReleaseCUDAProviderOptions will do the memory management for you. - * - * \param[in] options - * \param[in] cuda_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA_V2, - _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options); - - /// @} - /// \name OrtCUDAProviderOptionsV2 - /// @{ - - /** \brief Create an OrtCUDAProviderOptionsV2 - * - * \param[out] out Newly created ::OrtCUDAProviderOptionsV2. Must be released with OrtApi::ReleaseCudaProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out); - - /** \brief Set options in a CUDA Execution Provider. - * - * Please refer to https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#configuration-options - * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtCUDAProviderOptionsV2 - * and value should be its related range. Recreates the options and only sets the supplied values. - * - * For example, key="device_id" and value="0" - * - * \param[in] cuda_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** - * Get serialized CUDA provider options string. - * - * For example, "device_id=0;arena_extend_strategy=0;......" - * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtCUDAProviderOptionsV2 - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - * - * \since Version 1.11. - */ - void(ORT_API_CALL* ReleaseCUDAProviderOptions)(_Frees_ptr_opt_ OrtCUDAProviderOptionsV2* input); - - /// @} - - /** \brief Append MIGraphX provider to session options - * - * If MIGraphX is not available (due to a non MIGraphX enabled build, or if MIGraphX is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] migraphx_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_MIGraphX, - _In_ OrtSessionOptions* options, _In_ const OrtMIGraphXProviderOptions* migraphx_options); - - /** \brief Replace initialized Tensors with external data with the data provided in initializers. - * - * The function will find the initialized TensorProtos with external data in the graph with the provided names and - * replace them with the provided tensors. The API verifies that the TensorProto being replaced - * has an external data reference and has the same name, dimensions and data type as its replacement. The replacement - * will occur before any of the optimizations take place. The data will be copied into the graph - * since TensorProto can't refer to the user provided buffers. - * - * Once the model has been loaded, the OrtValue(s) added to SessionOptions instance will be removed - * from the internal SessionOptions copy to save memory, the user provided buffers can then be deallocated - * and the SessionOptions instance that refers to them can be destroyed. - * - * \param[in] options - * \param[in] initializer_names Array of null terminated UTF-8 encoded strings of the initializers names. - * \param[in] initializers Array of ::OrtValue type - * \param[in] num_initializers Number of elements in the initializer_names and initializers - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.12. - */ - ORT_API2_STATUS(AddExternalInitializers, _In_ OrtSessionOptions* options, - _In_reads_(num_initializers) const char* const* initializer_names, - _In_reads_(num_initializers) const OrtValue* const* initializers, size_t num_initializers); - - /** \brief: Create attribute of onnxruntime operator - * - * \param[in] name Name of the attribute - * \param[in] data Data content of the attribute - * \param[in] len Number of bytes stored in data - * \param[in] type Data type - * \param[out] op_attr Attribute that has been created, which must be released by OrtApi::ReleaseOpAttr - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CreateOpAttr, - _In_ const char* name, - _In_ const void* data, - _In_ int len, - _In_ OrtOpAttrType type, - _Outptr_ OrtOpAttr** op_attr); - - /* \brief: Release op attribute - * - * \param[in] opAttr Attribute created by OrtApi::CreateOpAttr - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(OpAttr); - - /** \brief: Create onnxruntime native operator - * - * \param[in] info Kernel info - * \param[in] op_name Operator name - * \param[in] domain Operator domain - * \param[in] version Operator opset version - * \param[in] type_constraint_names Name of the type contraints, such as "T" or "T1" - * \param[in] type_constraint_values Type of each contraints - * \param[in] type_constraint_count Number of contraints - * \param[in] attr_values Attributes used to initialize the operator - * \param[in] attr_count Number of the attributes - * \param[in] input_count Number of inputs - * \param[in] output_count Number of outputs - * \param[out] ort_op Operator that has been created - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CreateOp, - _In_ const OrtKernelInfo* info, - _In_z_ const char* op_name, - _In_z_ const char* domain, - int version, - _In_reads_(type_constraint_count) const char** type_constraint_names, - _In_reads_(type_constraint_count) const ONNXTensorElementDataType* type_constraint_values, - int type_constraint_count, - _In_reads_(attr_count) const OrtOpAttr* const* attr_values, - int attr_count, - int input_count, - int output_count, - _Outptr_ OrtOp** ort_op); - - /** \brief: Invoke the operator created by OrtApi::CreateOp - * The inputs must follow the order as specified in onnx specification - * - * \param[in] context Kernel context - * \param[in] ort_op Operator that has been created - * \param[in] input_values Array of inputs - * \param[in] input_count Number of inputs - * \param[in] output_values Array of outputs - * \param[in] output_count Number of outputs - * - * \since Version 1.12. - */ - ORT_API2_STATUS(InvokeOp, - _In_ const OrtKernelContext* context, - _In_ const OrtOp* ort_op, - _In_ const OrtValue* const* input_values, - _In_ int input_count, - _Inout_ OrtValue* const* output_values, - _In_ int output_count); - - /* \brief: Release an onnxruntime operator - * - * \param[in] Op Operator created by OrtApi::CreateOp - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(Op); - - /** \brief: Append execution provider to the session options. - * \param[in] options - * \param[in] provider_name - provider to add. - * \param[in] provider_options_keys - keys to configure the provider options - * \param[in] provider_options_values - values to configure the provider options - * \param[in] num_keys - number of keys passed in - * - * Currently supported providers: - * QNN - * SNPE - * XNNPACK - * - * Note: If an execution provider has a dedicated SessionOptionsAppendExecutionProvider_ function - * that should be used to add it. - * - * QNN supported keys: - * "backend_path": file path to QNN backend library. - * "profiling_level": QNN profiling level, options: "off", "basic", "detailed". Default to off. - * "profiling_file_path": QNN profiling file path if ETW not enabled. - * "rpc_control_latency": QNN RPC control latency. - * "vtcm_mb": QNN VTCM size in MB. default to 0(not set). - * "htp_performance_mode": QNN performance mode, options: "burst", "balanced", "default", "high_performance", - * "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", "sustained_high_performance". Default to "default". - * "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will - * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and - * may alter model/EP partitioning. Use only for debugging. - * "qnn_context_priority": QNN context priority, options: "low", "normal", "normal_high", "high". Default to "normal". - * "htp_graph_finalization_optimization_mode": Set the optimization mode for graph finalization on the HTP backend. Available options: - * - "0": Default. - * - "1": Faster preparation time, less optimal graph. - * - "2": Longer preparation time, more optimal graph. - * - "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific details. - * "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values. Defaults to "0" (unknown). - * "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators. Available options: - * - "0": Default (none). - * - "68" - * - "69" - * - "73" - * - "75" - * "device_id": The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device). - "enable_htp_fp16_precision": Only used for float32 model. - Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision. - - "0": Default. With fp32 precision. - - "1": With fp16 precision. - * - * SNPE supported keys: - * "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16", - * "DSP", "DSP_FIXED8_TF", "AIP_FIXED_TF", "AIP_FIXED8_TF". - * Mapping to SNPE Runtime_t definition: CPU, CPU_FLOAT32 => zdl::DlSystem::Runtime_t::CPU; - * GPU, GPU_FLOAT32_16_HYBRID => zdl::DlSystem::Runtime_t::GPU; - * GPU_FLOAT16 => zdl::DlSystem::Runtime_t::GPU_FLOAT16; - * DSP, DSP_FIXED8_TF => zdl::DlSystem::Runtime_t::DSP. - * AIP_FIXED_TF, AIP_FIXED8_TF => zdl::DlSystem::Runtime_t::AIP_FIXED_TF. - * "priority": execution priority, options: "low", "normal". - * "buffer_type": ITensor or user buffers, options: "ITENSOR", user buffer with different types - "TF8", "TF16", "UINT8", "FLOAT". - * "ITENSOR" -- default, ITensor which is float only. - * "TF8" -- quantized model required, "FLOAT" -- for both quantized or non-quantized model - * "enable_init_cache": enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. - * If SNPE is not available (due to a non Snpe enabled build or its dependencies not being installed), this function will fail. - * - * XNNPACK supported keys: - * "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider. - * default value is 0, which means to use the session thread-pool size. - * - * \since Version 1.12. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider, _In_ OrtSessionOptions* options, - _In_ const char* provider_name, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /* \brief: Get a copy of kernel info - * - * \param[in] info Kernel info - * \param[out] info_copy Copy of kernel info - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CopyKernelInfo, - _In_ const OrtKernelInfo* info, - _Outptr_ OrtKernelInfo** info_copy); - - /* \brief: Release kernel info - * - * \param[in] KernelInfo A copy of kernel info returned by CopyKernelInfo - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(KernelInfo); - - /// \name Ort Training - /// @{ - /** \brief Gets the Training C Api struct - * - * Call this function to access the ::OrtTrainingApi structure that holds pointers to functions that enable - * training with onnxruntime. - * \note A NULL pointer will be returned and no error message will be printed if the training api - * is not supported with this build. A NULL pointer will be returned and an error message will be - * printed if the provided version is unsupported, for example when using a runtime older than the - * version created with this header file. - * - * \param[in] version Must be ::ORT_API_VERSION - * \return The ::OrtTrainingApi struct for the version requested. - * - * \since Version 1.13 - */ - const OrtTrainingApi*(ORT_API_CALL* GetTrainingApi)(uint32_t version)NO_EXCEPTION; - - /// @} - - /** \brief Append CANN provider to session options - * - * If CANN is not available (due to a non CANN enabled build, or if CANN is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] cann_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CANN, - _In_ OrtSessionOptions* options, _In_ const OrtCANNProviderOptions* cann_options); - - /** \brief Create an OrtCANNProviderOptions - * - * \param[out] out created ::OrtCANNProviderOptions. Must be released with OrtApi::ReleaseCANNProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(CreateCANNProviderOptions, _Outptr_ OrtCANNProviderOptions** out); - - /** \brief Set options in a CANN Execution Provider. - * - * \param[in] cann_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(UpdateCANNProviderOptions, _Inout_ OrtCANNProviderOptions* cann_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Get serialized CANN provider options string. - * - * \param[in] cann_options OrtCANNProviderOptions instance - * \param[in] allocator a ptr to an instance of OrtAllocator obtained with CreateAllocator() - * or GetAllocatorWithDefaultOptions(), the specified allocator will be used to allocate - * continuous buffers for output strings and lengths. - * \param[out] ptr is a UTF-8 null terminated string allocated using 'allocator'. - * The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.13. - */ - ORT_API2_STATUS(GetCANNProviderOptionsAsString, _In_ const OrtCANNProviderOptions* cann_options, - _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an OrtCANNProviderOptions - * - * \param[in] the pointer of OrtCANNProviderOptions which will been deleted - * - * \since Version 1.13. - */ - void(ORT_API_CALL* ReleaseCANNProviderOptions)(_Frees_ptr_opt_ OrtCANNProviderOptions* input); - - /* \brief Get OrtDevice type from MemoryInfo - * - * \since Version 1.14 - */ - void(ORT_API_CALL* MemoryInfoGetDeviceType)(_In_ const OrtMemoryInfo* ptr, _Out_ OrtMemoryInfoDeviceType* out); - - /* \brief Update the OrtEnv instance with custom log severity level - * - * \param[in] ort_env The OrtEnv instance being used - * \param[in] log_severity_level The log severity level. - * - * \since Version 1.14. - */ - ORT_API2_STATUS(UpdateEnvWithCustomLogLevel, _In_ OrtEnv* ort_env, OrtLoggingLevel log_severity_level); - - /* \brief Set affinities for intra op threads - * - * Affinity string follows format: - * logical_processor_id,logical_processor_id;logical_processor_id,logical_processor_id - * Semicolon isolates configurations among threads, while comma split processors where ith thread expected to attach to. - * e.g. 1,2,3;4,5 - * specifies affinities for two threads, with the 1st thread attach to the 1st, 2nd, and 3rd processor, and 2nd thread to the 4th and 5th. - * To ease the configuration, an "interval" is also allowed: - * e.g. 1-8;8-16;17-24 - * orders that the 1st thread runs on first eight processors, 2nd thread runs on next eight processors, and so forth. - * Note: - * 1. Once set, the number of thread affinities must equal to intra_op_num_threads - 1, - * ort does not set affinity on the main thread which is started and managed by the calling app; - * 2. For windows, ort will infer the group id from a logical processor id, for example, assuming there are two groups with each has 64 logical processors, - * an id of 64 will be inferred as the last processor of the 1st group, while 65 will be interpreted as the 1st processor of the second group. - * Hence 64-65 is an invalid configuration, because a windows thread cannot be attached to processors across group boundary. - * - * \since Version 1.14 - */ - ORT_API2_STATUS(SetGlobalIntraOpThreadAffinity, _Inout_ OrtThreadingOptions* tp_options, const char* affinity_string); - - /** \brief Register custom ops from a shared library. - * - * Loads a shared library (.dll on windows, .so on linux, etc) named 'library_name' and looks for this entry point: - * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); - * It then passes in the provided session options to this function along with the api base. - * - * The handle to the loaded library is automatically released by ORT when the last OrtSession that references the - * library handle is released. If no OrtSession is created, then the library handle is released when the provided - * OrtSessionOptions is released. - * - * \param[in] options The session options. - * \param[in] library_name The name of the shared library to load and register. Refer to OS-specific dynamic library - * loading utilities (e.g., LoadLibraryEx on Windows or dlopen on Linux/MacOS) for information - * on the format of library names and search paths. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(RegisterCustomOpsLibrary_V2, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* library_name); - - /** \brief Register custom ops by calling a RegisterCustomOpsFn function. - * - * Searches for registration_func_name and if found calls it. - * - * The library containing the function must either be linked against or previously loaded by the executable. - * - * If you want ONNX Runtime to load the library and manage its lifetime, use RegisterCustomOpsLibrary_V2. - * - * RegisterCustomOpsUsingFunction can be used in scenarios where it may not be possible for ONNX Runtime to load - * the library from a path. e.g. mobile platforms where the library must be linked into the app. - * - * The registration function must have the signature of RegisterCustomOpsFn: - * OrtStatus* (*fn)(OrtSessionOptions* options, const OrtApiBase* api); - * - * See https://onnxruntime.ai/docs/reference/operators/add-custom-op.html for details on how the registration - * function should be implemented. - * - * \param[in] options OrtSessionOptions that is passed through as the first argument in the call to the - * registration function. - * \param[in] registration_func_name Name of registration function to use. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(RegisterCustomOpsUsingFunction, _Inout_ OrtSessionOptions* options, - _In_ const char* registration_func_name); - - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Get the number of inputs from ::OrtKernelInfo. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the number of inputs - * during kernel/session creation. - * - * \param[in] info Instance of ::OrtKernelInfo. - * \param[out] out Pointer to variable assigned with the result on success. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetInputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); - - /** \brief Get the number of outputs from ::OrtKernelInfo. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the number of outputs - * during kernel/session creation. - * - * \param[in] info Instance of ::OrtKernelInfo. - * \param[out] out Pointer to variable assigned with the result on success. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetOutputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); - - /** \brief Get the name of a ::OrtKernelInfo's input. - * - * Used in the CreateKernel callback of an OrtCustomOp to query an input's name - * during kernel/session creation. - * - * If `out` is nullptr, the value of `size` is set to the size of the name - * string (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the name string's size, - * the value of `size` is set to the true size of the string (including null-terminator), - * the provided memory is filled with the string's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string - * and a failure status is returned. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index The index of the input name to get. Returns a failure status if out-of-bounds. - * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the input's name. - * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetInputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, - _Inout_ size_t* size); - - /** \brief Get the name of a ::OrtKernelInfo's output. - * - * Used in the CreateKernel callback of an OrtCustomOp to query an output's name - * during kernel/session creation. - * - * If `out` is nullptr, the value of `size` is set to the size of the name - * string (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the name string's size, - * the value of `size` is set to the true size of the string (including null-terminator), - * the provided memory is filled with the string's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string - * and a failure status is returned. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index The index of the output name to get. Returns a failure status if out-of-bounds. - * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the output's - * name. - * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetOutputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, - _Inout_ size_t* size); - - /** \brief Get the type information for a ::OrtKernelInfo's input. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information - * of an input during kernel/session creation. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index Which input to get the type information for - * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetInputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, - _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get the type information for a ::OrtKernelInfo's output. - * - * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information - * of an output during kernel/session creation. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[in] index Which input to get the type information for - * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(KernelInfo_GetOutputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, - _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get a ::OrtValue tensor stored as an attribute in the graph node. - * - * Used in the CreateKernel callback of an OrtCustomOp to get a tensor attribute. - * - * \param[in] info ::OrtKernelInfo instance. - * \param[in] name UTF-8 null-terminated string representing the attribute's name. - * \param[in] allocator Allocator used to allocate the internal tensor state. - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue, - * which will also free internal tensor state allocated with the provided allocator. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_tensor, _In_ const OrtKernelInfo* info, _In_z_ const char* name, - _Inout_ OrtAllocator* allocator, _Outptr_ OrtValue** out); - - /// @} - /// \name OrtSessionOptions - /// Custom operator APIs - /// @{ - - /** \brief Checks if the given session configuration entry exists. - * - * The config_key formats are defined in onnxruntime_session_options_config_keys.h - * - * Can be used in a custom operator library to check for session configuration entries - * that target one or more custom operators in the library. Example: The config entry - * custom_op.myop.some_key targets a custom op named "myop". - * - * \param[in] options The ::OrtSessionOptions instance. - * \param[in] config_key A null-terminated UTF-8 string representation of the configuration key. - * \param[out] out Pointer set to 1 if the entry exists and 0 otherwise. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(HasSessionConfigEntry, _In_ const OrtSessionOptions* options, - _In_z_ const char* config_key, _Out_ int* out); - - /** \brief Get a session configuration value. - * - * Returns a failure status if the configuration key does not exist. - * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h - * - * If `config_value` is nullptr, the value of `size` is set to the true size of the string - * value (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual string value's size, - * the value of `size` is set to the true size of the string value, the provided memory - * is filled with the value's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string value's size and `config_value` - * is not nullptr, the value of `size` is set to the true size of the string value - * and a failure status is returned. - * - * Can be used in a custom operator library to get session configuration entries - * that target one or more custom operators in the library. Example: The config entry - * custom_op.myop.some_key targets a custom op named "myop". - * - * \param[in] options The session options. - * \param[in] config_key A null-terminated UTF-8 string representation of the config key. - * \param[in] config_value Pointer to memory where the null-terminated UTF-8 string value will be stored. - * \param[in,out] size Pointer to the size of the `config_value` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.14 - */ - ORT_API2_STATUS(GetSessionConfigEntry, _In_ const OrtSessionOptions* options, - _In_z_ const char* config_key, _Out_ char* config_value, _Inout_ size_t* size); - - /// @} - - /** \brief Append dnnl provider to session options - * - * If oneDNN is not available, this function will return failure. - * - * \param[in] options - * \param[in] dnnl_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_Dnnl, - _In_ OrtSessionOptions* options, _In_ const OrtDnnlProviderOptions* dnnl_options); - - /** \brief Create an OrtDnnlProviderOptions - * - * \param[out] out Newly created ::OrtDnnlProviderOptions. Must be released with OrtApi::ReleaseDnnlProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(CreateDnnlProviderOptions, _Outptr_ OrtDnnlProviderOptions** out); - - /** \brief Set options in a oneDNN Execution Provider. - * - * Key should be in null terminated string format of the member of ::OrtDnnlProviderOptions - * and value should be its related range. - * - * For example, key="use_arena" and value="1" - * - * \param[in] dnnl_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(UpdateDnnlProviderOptions, _Inout_ OrtDnnlProviderOptions* dnnl_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** - * Get serialized oneDNN provider options string. - * - * For example, "use_arena=1;......" - * - * \param dnnl_options - OrtDnnlProviderOptions instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(GetDnnlProviderOptionsAsString, _In_ const OrtDnnlProviderOptions* dnnl_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtDnnlProviderOptions - * - * \since Version 1.15. - */ - void(ORT_API_CALL* ReleaseDnnlProviderOptions)(_Frees_ptr_opt_ OrtDnnlProviderOptions* input); - - /// \name OrtKernelInfo - /// Custom operator APIs. - /// @{ - - /** \brief Get the graph node name from ::OrtKernelInfo. - * - * If `out` is nullptr, the value of `size` is set to the size of the name - * string (including null-terminator), and a success status is returned. - * - * If the `size` parameter is greater than or equal to the name string's size, - * the value of `size` is set to the true size of the string (including null-terminator), - * the provided memory is filled with the string's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string - * and a failure status is returned. - * - * Can be used in a custom operator's CreateKernel callback to get the name of the operator's node name in the graph. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the name. - * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(KernelInfo_GetNodeName, _In_ const OrtKernelInfo* info, _Out_ char* out, _Inout_ size_t* size); - - /** \brief Get the session logger from ::OrtKernelInfo. - * - * Used in the CreateKernel callback of an OrtCustomOp to get a logger that can be used to log - * messages. - * - * \param[in] info An instance of ::OrtKernelInfo. - * \param[out] logger Pointer set to the session's ::OrtLogger. Owned by ONNX Runtime, so do not free. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(KernelInfo_GetLogger, _In_ const OrtKernelInfo* info, _Outptr_ const OrtLogger** logger); - - /// @} - /// \name OrtKernelContext - /// Custom operator APIs. - /// @{ - - /** \brief Get the runtime logger from ::OrtKernelContext. - * - * Used in the KernelCompute callback of an OrtCustomOp to get a logger that can be used to log - * messages during inference. - * - * \param[in] context An instance of ::OrtKernelContext. - * \param[out] logger Pointer set to the kernel context's ::OrtLogger. Owned by ONNX Runtime, so do not free. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(KernelContext_GetLogger, _In_ const OrtKernelContext* context, _Outptr_ const OrtLogger** logger); - - /// @} - /// \name OrtLogger - /// Custom operator APIs. - /// @{ - - /** \brief Logs a message at the given severity level using the provided ::OrtLogger. - * - * Only messages with a severity level equal or greater than the ::OrtLogger's logging severity level - * are logged. Use OrtApi::Logger_GetLoggingSeverityLevel to get the ::OrtLogger's logging severity - * level. - * - * Can be used in custom operators to log messages with the logger retrieved via OrtApi::KernelInfo_GetLogger. - * - * \param[in] logger The ::OrtLogger instance. - * \param[in] log_severity_level The message's severity level. - * \param[in] message The message to log. - * \param[in] file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. - * \param[in] line_number The file line number in which the message is logged. Usually the value of __LINE__. - * \param[in] func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(Logger_LogMessage, _In_ const OrtLogger* logger, OrtLoggingLevel log_severity_level, - _In_z_ const char* message, _In_z_ const ORTCHAR_T* file_path, int line_number, - _In_z_ const char* func_name); - - /** \brief Get the logging severity level of the ::OrtLogger. - * - * Can be used in a custom operator to get the logging serverity level of the ::OrtLogger associated with - * the ::OrtKernelInfo. - * - * \param[in] logger The ::OrtLogger instance. - * \param[out] out Pointer to variable assigned with the logging severity level on success. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.15 - */ - ORT_API2_STATUS(Logger_GetLoggingSeverityLevel, _In_ const OrtLogger* logger, _Out_ OrtLoggingLevel* out); - - /// @} - - /** \brief Get a ::OrtValue tensor stored as a constant initializer in the graph node. - * - * Used in the CreateKernel callback of an OrtCustomOp to get a tensor value. - * - * \param[in] info ::OrtKernelInfo instance. - * \param[in] index The node index. - * \param[out] is_constant Is it a constant node input or not. - * \param[out] out The OrtValue tensor value. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(KernelInfoGetConstantInput_tensor, _In_ const OrtKernelInfo* info, size_t index, _Out_ int* is_constant, _Outptr_ const OrtValue** out); - - /** \brief Get Optional Type information from an ::OrtTypeInfo - * - * This augments ::OrtTypeInfo to return an ::OrtOptionalTypeInfo when the type is optional. - * The OrtOptionalTypeInfo also has a nested ::OrtTypeInfo that describes the type of the optional value. - * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. - * The actual OrtValues that are supplied in place of optional type inputs should contain - * specific type that is described by ::OrtOptionalTypeInfo. - * - * So the picture: ::OrtTypeInfo -> ::OrtOptionalTypeInfo -> ::OrtTypeInfo (describes the type that can be supplied - * in place of the optional type when creating the actual ::OrtValue). - * - * \param[in] type_info - * \param[out] out A pointer to the ::OrtOptionalTypeInfo. Do not free this value, - * it is owned by OrtTypeInfo instance. When the type_info does not represent - * optional type, nullptr is returned in out. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(CastTypeInfoToOptionalTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtOptionalTypeInfo** out); - - /** \brief Get OrtTypeInfo for the allowed contained type from an ::OrtOptionalTypeInfo. - * - * This augments ::OrtOptionalTypeInfo to return an ::OrtTypeInfo for the contained type. - * The OrtOptionalTypeInfo has a nested ::OrtTypeInfo that describes the type of the optional value. - * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. - * The actual OrtValues that are supplied in place of optional type inputs should contain - * specific type that is described by the returned ::OrtTypeInfo. - * - * \param[in] optional_type_info - * \param[out] out A pointer to the ::OrtTypeInfo for what the optional value could be. - * it is owned by OrtOptionalTypeInfo instance. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(GetOptionalContainedTypeInfo, _In_ const OrtOptionalTypeInfo* optional_type_info, - _Outptr_ OrtTypeInfo** out); - - /** \brief Set a single string in a string tensor - * Do not zero terminate the string data. - * - * \param[in] value A string tensor - * \param[in] index - flat index of the element - * \param[in] length_in_bytes length of the buffer in utf-8 bytes (without the null terminator) - * \param[inout] buffer - address of return value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetResizedStringTensorElementBuffer, _Inout_ OrtValue* value, _In_ size_t index, _In_ size_t length_in_bytes, _Inout_ char** buffer); - - /** \brief Get Allocator from KernelContext for a specific memoryInfo. Please use C API ReleaseAllocator to release out object - * - * \param[in] context OrtKernelContext instance - * \param[in] mem_info OrtMemoryInfo instance - * \param[out] out A pointer to OrtAllocator. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.15. - */ - ORT_API2_STATUS(KernelContext_GetAllocator, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _Outptr_ OrtAllocator** out); - - /** \brief Returns a null terminated string of the build info including git info and cxx flags - * - * \return UTF-8 encoded version string. Do not deallocate the returned buffer. - * - * \since Version 1.15. - */ - const char*(ORT_API_CALL* GetBuildInfoString)(void); - - /// \name OrtROCMProviderOptions - /// @{ - - /** \brief Create an OrtROCMProviderOptions - * - * \param[out] out Newly created ::OrtROCMProviderOptions. Must be released with OrtApi::ReleaseROCMProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.16. - */ - ORT_API2_STATUS(CreateROCMProviderOptions, _Outptr_ OrtROCMProviderOptions** out); - - /** \brief Set options in a ROCm Execution Provider. - * - * Please refer to https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html - * to know the available keys and values. Key should be in null terminated string format of the member of - * ::OrtROCMProviderOptions and value should be its related range. - * - * For example, key="device_id" and value="0" - * - * \param[in] rocm_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.16. - */ - ORT_API2_STATUS(UpdateROCMProviderOptions, _Inout_ OrtROCMProviderOptions* rocm_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** - * Get serialized ROCm provider options string. - * - * For example, "device_id=0;arena_extend_strategy=0;......" - * - * \param rocm_options - OrtROCMProviderOptions instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.16. - */ - ORT_API2_STATUS(GetROCMProviderOptionsAsString, _In_ const OrtROCMProviderOptions* rocm_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtROCMProviderOptions - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - * - * \since Version 1.16. - */ - void(ORT_API_CALL* ReleaseROCMProviderOptions)(_Frees_ptr_opt_ OrtROCMProviderOptions* input); - - /** \brief Create an allocator with specific type and register it with the ::OrtEnv - * This API enhance CreateAndRegisterAllocator that it can create an allocator with specific type, not just CPU allocator - * Enables sharing the allocator between multiple sessions that use the same env instance. - * Lifetime of the created allocator will be valid for the duration of the environment. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * \param[in] env OrtEnv instance - * \param[in] provider_type ExecutionProvider type - * \param[in] mem_info OrtMemoryInfo instance - * \param[in] arena_cfg Arena configuration - * \param[in] provider_options_keys key of the provider options map - * \param[in] provider_options_values value of the provider options map - * \param[in] num_keys Length of the provider options map - */ - ORT_API2_STATUS(CreateAndRegisterAllocatorV2, _Inout_ OrtEnv* env, _In_ const char* provider_type, _In_ const OrtMemoryInfo* mem_info, _In_ const OrtArenaCfg* arena_cfg, - _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys); - - /** \brief Run the model asynchronously in a thread owned by intra op thread pool - * - * \param[in] session - * \param[in] run_options If nullptr, will use a default ::OrtRunOptions - * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names - * \param[in] input Array of ::OrtValue%s of the input values - * \param[in] input_len Number of elements in the input_names and inputs arrays - * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names - * \param[in] output_names_len Number of elements in the output_names and outputs array - * \param[out] output OrtValue* array of size output_names_len. - * On calling RunAsync, output[i] could either be a null or a pointer to a preallocated OrtValue. - * Later, the output array will be passed to run_async_callback with all null(s) filled with valid - * OrtValue pointer(s) allocated by onnxruntime. - * NOTE: it is customer's duty to finally release the output array and each of its member, - * regardless of whether the member (OrtValue*) is allocated by onnxruntime or preallocated by the customer. - * \param[in] run_async_callback Callback function on model run completion - * \param[in] user_data User data that pass back to run_async_callback - */ - ORT_API2_STATUS(RunAsync, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, - _In_reads_(input_len) const char* const* input_names, - _In_reads_(input_len) const OrtValue* const* input, size_t input_len, - _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, - _Inout_updates_all_(output_names_len) OrtValue** output, - _In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data); - - /** - * Update TensorRT EP provider option where its data type is pointer, for example 'user_compute_stream'. - * If the data type of the provider option can be represented by string please use UpdateTensorRTProviderOptions. - * - * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. - * - * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance - * \param key - Name of the provider option - * \param value - A pointer to the instance that will be assigned to this provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value); - - /** - * Get TensorRT EP provider option where its data type is pointer. - * If the data type of the provider option can be represented by string please use GetTensorRTProviderOptionsAsString. - * - * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance - * \param key - Name of the provider option - * \param ptr - A pointer to the instance that is kept by the provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr); - - /** - * Update CUDA EP provider option where its data type is pointer, for example 'user_compute_stream'. - * If the data type of the provider option can be represented by string please use UpdateCUDAProviderOptions. - * - * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. - * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance - * \param key - Name of the provider option - * \param value - A pointer to the instance that will be assigned to this provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(UpdateCUDAProviderOptionsWithValue, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _In_ void* value); - - /** - * Get CUDA EP provider option where its data type is pointer. - * If the data type of the provider option can be represented by string please use GetCUDAProviderOptionsAsString. - * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance - * \param key - Name of the provider option - * \param ptr - A pointer to the instance that is kept by the provider option - * - * \since Version 1.16. - */ - ORT_API2_STATUS(GetCUDAProviderOptionsByName, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _Outptr_ void** ptr); - - /** - * Get a EP resource. - * E.g. a cuda stream or a cublas handle - * - * \param context - Kernel context - * \param resouce_version - Version of the resource - * \param resource_id - Type of resource - * \param resource - A pointer to returned resource - * - * \since Version 1.16. - */ - ORT_API2_STATUS(KernelContext_GetResource, _In_ const OrtKernelContext* context, _In_ int resouce_version, _In_ int resource_id, _Outptr_ void** resource); - - /** \brief Set user logging function - * - * By default the logger created by the CreateEnv* functions is used to create the session logger as well. - * This function allows a user to override this default session logger with a logger of their own choosing. This way - * the user doesn't have to create a separate environment with a custom logger. This addresses the problem when - * the user already created an env but now wants to use a different logger for a specific session (for debugging or - * other reasons). - * - * \param[in] options - * \param[in] user_logging_function A pointer to a logging function. - * \param[in] user_logging_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `user_logging_function`. This parameter is optional. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.17. - */ - ORT_API2_STATUS(SetUserLoggingFunction, _Inout_ OrtSessionOptions* options, - _In_ OrtLoggingFunction user_logging_function, _In_opt_ void* user_logging_param); - - /** - * Get number of input from OrtShapeInferContext - * - * \param[in] context - * \param[out] out The number of inputs - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_GetInputCount, _In_ const OrtShapeInferContext* context, _Out_ size_t* out); - - /** - * Get type and shape info of an input - * - * \param[in] context - * \param[in] index The index of the input - * \param[out] info Type shape info of the input - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_GetInputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _Outptr_ OrtTensorTypeAndShapeInfo** info); - - /** - * Get attribute from OrtShapeInferContext. Note that OrtShapeInferContext is a per-node context, one could only read attribute from current node. - * - * \param[in] context - * \param[in] attr_name Name of the attribute - * \param[out] attr Handle of the attribute fetched - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_GetAttribute, _In_ const OrtShapeInferContext* context, _In_ const char* attr_name, _Outptr_ const OrtOpAttr** attr); - - /** - * Set type and shape info of an ouput - * - * \param[in] context - * \param[in] index The index of the ouput - * \param[out] info Type shape info of the output - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ShapeInferContext_SetOutputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _In_ const OrtTensorTypeAndShapeInfo* info); - - /** - * Set symbolic shape to type shape info - * - * \param[in] info Type shape info - * \param[in] dim_params Symbolic strings - * \param[in] dim_params_length Number of strings - * - * \since Version 1.17. - */ - ORT_API2_STATUS(SetSymbolicDimensions, _In_ OrtTensorTypeAndShapeInfo* info, _In_ const char* dim_params[], _In_ size_t dim_params_length); - - /** - * Read contents of an attribute to data - * - * \param[in] op_attr - * \param[in] type Attribute type - * \param[out] data Memory address to save raw content of the attribute - * \param[in] len Number of bytes allowed to store in data - * \param[out] out Number of bytes required to save the data when the call failed, or the real number of bytes saved to data on success - * - * \since Version 1.17. - */ - ORT_API2_STATUS(ReadOpAttr, _In_ const OrtOpAttr* op_attr, _In_ OrtOpAttrType type, _Inout_ void* data, _In_ size_t len, _Out_ size_t* out); - - /** \brief Set whether to use deterministic compute. - * - * Default is false. If set to true, this will enable deterministic compute for GPU kernels where possible. - * Note that this most likely will have a performance cost. - * - * \param[in] options - * \param[in] value - * - * \since Version 1.17. - */ - ORT_API2_STATUS(SetDeterministicCompute, _Inout_ OrtSessionOptions* options, bool value); - - /** - * Run fn in parallel - * - * \param[in] context - * \param[in] fn Function accepting usr_data and an integer as iterator - * \param[in] total The number of times fn is to be invoked - * \param[in] num_batch Number of batches by which the "total" is to be divided in maximum. When zero, there is no limit - * \param[in] usr_data User data to be passed back to fn - * - * \since Version 1.17. - */ - ORT_API2_STATUS(KernelContext_ParallelFor, _In_ const OrtKernelContext* context, _In_ void (*fn)(void*, size_t), _In_ size_t total, _In_ size_t num_batch, _In_ void* usr_data); - - /** \brief Append OpenVINO execution provider to the session options - * - * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. - * - * \param[in] options - * \param[in] provider_options_keys - * \param[in] provider_options_values - * \param[in] num_keys - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO_V2, - _In_ OrtSessionOptions* options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Append VitisAI provider to session options - * - * If VitisAI is not available (due to a non VitisAI enabled build, or if VitisAI is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] provider_options_keys - * \param[in] provider_options_values - * \param[in] num_keys - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_VitisAI, - _In_ OrtSessionOptions* options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Get scratch buffer from the corresponding allocator under the sepcific OrtMemoryInfo object. - * NOTE: callers are responsible to release this scratch buffer from the corresponding allocator - * \param[in] context OrtKernelContext instance - * \param[in] mem_info OrtMemoryInfo instance - * \param[in] count_or_bytes How many bytes is this scratch buffer - * \param[out] out A pointer to the scrach buffer - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out); - - /** \brief Get allocator from KernelInfo for a specific memory type. Please use C API ReleaseAllocator to release out object - * - * \param[in] info OrtKernelInfo instance - * \param[in] mem_type OrtMemType object - * \param[out] out A pointer to OrtAllocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out); - - /** \brief Replace initialized Tensors with external data with the provided files in memory - * - * The function will find the initialized TensorProtos with external data in the graph with the provided - * external file names and the file content in memory. The API gets the external file name, offset, data length - * from TensorProto, and locate the tensor data from the file in memory buffer. - * It creates a Tensor to replace the existing Tensor in graph. The replacement - * will occur before any of the optimizations take place. The data will be copied into the graph - * since TensorProto can't refer to the user provided buffers. - * - * \param[in] session options - * \param[in] external_initializer_file_names Array of null terminated UTF-8 encoded strings of the file names - * which holds the external initializers. - * \param[in] external_initializer_file_buffer_array Array of pointers to the buffer of the file content. - * The buffer can be freed after session creation. - * \param[in] external_initializer_file_lengths Array of size_t to indicate the length of file content - * \param[in] num_external_initializer_files Number of external files - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddExternalInitializersFromFilesInMemory, _In_ OrtSessionOptions* options, - _In_reads_(num_external_initializer_files) const ORTCHAR_T* const* external_initializer_file_names, - _In_reads_(num_external_initializer_files) char* const* external_initializer_file_buffer_array, - _In_reads_(num_external_initializer_files) const size_t* external_initializer_file_lengths, - size_t num_external_initializer_files); -}; - -/* - * Steps to use a custom op: - * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops - * 2 Create an OrtCustomOp structure for each op and add them to the domain - * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options - */ - -// Specifies some characteristics of inputs/outputs of custom ops: -// Specify if the inputs/outputs are one of: -// 1) Non-optional (input/output must be present in the node) -// 2) Optional (input/output may be absent in the node) -// 3) Variadic: A variadic input or output specifies N (i.e., the minimum arity) or more operands. -// Only the last input or output of a custom op may be marked as variadic. -// The homogeneity of the variadic input or output determines whether all operands must be of the same -// tensor element type. -typedef enum OrtCustomOpInputOutputCharacteristic { - INPUT_OUTPUT_REQUIRED = 0, - INPUT_OUTPUT_OPTIONAL, - INPUT_OUTPUT_VARIADIC, -} OrtCustomOpInputOutputCharacteristic; - -/* - * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by - * the implementor of the custom op. - */ -struct OrtCustomOp { - uint32_t version; // Must be initialized to ORT_API_VERSION - - // This callback creates the kernel, which is a user defined - // parameter that is passed to the Kernel* callbacks below. It is - // recommended to use CreateKernelV2 which allows for a safe error - // propagation by returning an OrtStatusPtr. - void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, - _In_ const OrtKernelInfo* info); - - // Returns the name of the op - const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op); - - // Returns the type of the execution provider, return nullptr to use CPU execution provider - const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op); - - // Returns the count and types of the input & output tensors - ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op); - ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op); - - // Perform a computation step. It is recommended to use - // KernelComputeV2 which allows for a safe error propagation by - // returning an OrtStatusPtr. - void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context); - void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel); - - // Returns the characteristics of the input & output tensors - OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetInputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetOutputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - - // Returns the memory type of the input tensors. This API allows the custom op - // to place the inputs on specific devices. By default, it returns - // OrtMemTypeDefault, which means the input is placed on the default device for - // the execution provider. If the inputs need to be with different memory tyeps, - // this function can be overridden to return the specific memory types. - OrtMemType(ORT_API_CALL* GetInputMemoryType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - - // Returns the minimum number of input arguments expected for the variadic input. - // Applicable only for custom ops that have a variadic input. - int(ORT_API_CALL* GetVariadicInputMinArity)(_In_ const struct OrtCustomOp* op); - - // Returns true (non-zero) if all arguments of a variadic input have to be of the same type (homogeneous), - // and false (zero) otherwise. - // Applicable only for custom ops that have a variadic input. - int(ORT_API_CALL* GetVariadicInputHomogeneity)(_In_ const struct OrtCustomOp* op); - - // Returns the minimum number of output values expected for the variadic output. - // Applicable only for custom ops that have a variadic output. - int(ORT_API_CALL* GetVariadicOutputMinArity)(_In_ const struct OrtCustomOp* op); - - // Returns true (non-zero) if all outputs values of a variadic output have to be of the same type (homogeneous), - // and false (zero) otherwise. - // Applicable only for custom ops that have a variadic output. - int(ORT_API_CALL* GetVariadicOutputHomogeneity)(_In_ const struct OrtCustomOp* op); - - // Create the kernel state which is passed to each compute call. - OrtStatusPtr(ORT_API_CALL* CreateKernelV2)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, - _In_ const OrtKernelInfo* info, - _Out_ void** kernel); - - // Perform the computation step. - OrtStatusPtr(ORT_API_CALL* KernelComputeV2)(_In_ void* op_kernel, _In_ OrtKernelContext* context); - - OrtStatusPtr(ORT_API_CALL* InferOutputShapeFn)(_In_ const struct OrtCustomOp* op, _In_ OrtShapeInferContext*); - - // Get start range - int(ORT_API_CALL* GetStartVersion)(_In_ const struct OrtCustomOp* op); - int(ORT_API_CALL* GetEndVersion)(_In_ const struct OrtCustomOp* op); - - // Get the inplace_map that defines which output can reuse which input - // Callers will provide 2 raw int* and pass in their address, this function will fill these 2 arrays - // when return, output (*output_index)[i] may reuse the input (*input_index[i]). - // The return value is the size of these 2 arrays. - // Callers are responsible to delete these 2 arrays after use by calling OrtCustomOp::ReleaseMayInplace(). - size_t(ORT_API_CALL* GetMayInplace)(_Out_ int** input_index, _Out_ int** output_index); - - // Release the pointer input_index and output_index allocated from GetMayInplace() function. - // If GetMayInplace() is defined, this function MUST be defined as well. - void(ORT_API_CALL* ReleaseMayInplace)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); - - // Same as GetMayInplace() and ReleaseMayInplace() - size_t(ORT_API_CALL* GetAliasMap)(_Out_ int** input_index, _Out_ int** output_index); - void(ORT_API_CALL* ReleaseAliasMap)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); -}; - -/* - * This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists - * - * \param device_id CUDA device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id); - -/* - * This is the old way to add the ROCm provider to the session, please use - * SessionOptionsAppendExecutionProvider_ROCM above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with - * HIP support and the ROCm provider shared library exists - * - * \param device_id HIP device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id); - -/* - * This is the old way to add the MIGraphX provider to the session, please use - * SessionOptionsAppendExecutionProvider_MIGraphX above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with - * HIP support and the MIGraphX provider shared library exists - * - * \param device_id HIP device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, int device_id); - -/* - * This is the old way to add the oneDNN provider to the session, please use - * SessionOptionsAppendExecutionProvider_oneDNN above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with - * oneDNN support and the oneDNN provider shared library exists - * - * \param use_arena zero: false. non-zero: true. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena); - -/* - * This is the old way to add the TensorRT provider to the session, please use SessionOptionsAppendExecutionProvider_TensorRT_V2 above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with TensorRT support and the TensorRT provider shared library exists - * - * \param device_id CUDA device id, starts from zero. - */ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id); - -#ifdef __cplusplus -} -#endif -/// @} diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_cxx_api.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_cxx_api.h deleted file mode 100644 index 8091fd4..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_cxx_api.h +++ /dev/null @@ -1,2387 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Summary: The Ort C++ API is a header only wrapper around the Ort C API. -// -// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors -// and automatically releasing resources in the destructors. The primary purpose of C++ API is exception safety so -// all the resources follow RAII and do not leak memory. -// -// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers. -// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). However, you can't use them -// until you assign an instance that actually holds an underlying object. -// -// For Ort objects only move assignment between objects is allowed, there are no copy constructors. -// Some objects have explicit 'Clone' methods for this purpose. -// -// ConstXXXX types are copyable since they do not own the underlying C object, so you can pass them to functions as arguments -// by value or by reference. ConstXXXX types are restricted to const only interfaces. -// -// UnownedXXXX are similar to ConstXXXX but also allow non-const interfaces. -// -// The lifetime of the corresponding owning object must eclipse the lifetimes of the ConstXXXX/UnownedXXXX types. They exists so you do not -// have to fallback to C types and the API with the usual pitfalls. In general, do not use C API from your C++ code. - -#pragma once -#include "onnxruntime_c_api.h" -#include "onnxruntime_float16.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef ORT_NO_EXCEPTIONS -#include -#endif - -/** \brief All C++ Onnxruntime APIs are defined inside this namespace - * - */ -namespace Ort { - -/** \brief All C++ methods that can fail will throw an exception of this type - * - * If ORT_NO_EXCEPTIONS is defined, then any error will result in a call to abort() - */ -struct Exception : std::exception { - Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {} - - OrtErrorCode GetOrtErrorCode() const { return code_; } - const char* what() const noexcept override { return message_.c_str(); } - - private: - std::string message_; - OrtErrorCode code_; -}; - -#ifdef ORT_NO_EXCEPTIONS -// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors. -// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW -#ifndef ORT_CXX_API_THROW -#define ORT_CXX_API_THROW(string, code) \ - do { \ - std::cerr << Ort::Exception(string, code) \ - .what() \ - << std::endl; \ - abort(); \ - } while (false) -#endif -#else -#define ORT_CXX_API_THROW(string, code) \ - throw Ort::Exception(string, code) -#endif - -// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi, -// it's in a template so that we can define a global variable in a header and make -// it transparent to the users of the API. -template -struct Global { - static const OrtApi* api_; -}; - -// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it. -template -#ifdef ORT_API_MANUAL_INIT -const OrtApi* Global::api_{}; -inline void InitApi() noexcept { Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); } - -// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is -// required by C++ APIs. -// -// Example mycustomop.cc: -// -// #define ORT_API_MANUAL_INIT -// #include -// #undef ORT_API_MANUAL_INIT -// -// OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) { -// Ort::InitApi(api_base->GetApi(ORT_API_VERSION)); -// // ... -// } -// -inline void InitApi(const OrtApi* api) noexcept { Global::api_ = api; } -#else -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(push) -// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers. -// Please define ORT_API_MANUAL_INIT if it conerns you. -#pragma warning(disable : 26426) -#endif -const OrtApi* Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(pop) -#endif -#endif - -/// This returns a reference to the OrtApi interface in use -inline const OrtApi& GetApi() noexcept { return *Global::api_; } - -/// -/// This function returns the onnxruntime version string -/// -/// version string major.minor.rev -std::string GetVersionString(); - -/// -/// This function returns the onnxruntime build information: including git branch, -/// git commit id, build type(Debug/Release/RelWithDebInfo) and cmake cpp flags. -/// -/// string -std::string GetBuildInfoString(); - -/// -/// This is a C++ wrapper for OrtApi::GetAvailableProviders() and -/// returns a vector of strings representing the available execution providers. -/// -/// vector of strings -std::vector GetAvailableProviders(); - -/** \brief IEEE 754 half-precision floating point data type - * - * \details This struct is used for converting float to float16 and back - * so the user could feed inputs and fetch outputs using these type. - * - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data. - * - * \code{.unparsed} - * // This example demonstrates converion from float to float16 - * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; - * std::vector fp16_values; - * fp16_values.reserve(std::size(values)); - * std::transform(std::begin(values), std::end(values), std::back_inserter(fp16_values), - * [](float value) { return Ort::Float16_t(value); }); - * - * \endcode - */ -struct Float16_t : onnxruntime_float16::Float16Impl { - private: - /// - /// Constructor from a 16-bit representation of a float16 value - /// No conversion is done here. - /// - /// 16-bit representation - constexpr explicit Float16_t(uint16_t v) noexcept { val = v; } - - public: - using Base = onnxruntime_float16::Float16Impl; - - /// - /// Default constructor - /// - Float16_t() = default; - - /// - /// Explicit conversion to uint16_t representation of float16. - /// - /// uint16_t bit representation of float16 - /// new instance of Float16_t - constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); } - - /// - /// __ctor from float. Float is converted into float16 16-bit representation. - /// - /// float value - explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); } - - /// - /// Converts float16 to float - /// - /// float representation of float16 value - float ToFloat() const noexcept { return Base::ToFloatImpl(); } - - /// - /// Checks if the value is negative - /// - /// true if negative - using Base::IsNegative; - - /// - /// Tests if the value is NaN - /// - /// true if NaN - using Base::IsNaN; - - /// - /// Tests if the value is finite - /// - /// true if finite - using Base::IsFinite; - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - using Base::IsPositiveInfinity; - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - using Base::IsNegativeInfinity; - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - using Base::IsInfinity; - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - using Base::IsNaNOrZero; - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - using Base::IsNormal; - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - using Base::IsSubnormal; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - using Base::Abs; - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - using Base::Negate; - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - using Base::AreZero; - - /// - /// User defined conversion operator. Converts Float16_t to float. - /// - explicit operator float() const noexcept { return ToFloat(); } - - using Base::operator==; - using Base::operator!=; - using Base::operator<; -}; - -static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match"); - -/** \brief bfloat16 (Brain Floating Point) data type - * - * \details This struct is used for converting float to bfloat16 and back - * so the user could feed inputs and fetch outputs using these type. - * - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data. - * - * \code{.unparsed} - * // This example demonstrates converion from float to float16 - * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; - * std::vector bfp16_values; - * bfp16_values.reserve(std::size(values)); - * std::transform(std::begin(values), std::end(values), std::back_inserter(bfp16_values), - * [](float value) { return Ort::BFloat16_t(value); }); - * - * \endcode - */ -struct BFloat16_t : onnxruntime_float16::BFloat16Impl { - private: - /// - /// Constructor from a uint16_t representation of bfloat16 - /// used in FromBits() to escape overload resolution issue with - /// constructor from float. - /// No conversion is done. - /// - /// 16-bit bfloat16 value - constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; } - - public: - using Base = onnxruntime_float16::BFloat16Impl; - - BFloat16_t() = default; - - /// - /// Explicit conversion to uint16_t representation of bfloat16. - /// - /// uint16_t bit representation of bfloat16 - /// new instance of BFloat16_t - static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); } - - /// - /// __ctor from float. Float is converted into bfloat16 16-bit representation. - /// - /// float value - explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); } - - /// - /// Converts bfloat16 to float - /// - /// float representation of bfloat16 value - float ToFloat() const noexcept { return Base::ToFloatImpl(); } - - /// - /// Checks if the value is negative - /// - /// true if negative - using Base::IsNegative; - - /// - /// Tests if the value is NaN - /// - /// true if NaN - using Base::IsNaN; - - /// - /// Tests if the value is finite - /// - /// true if finite - using Base::IsFinite; - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - using Base::IsPositiveInfinity; - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - using Base::IsNegativeInfinity; - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - using Base::IsInfinity; - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - using Base::IsNaNOrZero; - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - using Base::IsNormal; - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - using Base::IsSubnormal; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - using Base::Abs; - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - using Base::Negate; - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - using Base::AreZero; - - /// - /// User defined conversion operator. Converts BFloat16_t to float. - /// - explicit operator float() const noexcept { return ToFloat(); } - - // We do not have an inherited impl for the below operators - // as the internal class implements them a little differently - bool operator==(const BFloat16_t& rhs) const noexcept; - bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); } - bool operator<(const BFloat16_t& rhs) const noexcept; -}; - -static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); - -/** \brief float8e4m3fn (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E4M3FN_t { - uint8_t value; - constexpr Float8E4M3FN_t() noexcept : value(0) {} - constexpr Float8E4M3FN_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E4M3FN_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E4M3FN_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E4M3FN_t) == sizeof(uint8_t), "Sizes must match"); - -/** \brief float8e4m3fnuz (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E4M3FNUZ_t { - uint8_t value; - constexpr Float8E4M3FNUZ_t() noexcept : value(0) {} - constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E4M3FNUZ_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E4M3FNUZ_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E4M3FNUZ_t) == sizeof(uint8_t), "Sizes must match"); - -/** \brief float8e5m2 (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E5M2_t { - uint8_t value; - constexpr Float8E5M2_t() noexcept : value(0) {} - constexpr Float8E5M2_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E5M2_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E5M2_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E5M2_t) == sizeof(uint8_t), "Sizes must match"); - -/** \brief float8e5m2fnuz (Float8 Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint8_t. - * See https://onnx.ai/onnx/technical/float8.html for further details. - */ -struct Float8E5M2FNUZ_t { - uint8_t value; - constexpr Float8E5M2FNUZ_t() noexcept : value(0) {} - constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept : value(v) {} - constexpr operator uint8_t() const noexcept { return value; } - // nan values are treated like any other value for operator ==, != - constexpr bool operator==(const Float8E5M2FNUZ_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float8E5M2FNUZ_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float8E5M2FNUZ_t) == sizeof(uint8_t), "Sizes must match"); - -namespace detail { -// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type -// This can't be done in the C API since C doesn't have function overloading. -#define ORT_DEFINE_RELEASE(NAME) \ - inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); } - -ORT_DEFINE_RELEASE(Allocator); -ORT_DEFINE_RELEASE(MemoryInfo); -ORT_DEFINE_RELEASE(CustomOpDomain); -ORT_DEFINE_RELEASE(ThreadingOptions); -ORT_DEFINE_RELEASE(Env); -ORT_DEFINE_RELEASE(RunOptions); -ORT_DEFINE_RELEASE(Session); -ORT_DEFINE_RELEASE(SessionOptions); -ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo); -ORT_DEFINE_RELEASE(SequenceTypeInfo); -ORT_DEFINE_RELEASE(MapTypeInfo); -ORT_DEFINE_RELEASE(TypeInfo); -ORT_DEFINE_RELEASE(Value); -ORT_DEFINE_RELEASE(ModelMetadata); -ORT_DEFINE_RELEASE(IoBinding); -ORT_DEFINE_RELEASE(ArenaCfg); -ORT_DEFINE_RELEASE(Status); -ORT_DEFINE_RELEASE(OpAttr); -ORT_DEFINE_RELEASE(Op); -ORT_DEFINE_RELEASE(KernelInfo); - -#undef ORT_DEFINE_RELEASE - -/** \brief This is a tagging template type. Use it with Base to indicate that the C++ interface object - * has no ownership of the underlying C object. - */ -template -struct Unowned { - using Type = T; -}; - -/** \brief Used internally by the C++ API. C++ wrapper types inherit from this. - * This is a zero cost abstraction to wrap the C API objects and delete them on destruction. - * - * All of the C++ classes - * a) serve as containers for pointers to objects that are created by the underlying C API. - * Their size is just a pointer size, no need to dynamically allocate them. Use them by value. - * b) Each of struct XXXX, XXX instances function as smart pointers to the underlying C API objects. - * they would release objects owned automatically when going out of scope, they are move-only. - * c) ConstXXXX and UnownedXXX structs function as non-owning, copyable containers for the above pointers. - * ConstXXXX allow calling const interfaces only. They give access to objects that are owned by somebody else - * such as Onnxruntime or instances of XXXX classes. - * d) serve convenient interfaces that return C++ objects and further enhance exception and type safety so they can be used - * in C++ code. - * - */ - -/// -/// This is a non-const pointer holder that is move-only. Disposes of the pointer on destruction. -/// -template -struct Base { - using contained_type = T; - - constexpr Base() = default; - constexpr explicit Base(contained_type* p) noexcept : p_{p} {} - ~Base() { OrtRelease(p_); } - - Base(const Base&) = delete; - Base& operator=(const Base&) = delete; - - Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } - Base& operator=(Base&& v) noexcept { - OrtRelease(p_); - p_ = v.release(); - return *this; - } - - constexpr operator contained_type*() const noexcept { return p_; } - - /// \brief Relinquishes ownership of the contained C object pointer - /// The underlying object is not destroyed - contained_type* release() { - T* p = p_; - p_ = nullptr; - return p; - } - - protected: - contained_type* p_{}; -}; - -// Undefined. For const types use Base> -template -struct Base; - -/// -/// Covers unowned pointers owned by either the ORT -/// or some other instance of CPP wrappers. -/// Used for ConstXXX and UnownedXXXX types that are copyable. -/// Also convenient to wrap raw OrtXX pointers . -/// -/// -template -struct Base> { - using contained_type = typename Unowned::Type; - - constexpr Base() = default; - constexpr explicit Base(contained_type* p) noexcept : p_{p} {} - - ~Base() = default; - - Base(const Base&) = default; - Base& operator=(const Base&) = default; - - Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } - Base& operator=(Base&& v) noexcept { - p_ = nullptr; - std::swap(p_, v.p_); - return *this; - } - - constexpr operator contained_type*() const noexcept { return p_; } - - protected: - contained_type* p_{}; -}; - -// Light functor to release memory with OrtAllocator -struct AllocatedFree { - OrtAllocator* allocator_; - explicit AllocatedFree(OrtAllocator* allocator) - : allocator_(allocator) {} - void operator()(void* ptr) const { - if (ptr) allocator_->Free(allocator_, ptr); - } -}; - -} // namespace detail - -struct AllocatorWithDefaultOptions; -struct Env; -struct TypeInfo; -struct Value; -struct ModelMetadata; - -/** \brief unique_ptr typedef used to own strings allocated by OrtAllocators - * and release them at the end of the scope. The lifespan of the given allocator - * must eclipse the lifespan of AllocatedStringPtr instance - */ -using AllocatedStringPtr = std::unique_ptr; - -/** \brief The Status that holds ownership of OrtStatus received from C API - * Use it to safely destroy OrtStatus* returned from the C API. Use appropriate - * constructors to construct an instance of a Status object from exceptions. - */ -struct Status : detail::Base { - explicit Status(std::nullptr_t) noexcept {} ///< Create an empty object, must be assigned a valid one to be used - explicit Status(OrtStatus* status) noexcept; ///< Takes ownership of OrtStatus instance returned from the C API. - explicit Status(const Exception&) noexcept; ///< Creates status instance out of exception - explicit Status(const std::exception&) noexcept; ///< Creates status instance out of exception - Status(const char* message, OrtErrorCode code) noexcept; ///< Creates status instance out of null-terminated string message. - std::string GetErrorMessage() const; - OrtErrorCode GetErrorCode() const; - bool IsOK() const noexcept; ///< Returns true if instance represents an OK (non-error) status. -}; - -/** \brief The ThreadingOptions - * - * The ThreadingOptions used for set global threadpools' options of The Env. - */ -struct ThreadingOptions : detail::Base { - /// \brief Wraps OrtApi::CreateThreadingOptions - ThreadingOptions(); - - /// \brief Wraps OrtApi::SetGlobalIntraOpNumThreads - ThreadingOptions& SetGlobalIntraOpNumThreads(int intra_op_num_threads); - - /// \brief Wraps OrtApi::SetGlobalInterOpNumThreads - ThreadingOptions& SetGlobalInterOpNumThreads(int inter_op_num_threads); - - /// \brief Wraps OrtApi::SetGlobalSpinControl - ThreadingOptions& SetGlobalSpinControl(int allow_spinning); - - /// \brief Wraps OrtApi::SetGlobalDenormalAsZero - ThreadingOptions& SetGlobalDenormalAsZero(); - - /// \brief Wraps OrtApi::SetGlobalCustomCreateThreadFn - ThreadingOptions& SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /// \brief Wraps OrtApi::SetGlobalCustomThreadCreationOptions - ThreadingOptions& SetGlobalCustomThreadCreationOptions(void* ort_custom_thread_creation_options); - - /// \brief Wraps OrtApi::SetGlobalCustomJoinThreadFn - ThreadingOptions& SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); -}; - -/** \brief The Env (Environment) - * - * The Env holds the logging state used by all other objects. - * Note: One Env must be created before using any other Onnxruntime functionality - */ -struct Env : detail::Base { - explicit Env(std::nullptr_t) {} ///< Create an empty Env object, must be assigned a valid one to be used - - /// \brief Wraps OrtApi::CreateEnv - Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief Wraps OrtApi::CreateEnvWithCustomLogger - Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param); - - /// \brief Wraps OrtApi::CreateEnvWithGlobalThreadPools - Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief Wraps OrtApi::CreateEnvWithCustomLoggerAndGlobalThreadPools - Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, - OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief C Interop Helper - explicit Env(OrtEnv* p) : Base{p} {} - - Env& EnableTelemetryEvents(); ///< Wraps OrtApi::EnableTelemetryEvents - Env& DisableTelemetryEvents(); ///< Wraps OrtApi::DisableTelemetryEvents - - Env& UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level); ///< Wraps OrtApi::UpdateEnvWithCustomLogLevel - - Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocator - - Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, const std::unordered_map& options, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocatorV2 -}; - -/** \brief Custom Op Domain - * - */ -struct CustomOpDomain : detail::Base { - explicit CustomOpDomain(std::nullptr_t) {} ///< Create an empty CustomOpDomain object, must be assigned a valid one to be used - - /// \brief Wraps OrtApi::CreateCustomOpDomain - explicit CustomOpDomain(const char* domain); - - // This does not take ownership of the op, simply registers it. - void Add(const OrtCustomOp* op); ///< Wraps CustomOpDomain_Add -}; - -/** \brief RunOptions - * - */ -struct RunOptions : detail::Base { - explicit RunOptions(std::nullptr_t) {} ///< Create an empty RunOptions object, must be assigned a valid one to be used - RunOptions(); ///< Wraps OrtApi::CreateRunOptions - - RunOptions& SetRunLogVerbosityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel - int GetRunLogVerbosityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel - - RunOptions& SetRunLogSeverityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogSeverityLevel - int GetRunLogSeverityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogSeverityLevel - - RunOptions& SetRunTag(const char* run_tag); ///< wraps OrtApi::RunOptionsSetRunTag - const char* GetRunTag() const; ///< Wraps OrtApi::RunOptionsGetRunTag - - RunOptions& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddRunConfigEntry - - /** \brief Terminates all currently executing Session::Run calls that were made using this RunOptions instance - * - * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error - * Wraps OrtApi::RunOptionsSetTerminate - */ - RunOptions& SetTerminate(); - - /** \brief Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without it instantly terminating - * - * Wraps OrtApi::RunOptionsUnsetTerminate - */ - RunOptions& UnsetTerminate(); -}; - -namespace detail { -// Utility function that returns a SessionOption config entry key for a specific custom operator. -// Ex: custom_op.[custom_op_name].[config] -std::string MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config); -} // namespace detail - -/// -/// Class that represents session configuration entries for one or more custom operators. -/// -/// Example: -/// Ort::CustomOpConfigs op_configs; -/// op_configs.AddConfig("my_custom_op", "device_type", "CPU"); -/// -/// Passed to Ort::SessionOptions::RegisterCustomOpsLibrary. -/// -struct CustomOpConfigs { - CustomOpConfigs() = default; - ~CustomOpConfigs() = default; - CustomOpConfigs(const CustomOpConfigs&) = default; - CustomOpConfigs& operator=(const CustomOpConfigs&) = default; - CustomOpConfigs(CustomOpConfigs&& o) = default; - CustomOpConfigs& operator=(CustomOpConfigs&& o) = default; - - /** \brief Adds a session configuration entry/value for a specific custom operator. - * - * \param custom_op_name The name of the custom operator for which to add a configuration entry. - * Must match the name returned by the CustomOp's GetName() method. - * \param config_key The name of the configuration entry. - * \param config_value The value of the configuration entry. - * \return A reference to this object to enable call chaining. - */ - CustomOpConfigs& AddConfig(const char* custom_op_name, const char* config_key, const char* config_value); - - /** \brief Returns a flattened map of custom operator configuration entries and their values. - * - * The keys has been flattened to include both the custom operator name and the configuration entry key name. - * For example, a prior call to AddConfig("my_op", "key", "value") corresponds to the flattened key/value pair - * {"my_op.key", "value"}. - * - * \return An unordered map of flattened configurations. - */ - const std::unordered_map& GetFlattenedConfigs() const; - - private: - std::unordered_map flat_configs_; -}; - -/** \brief Options object used when creating a new Session object - * - * Wraps ::OrtSessionOptions object and methods - */ - -struct SessionOptions; - -namespace detail { -// we separate const-only methods because passing const ptr to non-const methods -// is only discovered when inline methods are compiled which is counter-intuitive -template -struct ConstSessionOptionsImpl : Base { - using B = Base; - using B::B; - - SessionOptions Clone() const; ///< Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions - - std::string GetConfigEntry(const char* config_key) const; ///< Wraps OrtApi::GetSessionConfigEntry - bool HasConfigEntry(const char* config_key) const; ///< Wraps OrtApi::HasSessionConfigEntry - std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def); -}; - -template -struct SessionOptionsImpl : ConstSessionOptionsImpl { - using B = ConstSessionOptionsImpl; - using B::B; - - SessionOptionsImpl& SetIntraOpNumThreads(int intra_op_num_threads); ///< Wraps OrtApi::SetIntraOpNumThreads - SessionOptionsImpl& SetInterOpNumThreads(int inter_op_num_threads); ///< Wraps OrtApi::SetInterOpNumThreads - SessionOptionsImpl& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); ///< Wraps OrtApi::SetSessionGraphOptimizationLevel - SessionOptionsImpl& SetDeterministicCompute(bool value); ///< Wraps OrtApi::SetDeterministicCompute - - SessionOptionsImpl& EnableCpuMemArena(); ///< Wraps OrtApi::EnableCpuMemArena - SessionOptionsImpl& DisableCpuMemArena(); ///< Wraps OrtApi::DisableCpuMemArena - - SessionOptionsImpl& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file); ///< Wraps OrtApi::SetOptimizedModelFilePath - - SessionOptionsImpl& EnableProfiling(const ORTCHAR_T* profile_file_prefix); ///< Wraps OrtApi::EnableProfiling - SessionOptionsImpl& DisableProfiling(); ///< Wraps OrtApi::DisableProfiling - - SessionOptionsImpl& EnableOrtCustomOps(); ///< Wraps OrtApi::EnableOrtCustomOps - - SessionOptionsImpl& EnableMemPattern(); ///< Wraps OrtApi::EnableMemPattern - SessionOptionsImpl& DisableMemPattern(); ///< Wraps OrtApi::DisableMemPattern - - SessionOptionsImpl& SetExecutionMode(ExecutionMode execution_mode); ///< Wraps OrtApi::SetSessionExecutionMode - - SessionOptionsImpl& SetLogId(const char* logid); ///< Wraps OrtApi::SetSessionLogId - SessionOptionsImpl& SetLogSeverityLevel(int level); ///< Wraps OrtApi::SetSessionLogSeverityLevel - - SessionOptionsImpl& Add(OrtCustomOpDomain* custom_op_domain); ///< Wraps OrtApi::AddCustomOpDomain - - SessionOptionsImpl& DisablePerSessionThreads(); ///< Wraps OrtApi::DisablePerSessionThreads - - SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddSessionConfigEntry - - SessionOptionsImpl& AddInitializer(const char* name, const OrtValue* ort_val); ///< Wraps OrtApi::AddInitializer - SessionOptionsImpl& AddExternalInitializers(const std::vector& names, const std::vector& ort_values); ///< Wraps OrtApi::AddExternalInitializers - SessionOptionsImpl& AddExternalInitializersFromFilesInMemory(const std::vector>& external_initializer_file_names, - const std::vector& external_initializer_file_buffer_array, - const std::vector& external_initializer_file_lengths); ///< Wraps OrtApi::AddExternalInitializersFromFilesInMemory - - SessionOptionsImpl& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA - SessionOptionsImpl& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA_V2 - SessionOptionsImpl& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_ROCM - SessionOptionsImpl& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO_V2 - SessionOptionsImpl& AppendExecutionProvider_OpenVINO_V2(const std::unordered_map& provider_options = {}); - SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN - SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options); - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl - SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options); - /// Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports QNN, SNPE and XNNPACK. - SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name, - const std::unordered_map& provider_options = {}); - - SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn - SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options); ///< Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions - SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn - - ///< Registers the custom operator from the specified shared library via OrtApi::RegisterCustomOpsLibrary_V2. - ///< The custom operator configurations are optional. If provided, custom operator configs are set via - ///< OrtApi::AddSessionConfigEntry. - SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, const CustomOpConfigs& custom_op_configs = {}); - - SessionOptionsImpl& RegisterCustomOpsUsingFunction(const char* function_name); ///< Wraps OrtApi::RegisterCustomOpsUsingFunction - - ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_VitisAI - SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map& provider_options = {}); -}; -} // namespace detail - -using UnownedSessionOptions = detail::SessionOptionsImpl>; -using ConstSessionOptions = detail::ConstSessionOptionsImpl>; - -/** \brief Wrapper around ::OrtSessionOptions - * - */ -struct SessionOptions : detail::SessionOptionsImpl { - explicit SessionOptions(std::nullptr_t) {} ///< Create an empty SessionOptions object, must be assigned a valid one to be used - SessionOptions(); ///< Wraps OrtApi::CreateSessionOptions - explicit SessionOptions(OrtSessionOptions* p) : SessionOptionsImpl{p} {} ///< Used for interop with the C API - UnownedSessionOptions GetUnowned() const { return UnownedSessionOptions{this->p_}; } - ConstSessionOptions GetConst() const { return ConstSessionOptions{this->p_}; } -}; - -/** \brief Wrapper around ::OrtModelMetadata - * - */ -struct ModelMetadata : detail::Base { - explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used - explicit ModelMetadata(OrtModelMetadata* p) : Base{p} {} ///< Used for interop with the C API - - /** \brief Returns a copy of the producer name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetProducerNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetProducerName - - /** \brief Returns a copy of the graph name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetGraphNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphName - - /** \brief Returns a copy of the domain name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetDomainAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDomain - - /** \brief Returns a copy of the description. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDescription - - /** \brief Returns a copy of the graph description. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphDescription - - /** \brief Returns a vector of copies of the custom metadata keys. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance std::vector of smart pointers that would deallocate the buffers when out of scope. - * The OrtAllocator instance must be valid at the point of memory release. - */ - std::vector GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetCustomMetadataMapKeys - - /** \brief Looks up a value by a key in the Custom Metadata map - * - * \param key zero terminated string key to lookup - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * maybe nullptr if key is not found. - * - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataLookupCustomMetadataMap - - int64_t GetVersion() const; ///< Wraps OrtApi::ModelMetadataGetVersion -}; - -struct IoBinding; - -namespace detail { - -// we separate const-only methods because passing const ptr to non-const methods -// is only discovered when inline methods are compiled which is counter-intuitive -template -struct ConstSessionImpl : Base { - using B = Base; - using B::B; - - size_t GetInputCount() const; ///< Returns the number of model inputs - size_t GetOutputCount() const; ///< Returns the number of model outputs - size_t GetOverridableInitializerCount() const; ///< Returns the number of inputs that have defaults that can be overridden - - /** \brief Returns a copy of input name at the specified index. - * - * \param index must less than the value returned by GetInputCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator* allocator) const; - - /** \brief Returns a copy of output name at then specified index. - * - * \param index must less than the value returned by GetOutputCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const; - - /** \brief Returns a copy of the overridable initializer name at then specified index. - * - * \param index must less than the value returned by GetOverridableInitializerCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionGetOverridableInitializerName - - uint64_t GetProfilingStartTimeNs() const; ///< Wraps OrtApi::SessionGetProfilingStartTimeNs - ModelMetadata GetModelMetadata() const; ///< Wraps OrtApi::SessionGetModelMetadata - - TypeInfo GetInputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetInputTypeInfo - TypeInfo GetOutputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOutputTypeInfo - TypeInfo GetOverridableInitializerTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOverridableInitializerTypeInfo -}; - -template -struct SessionImpl : ConstSessionImpl { - using B = ConstSessionImpl; - using B::B; - - /** \brief Run the model returning results in an Ort allocated vector. - * - * Wraps OrtApi::Run - * - * The caller provides a list of inputs and a list of the desired outputs to return. - * - * See the output logs for more information on warnings/errors that occur while processing the model. - * Common errors are.. (TODO) - * - * \param[in] run_options - * \param[in] input_names Array of null terminated strings of length input_count that is the list of input names - * \param[in] input_values Array of Value objects of length input_count that is the list of input values - * \param[in] input_count Number of inputs (the size of the input_names & input_values arrays) - * \param[in] output_names Array of C style strings of length output_count that is the list of output names - * \param[in] output_count Number of outputs (the size of the output_names array) - * \return A std::vector of Value objects that directly maps to the output_names array (eg. output_name[0] is the first entry of the returned vector) - */ - std::vector Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, size_t output_count); - - /** \brief Run the model returning results in user provided outputs - * Same as Run(const RunOptions&, const char* const*, const Value*, size_t,const char* const*, size_t) - */ - void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count); - - void Run(const RunOptions& run_options, const IoBinding&); ///< Wraps OrtApi::RunWithBinding - - /** \brief Run the model asynchronously in a thread owned by intra op thread pool - * - * Wraps OrtApi::RunAsync - * - * \param[in] run_options - * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names - * \param[in] input_values Array of Value objects of length input_count - * \param[in] input_count Number of elements in the input_names and inputs arrays - * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names - * \param[out] output_values Array of provided Values to be filled with outputs. - * On calling RunAsync, output_values[i] could either be initialized by a null pointer or a preallocated OrtValue*. - * Later, on invoking the callback, each output_values[i] of null will be filled with an OrtValue* allocated by onnxruntime. - * Then, an OrtValue** pointer will be casted from output_values, and pass to the callback. - * NOTE: it is customer's duty to finally release output_values and each of its member, - * regardless of whether the member (Ort::Value) is allocated by onnxruntime or preallocated by the customer. - * \param[in] output_count Number of elements in the output_names and outputs array - * \param[in] callback Callback function on model run completion - * \param[in] user_data User data that pass back to the callback - */ - void RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data); - - /** \brief End profiling and return a copy of the profiling file name. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr EndProfilingAllocated(OrtAllocator* allocator); ///< Wraps OrtApi::SessionEndProfiling -}; - -} // namespace detail - -using ConstSession = detail::ConstSessionImpl>; -using UnownedSession = detail::SessionImpl>; - -/** \brief Wrapper around ::OrtSession - * - */ -struct Session : detail::SessionImpl { - explicit Session(std::nullptr_t) {} ///< Create an empty Session object, must be assigned a valid one to be used - Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); ///< Wraps OrtApi::CreateSession - Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container); ///< Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer - Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options); ///< Wraps OrtApi::CreateSessionFromArray - Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container); ///< Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer - - ConstSession GetConst() const { return ConstSession{this->p_}; } - UnownedSession GetUnowned() const { return UnownedSession{this->p_}; } -}; - -namespace detail { -template -struct MemoryInfoImpl : Base { - using B = Base; - using B::B; - - std::string GetAllocatorName() const; - OrtAllocatorType GetAllocatorType() const; - int GetDeviceId() const; - OrtMemoryInfoDeviceType GetDeviceType() const; - OrtMemType GetMemoryType() const; - - template - bool operator==(const MemoryInfoImpl& o) const; -}; -} // namespace detail - -// Const object holder that does not own the underlying object -using ConstMemoryInfo = detail::MemoryInfoImpl>; - -/** \brief Wrapper around ::OrtMemoryInfo - * - */ -struct MemoryInfo : detail::MemoryInfoImpl { - static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1); - explicit MemoryInfo(std::nullptr_t) {} ///< No instance is created - explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl{p} {} ///< Take ownership of a pointer created by C Api - MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type); - ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; } -}; - -namespace detail { -template -struct TensorTypeAndShapeInfoImpl : Base { - using B = Base; - using B::B; - - ONNXTensorElementDataType GetElementType() const; ///< Wraps OrtApi::GetTensorElementType - size_t GetElementCount() const; ///< Wraps OrtApi::GetTensorShapeElementCount - - size_t GetDimensionsCount() const; ///< Wraps OrtApi::GetDimensionsCount - - /** \deprecated use GetShape() returning std::vector - * [[deprecated]] - * This interface is unsafe to use - */ - [[deprecated("use GetShape()")]] void GetDimensions(int64_t* values, size_t values_count) const; ///< Wraps OrtApi::GetDimensions - - void GetSymbolicDimensions(const char** values, size_t values_count) const; ///< Wraps OrtApi::GetSymbolicDimensions - - std::vector GetShape() const; ///< Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape -}; - -} // namespace detail - -using ConstTensorTypeAndShapeInfo = detail::TensorTypeAndShapeInfoImpl>; - -/** \brief Wrapper around ::OrtTensorTypeAndShapeInfo - * - */ -struct TensorTypeAndShapeInfo : detail::TensorTypeAndShapeInfoImpl { - explicit TensorTypeAndShapeInfo(std::nullptr_t) {} ///< Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used - explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {} ///< Used for interop with the C API - ConstTensorTypeAndShapeInfo GetConst() const { return ConstTensorTypeAndShapeInfo{this->p_}; } -}; - -namespace detail { -template -struct SequenceTypeInfoImpl : Base { - using B = Base; - using B::B; - TypeInfo GetSequenceElementType() const; ///< Wraps OrtApi::GetSequenceElementType -}; - -} // namespace detail - -using ConstSequenceTypeInfo = detail::SequenceTypeInfoImpl>; - -/** \brief Wrapper around ::OrtSequenceTypeInfo - * - */ -struct SequenceTypeInfo : detail::SequenceTypeInfoImpl { - explicit SequenceTypeInfo(std::nullptr_t) {} ///< Create an empty SequenceTypeInfo object, must be assigned a valid one to be used - explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl{p} {} ///< Used for interop with the C API - ConstSequenceTypeInfo GetConst() const { return ConstSequenceTypeInfo{this->p_}; } -}; - -namespace detail { -template -struct OptionalTypeInfoImpl : Base { - using B = Base; - using B::B; - TypeInfo GetOptionalElementType() const; ///< Wraps OrtApi::CastOptionalTypeToContainedTypeInfo -}; - -} // namespace detail - -// This is always owned by the TypeInfo and can only be obtained from it. -using ConstOptionalTypeInfo = detail::OptionalTypeInfoImpl>; - -namespace detail { -template -struct MapTypeInfoImpl : detail::Base { - using B = Base; - using B::B; - ONNXTensorElementDataType GetMapKeyType() const; ///< Wraps OrtApi::GetMapKeyType - TypeInfo GetMapValueType() const; ///< Wraps OrtApi::GetMapValueType -}; - -} // namespace detail - -using ConstMapTypeInfo = detail::MapTypeInfoImpl>; - -/** \brief Wrapper around ::OrtMapTypeInfo - * - */ -struct MapTypeInfo : detail::MapTypeInfoImpl { - explicit MapTypeInfo(std::nullptr_t) {} ///< Create an empty MapTypeInfo object, must be assigned a valid one to be used - explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl{p} {} ///< Used for interop with the C API - ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; } -}; - -namespace detail { -template -struct TypeInfoImpl : detail::Base { - using B = Base; - using B::B; - - ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; ///< Wraps OrtApi::CastTypeInfoToTensorInfo - ConstSequenceTypeInfo GetSequenceTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToSequenceTypeInfo - ConstMapTypeInfo GetMapTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToMapTypeInfo - ConstOptionalTypeInfo GetOptionalTypeInfo() const; ///< wraps OrtApi::CastTypeInfoToOptionalTypeInfo - - ONNXType GetONNXType() const; -}; -} // namespace detail - -/// -/// Contains a constant, unowned OrtTypeInfo that can be copied and passed around by value. -/// Provides access to const OrtTypeInfo APIs. -/// -using ConstTypeInfo = detail::TypeInfoImpl>; - -/// -/// Type information that may contain either TensorTypeAndShapeInfo or -/// the information about contained sequence or map depending on the ONNXType. -/// -struct TypeInfo : detail::TypeInfoImpl { - explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used - explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl{p} {} ///< C API Interop - - ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; } -}; - -namespace detail { -// This structure is used to feed sparse tensor values -// information for use with FillSparseTensor() API -// if the data type for the sparse tensor values is numeric -// use data.p_data, otherwise, use data.str pointer to feed -// values. data.str is an array of const char* that are zero terminated. -// number of strings in the array must match shape size. -// For fully sparse tensors use shape {0} and set p_data/str -// to nullptr. -struct OrtSparseValuesParam { - const int64_t* values_shape; - size_t values_shape_len; - union { - const void* p_data; - const char** str; - } data; -}; - -// Provides a way to pass shape in a single -// argument -struct Shape { - const int64_t* shape; - size_t shape_len; -}; - -template -struct ConstValueImpl : Base { - using B = Base; - using B::B; - - /// - /// Obtains a pointer to a user defined data for experimental purposes - /// - template - void GetOpaqueData(const char* domain, const char* type_name, R&) const; ///< Wraps OrtApi::GetOpaqueValue - - bool IsTensor() const; ///< Returns true if Value is a tensor, false for other types like map/sequence/etc - bool HasValue() const; /// < Return true if OrtValue contains data and returns false if the OrtValue is a None - - size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements - Value GetValue(int index, OrtAllocator* allocator) const; - - /// - /// This API returns a full length of string data contained within either a tensor or a sparse Tensor. - /// For sparse tensor it returns a full length of stored non-empty strings (values). The API is useful - /// for allocating necessary memory and calling GetStringTensorContent(). - /// - /// total length of UTF-8 encoded bytes contained. No zero terminators counted. - size_t GetStringTensorDataLength() const; - - /// - /// The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor - /// into a supplied buffer. Use GetStringTensorDataLength() to find out the length of the buffer to allocate. - /// The user must also allocate offsets buffer with the number of entries equal to that of the contained - /// strings. - /// - /// Strings are always assumed to be on CPU, no X-device copy. - /// - /// user allocated buffer - /// length in bytes of the allocated buffer - /// a pointer to the offsets user allocated buffer - /// count of offsets, must be equal to the number of strings contained. - /// that can be obtained from the shape of the tensor or from GetSparseTensorValuesTypeAndShapeInfo() - /// for sparse tensors - void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const; - - /// - /// Returns a const typed pointer to the tensor contained data. - /// No type checking is performed, the caller must ensure the type matches the tensor type. - /// - /// - /// const pointer to data, no copies made - template - const R* GetTensorData() const; ///< Wraps OrtApi::GetTensorMutableData /// - - /// - /// Returns a non-typed pointer to a tensor contained data. - /// - /// const pointer to data, no copies made - const void* GetTensorRawData() const; - - /// - /// The API returns type information for data contained in a tensor. For sparse - /// tensors it returns type information for contained non-zero values. - /// It returns dense shape for sparse tensors. - /// - /// TypeInfo - TypeInfo GetTypeInfo() const; - - /// - /// The API returns type information for data contained in a tensor. For sparse - /// tensors it returns type information for contained non-zero values. - /// It returns dense shape for sparse tensors. - /// - /// TensorTypeAndShapeInfo - TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; - - /// - /// This API returns information about the memory allocation used to hold data. - /// - /// Non owning instance of MemoryInfo - ConstMemoryInfo GetTensorMemoryInfo() const; - - /// - /// The API copies UTF-8 encoded bytes for the requested string element - /// contained within a tensor or a sparse tensor into a provided buffer. - /// Use GetStringTensorElementLength() to obtain the length of the buffer to allocate. - /// - /// - /// - /// - void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const; - - /// - /// Returns string tensor UTF-8 encoded string element. - /// Use of this API is recommended over GetStringTensorElement() that takes void* buffer pointer. - /// - /// - /// std::string - std::string GetStringTensorElement(size_t element_index) const; - - /// - /// The API returns a byte length of UTF-8 encoded string element - /// contained in either a tensor or a spare tensor values. - /// - /// - /// byte length for the specified string element - size_t GetStringTensorElementLength(size_t element_index) const; - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// The API returns the sparse data format this OrtValue holds in a sparse tensor. - /// If the sparse tensor was not fully constructed, i.e. Use*() or Fill*() API were not used - /// the value returned is ORT_SPARSE_UNDEFINED. - /// - /// Format enum - OrtSparseFormat GetSparseFormat() const; - - /// - /// The API returns type and shape information for stored non-zero values of the - /// sparse tensor. Use GetSparseTensorValues() to obtain values buffer pointer. - /// - /// TensorTypeAndShapeInfo values information - TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const; - - /// - /// The API returns type and shape information for the specified indices. Each supported - /// indices have their own enum values even if a give format has more than one kind of indices. - /// Use GetSparseTensorIndicesData() to obtain pointer to indices buffer. - /// - /// enum requested - /// type and shape information - TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const; - - /// - /// The API retrieves a pointer to the internal indices buffer. The API merely performs - /// a convenience data type casting on the return type pointer. Make sure you are requesting - /// the right type, use GetSparseTensorIndicesTypeShapeInfo(); - /// - /// type to cast to - /// requested indices kind - /// number of indices entries - /// Pinter to the internal sparse tensor buffer containing indices. Do not free this pointer. - template - const R* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const; - - /// - /// Returns true if the OrtValue contains a sparse tensor - /// - /// - bool IsSparseTensor() const; - - /// - /// The API returns a pointer to an internal buffer of the sparse tensor - /// containing non-zero values. The API merely does casting. Make sure you - /// are requesting the right data type by calling GetSparseTensorValuesTypeAndShapeInfo() - /// first. - /// - /// numeric data types only. Use GetStringTensor*() to retrieve strings. - /// a pointer to the internal values buffer. Do not free this pointer. - template - const R* GetSparseTensorValues() const; - -#endif -}; - -template -struct ValueImpl : ConstValueImpl { - using B = ConstValueImpl; - using B::B; - - /// - /// Returns a non-const typed pointer to an OrtValue/Tensor contained buffer - /// No type checking is performed, the caller must ensure the type matches the tensor type. - /// - /// non-const pointer to data, no copies made - template - R* GetTensorMutableData(); - - /// - /// Returns a non-typed non-const pointer to a tensor contained data. - /// - /// pointer to data, no copies made - void* GetTensorMutableRawData(); - - /// - // Obtain a reference to an element of data at the location specified - /// by the vector of dims. - /// - /// - /// [in] expressed by a vecotr of dimensions offsets - /// - template - R& At(const std::vector& location); - - /// - /// Set all strings at once in a string tensor - /// - /// [in] An array of strings. Each string in this array must be null terminated. - /// [in] Count of strings in s (Must match the size of \p value's tensor shape) - void FillStringTensor(const char* const* s, size_t s_len); - - /// - /// Set a single string in a string tensor - /// - /// [in] A null terminated UTF-8 encoded string - /// [in] Index of the string in the tensor to set - void FillStringTensorElement(const char* s, size_t index); - - /// - /// Allocate if necessary and obtain a pointer to a UTF-8 - /// encoded string element buffer indexed by the flat element index, - /// of the specified length. - /// - /// This API is for advanced usage. It avoids a need to construct - /// an auxiliary array of string pointers, and allows to write data directly - /// (do not zero terminate). - /// - /// - /// - /// a pointer to a writable buffer - char* GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length); - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// pointer to the user allocated buffer with indices. Use nullptr for fully sparse tensors. - /// number of indices entries. Use 0 for fully sparse tensors - void UseCooIndices(int64_t* indices_data, size_t indices_num); - - /// - /// Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// pointer to the user allocated buffer with inner indices or nullptr for fully sparse tensors - /// number of csr inner indices or 0 for fully sparse tensors - /// pointer to the user allocated buffer with outer indices or nullptr for fully sparse tensors - /// number of csr outer indices or 0 for fully sparse tensors - void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num); - - /// - /// Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSparse format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// indices shape or a {0} for fully sparse - /// user allocated buffer with indices or nullptr for fully spare tensors - void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and COO indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information. - /// coo indices buffer or nullptr for fully sparse data - /// number of COO indices or 0 for fully sparse data - void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param, - const int64_t* indices_data, size_t indices_num); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and CSR indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information - /// csr inner indices pointer or nullptr for fully sparse tensors - /// number of csr inner indices or 0 for fully sparse tensors - /// pointer to csr indices data or nullptr for fully sparse tensors - /// number of csr outer indices or 0 - void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const int64_t* inner_indices_data, size_t inner_indices_num, - const int64_t* outer_indices_data, size_t outer_indices_num); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and BlockSparse indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information - /// indices shape. use {0} for fully sparse tensors - /// pointer to indices data or nullptr for fully sparse tensors - void FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const Shape& indices_shape, - const int32_t* indices_data); - -#endif -}; - -} // namespace detail - -using ConstValue = detail::ConstValueImpl>; -using UnownedValue = detail::ValueImpl>; - -/** \brief Wrapper around ::OrtValue - * - */ -struct Value : detail::ValueImpl { - using Base = detail::ValueImpl; - using OrtSparseValuesParam = detail::OrtSparseValuesParam; - using Shape = detail::Shape; - - explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used - explicit Value(OrtValue* p) : Base{p} {} ///< Used for interop with the C API - Value(Value&&) = default; - Value& operator=(Value&&) = default; - - ConstValue GetConst() const { return ConstValue{this->p_}; } - UnownedValue GetUnowned() const { return UnownedValue{this->p_}; } - - /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. - * \tparam T The numeric datatype. This API is not suitable for strings. - * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param p_data Pointer to the data buffer. - * \param p_data_element_count The number of elements in the data buffer. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - */ - template - static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len); - - /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. - * - * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param p_data Pointer to the data buffer. - * \param p_data_byte_count The number of bytes in the data buffer. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - * \param type The data type. - */ - static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type); - - /** \brief Creates an OrtValue with a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue. - * This overload will allocate the buffer for the tensor according to the supplied shape and data type. - * The allocated buffer will be owned by the returned OrtValue and will be freed when the OrtValue is released. - * The input data would need to be copied into the allocated buffer. - * This API is not suitable for strings. - * - * \tparam T The numeric datatype. This API is not suitable for strings. - * \param allocator The allocator to use. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - */ - template - static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len); - - /** \brief Creates an OrtValue with a tensor using the supplied OrtAllocator. - * Wraps OrtApi::CreateTensorAsOrtValue. - * The allocated buffer will be owned by the returned OrtValue and will be freed when the OrtValue is released. - * The input data would need to be copied into the allocated buffer. - * This API is not suitable for strings. - * - * \param allocator The allocator to use. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - * \param type The data type. - */ - static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type); - - /** \brief Creates an OrtValue with a Map Onnx type representation. - * The API would ref-count the supplied OrtValues and they will be released - * when the returned OrtValue is released. The caller may release keys and values after the call - * returns. - * - * \param keys an OrtValue containing a tensor with primitive data type keys. - * \param values an OrtValue that may contain a tensor. Ort currently supports only primitive data type values. - */ - static Value CreateMap(const Value& keys, const Value& values); ///< Wraps OrtApi::CreateValue - - /** \brief Creates an OrtValue with a Sequence Onnx type representation. - * The API would ref-count the supplied OrtValues and they will be released - * when the returned OrtValue is released. The caller may release the values after the call - * returns. - * - * \param values a vector of OrtValues that must have the same Onnx value type. - */ - static Value CreateSequence(const std::vector& values); ///< Wraps OrtApi::CreateValue - - /** \brief Creates an OrtValue wrapping an Opaque type. - * This is used for experimental support of non-tensor types. - * - * \tparam T - the type of the value. - * \param domain - zero terminated utf-8 string. Domain of the type. - * \param type_name - zero terminated utf-8 string. Name of the type. - * \param value - the value to be wrapped. - */ - template - static Value CreateOpaque(const char* domain, const char* type_name, const T& value); ///< Wraps OrtApi::CreateOpaqueValue - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// This is a simple forwarding method to the other overload that helps deducing - /// data type enum value from the type of the buffer. - /// - /// numeric datatype. This API is not suitable for strings. - /// Memory description where the user buffers reside (CPU vs GPU etc) - /// pointer to the user supplied buffer, use nullptr for fully sparse tensors - /// a would be dense shape of the tensor - /// non zero values shape. Use a single 0 shape for fully sparse tensors. - /// - template - static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, - const Shape& values_shape); - - /// - /// Creates an OrtValue instance containing SparseTensor. This constructs - /// a sparse tensor that makes use of user allocated buffers. It does not make copies - /// of the user provided data and does not modify it. The lifespan of user provided buffers should - /// eclipse the life span of the resulting OrtValue. This call constructs an instance that only contain - /// a pointer to non-zero values. To fully populate the sparse tensor call UseIndices() API below - /// to supply a sparse format specific indices. - /// This API is not suitable for string data. Use CreateSparseTensor() with allocator specified so strings - /// can be properly copied into the allocated buffer. - /// - /// Memory description where the user buffers reside (CPU vs GPU etc) - /// pointer to the user supplied buffer, use nullptr for fully sparse tensors - /// a would be dense shape of the tensor - /// non zero values shape. Use a single 0 shape for fully sparse tensors. - /// data type - /// Ort::Value instance containing SparseTensor - static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, - const Shape& values_shape, ONNXTensorElementDataType type); - - /// - /// This is a simple forwarding method to the below CreateSparseTensor. - /// This helps to specify data type enum in terms of C++ data type. - /// Use CreateSparseTensor - /// - /// numeric data type only. String data enum must be specified explicitly. - /// allocator to use - /// a would be dense shape of the tensor - /// Ort::Value - template - static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape); - - /// - /// Creates an instance of OrtValue containing sparse tensor. The created instance has no data. - /// The data must be supplied by on of the FillSparseTensor() methods that take both non-zero values - /// and indices. The data will be copied into a buffer that would be allocated using the supplied allocator. - /// Use this API to create OrtValues that contain sparse tensors with all supported data types including - /// strings. - /// - /// allocator to use. The allocator lifespan must eclipse that of the resulting OrtValue - /// a would be dense shape of the tensor - /// data type - /// an instance of Ort::Value - static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type); - -#endif // !defined(DISABLE_SPARSE_TENSORS) -}; - -/// -/// Represents native memory allocation coming from one of the -/// OrtAllocators registered with OnnxRuntime. -/// Use it to wrap an allocation made by an allocator -/// so it can be automatically released when no longer needed. -/// -struct MemoryAllocation { - MemoryAllocation(OrtAllocator* allocator, void* p, size_t size); - ~MemoryAllocation(); - MemoryAllocation(const MemoryAllocation&) = delete; - MemoryAllocation& operator=(const MemoryAllocation&) = delete; - MemoryAllocation(MemoryAllocation&&) noexcept; - MemoryAllocation& operator=(MemoryAllocation&&) noexcept; - - void* get() { return p_; } - size_t size() const { return size_; } - - private: - OrtAllocator* allocator_; - void* p_; - size_t size_; -}; - -namespace detail { -template -struct AllocatorImpl : Base { - using B = Base; - using B::B; - - void* Alloc(size_t size); - MemoryAllocation GetAllocation(size_t size); - void Free(void* p); - ConstMemoryInfo GetInfo() const; -}; - -} // namespace detail - -/** \brief Wrapper around ::OrtAllocator default instance that is owned by Onnxruntime - * - */ -struct AllocatorWithDefaultOptions : detail::AllocatorImpl> { - explicit AllocatorWithDefaultOptions(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance - AllocatorWithDefaultOptions(); -}; - -/** \brief Wrapper around ::OrtAllocator - * - */ -struct Allocator : detail::AllocatorImpl { - explicit Allocator(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance - Allocator(const Session& session, const OrtMemoryInfo*); -}; - -using UnownedAllocator = detail::AllocatorImpl>; - -namespace detail { -namespace binding_utils { -// Bring these out of template -std::vector GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator*); -std::vector GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator*); -} // namespace binding_utils - -template -struct ConstIoBindingImpl : Base { - using B = Base; - using B::B; - - std::vector GetOutputNames() const; - std::vector GetOutputNames(OrtAllocator*) const; - std::vector GetOutputValues() const; - std::vector GetOutputValues(OrtAllocator*) const; -}; - -template -struct IoBindingImpl : ConstIoBindingImpl { - using B = ConstIoBindingImpl; - using B::B; - - void BindInput(const char* name, const Value&); - void BindOutput(const char* name, const Value&); - void BindOutput(const char* name, const OrtMemoryInfo*); - void ClearBoundInputs(); - void ClearBoundOutputs(); - void SynchronizeInputs(); - void SynchronizeOutputs(); -}; - -} // namespace detail - -using ConstIoBinding = detail::ConstIoBindingImpl>; -using UnownedIoBinding = detail::IoBindingImpl>; - -/** \brief Wrapper around ::OrtIoBinding - * - */ -struct IoBinding : detail::IoBindingImpl { - explicit IoBinding(std::nullptr_t) {} ///< Create an empty object for convenience. Sometimes, we want to initialize members later. - explicit IoBinding(Session& session); - ConstIoBinding GetConst() const { return ConstIoBinding{this->p_}; } - UnownedIoBinding GetUnowned() const { return UnownedIoBinding{this->p_}; } -}; - -/*! \struct Ort::ArenaCfg - * \brief it is a structure that represents the configuration of an arena based allocator - * \details Please see docs/C_API.md for details - */ -struct ArenaCfg : detail::Base { - explicit ArenaCfg(std::nullptr_t) {} ///< Create an empty ArenaCfg object, must be assigned a valid one to be used - /** - * Wraps OrtApi::CreateArenaCfg - * \param max_mem - use 0 to allow ORT to choose the default - * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested - * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default - * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default - * See docs/C_API.md for details on what the following parameters mean and how to choose these values - */ - ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk); -}; - -// -// Custom OPs (only needed to implement custom OPs) -// - -/// -/// This struct provides life time management for custom op attribute -/// -struct OpAttr : detail::Base { - OpAttr(const char* name, const void* data, int len, OrtOpAttrType type); -}; - -/** - * Macro that logs a message using the provided logger. Throws an exception if OrtApi::Logger_LogMessage fails. - * Example: ORT_CXX_LOG(logger, ORT_LOGGING_LEVEL_INFO, "Log a message"); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param message A null-terminated UTF-8 message to log. - */ -#define ORT_CXX_LOG(logger, message_severity, message) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - Ort::ThrowOnError(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), message)); \ - } \ - } while (false) - -/** - * Macro that logs a message using the provided logger. Can be used in noexcept code since errors are silently ignored. - * Example: ORT_CXX_LOG_NOEXCEPT(logger, ORT_LOGGING_LEVEL_INFO, "Log a message"); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param message A null-terminated UTF-8 message to log. - */ -#define ORT_CXX_LOG_NOEXCEPT(logger, message_severity, message) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - static_cast(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), message)); \ - } \ - } while (false) - -/** - * Macro that logs a printf-like formatted message using the provided logger. Throws an exception if - * OrtApi::Logger_LogMessage fails or if a formatting error occurs. - * Example: ORT_CXX_LOGF(logger, ORT_LOGGING_LEVEL_INFO, "Log an int: %d", 12); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. - * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. - * \param ... Zero or more variadic arguments referenced by the format string. - */ -#define ORT_CXX_LOGF(logger, message_severity, /*format,*/...) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - Ort::ThrowOnError(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), __VA_ARGS__)); \ - } \ - } while (false) - -/** - * Macro that logs a printf-like formatted message using the provided logger. Can be used in noexcept code since errors - * are silently ignored. - * Example: ORT_CXX_LOGF_NOEXCEPT(logger, ORT_LOGGING_LEVEL_INFO, "Log an int: %d", 12); - * - * \param logger The Ort::Logger instance to use. Must be a value or reference. - * \param message_severity The logging severity level of the message. - * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. - * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. - * \param ... Zero or more variadic arguments referenced by the format string. - */ -#define ORT_CXX_LOGF_NOEXCEPT(logger, message_severity, /*format,*/...) \ - do { \ - if (message_severity >= logger.GetLoggingSeverityLevel()) { \ - static_cast(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \ - static_cast(__FUNCTION__), __VA_ARGS__)); \ - } \ - } while (false) - -/// -/// This class represents an ONNX Runtime logger that can be used to log information with an -/// associated severity level and source code location (file path, line number, function name). -/// -/// A Logger can be obtained from within custom operators by calling Ort::KernelInfo::GetLogger(). -/// Instances of Ort::Logger are the size of two pointers and can be passed by value. -/// -/// Use the ORT_CXX_LOG macros to ensure the source code location is set properly from the callsite -/// and to take advantage of a cached logging severity level that can bypass calls to the underlying C API. -/// -struct Logger { - /** - * Creates an empty Ort::Logger. Must be initialized from a valid Ort::Logger before use. - */ - Logger() = default; - - /** - * Creates an empty Ort::Logger. Must be initialized from a valid Ort::Logger before use. - */ - explicit Logger(std::nullptr_t) {} - - /** - * Creates a logger from an ::OrtLogger instance. Caches the logger's current severity level by calling - * OrtApi::Logger_GetLoggingSeverityLevel. Throws an exception if OrtApi::Logger_GetLoggingSeverityLevel fails. - * - * \param logger The ::OrtLogger to wrap. - */ - explicit Logger(const OrtLogger* logger); - - ~Logger() = default; - - Logger(const Logger&) = default; - Logger& operator=(const Logger&) = default; - - Logger(Logger&& v) noexcept = default; - Logger& operator=(Logger&& v) noexcept = default; - - /** - * Returns the logger's current severity level from the cached member. - * - * \return The current ::OrtLoggingLevel. - */ - OrtLoggingLevel GetLoggingSeverityLevel() const noexcept; - - /** - * Logs the provided message via OrtApi::Logger_LogMessage. Use the ORT_CXX_LOG or ORT_CXX_LOG_NOEXCEPT - * macros to properly set the source code location and to use the cached severity level to potentially bypass - * calls to the underlying C API. - * - * \param log_severity_level The message's logging severity level. - * \param file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. - * \param line_number The file line number in which the message is logged. Usually the value of __LINE__. - * \param func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. - * \param message The message to log. - * \return A Ort::Status value to indicate error or success. - */ - Status LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, - const char* func_name, const char* message) const noexcept; - - /** - * Logs a printf-like formatted message via OrtApi::Logger_LogMessage. Use the ORT_CXX_LOGF or ORT_CXX_LOGF_NOEXCEPT - * macros to properly set the source code location and to use the cached severity level to potentially bypass - * calls to the underlying C API. Returns an error status if a formatting error occurs. - * - * \param log_severity_level The message's logging severity level. - * \param file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. - * \param line_number The file line number in which the message is logged. Usually the value of __LINE__. - * \param func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. - * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. - * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. - * \param args Zero or more variadic arguments referenced by the format string. - * \return A Ort::Status value to indicate error or success. - */ - template - Status LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, - const char* func_name, const char* format, Args&&... args) const noexcept; - - private: - const OrtLogger* logger_{}; - OrtLoggingLevel cached_severity_level_{}; -}; - -/// -/// This class wraps a raw pointer OrtKernelContext* that is being passed -/// to the custom kernel Compute() method. Use it to safely access context -/// attributes, input and output parameters with exception safety guarantees. -/// See usage example in onnxruntime/test/testdata/custom_op_library/custom_op_library.cc -/// -struct KernelContext { - explicit KernelContext(OrtKernelContext* context); - size_t GetInputCount() const; - size_t GetOutputCount() const; - // If input is optional and is not present, the method returns en empty ConstValue - // which can be compared to nullptr. - ConstValue GetInput(size_t index) const; - // If outout is optional and is not present, the method returns en empty UnownedValue - // which can be compared to nullptr. - UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const; - UnownedValue GetOutput(size_t index, const std::vector& dims) const; - void* GetGPUComputeStream() const; - Logger GetLogger() const; - OrtAllocator* GetAllocator(const OrtMemoryInfo& memory_info) const; - OrtKernelContext* GetOrtKernelContext() const { return ctx_; } - void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const; - - private: - OrtKernelContext* ctx_; -}; - -struct KernelInfo; - -namespace detail { -namespace attr_utils { -void GetAttr(const OrtKernelInfo* p, const char* name, float&); -void GetAttr(const OrtKernelInfo* p, const char* name, int64_t&); -void GetAttr(const OrtKernelInfo* p, const char* name, std::string&); -void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector&); -void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector&); -} // namespace attr_utils - -template -struct KernelInfoImpl : Base { - using B = Base; - using B::B; - - KernelInfo Copy() const; - - template // R is only implemented for float, int64_t, and string - R GetAttribute(const char* name) const { - R val; - attr_utils::GetAttr(this->p_, name, val); - return val; - } - - template // R is only implemented for std::vector, std::vector - std::vector GetAttributes(const char* name) const { - std::vector result; - attr_utils::GetAttrs(this->p_, name, result); - return result; - } - - Value GetTensorAttribute(const char* name, OrtAllocator* allocator) const; - - size_t GetInputCount() const; - size_t GetOutputCount() const; - - std::string GetInputName(size_t index) const; - std::string GetOutputName(size_t index) const; - - TypeInfo GetInputTypeInfo(size_t index) const; - TypeInfo GetOutputTypeInfo(size_t index) const; - - ConstValue GetTensorConstantInput(size_t index, int* is_constant) const; - - std::string GetNodeName() const; - Logger GetLogger() const; -}; - -} // namespace detail - -using ConstKernelInfo = detail::KernelInfoImpl>; - -/// -/// This struct owns the OrtKernInfo* pointer when a copy is made. -/// For convenient wrapping of OrtKernelInfo* passed to kernel constructor -/// and query attributes, warp the pointer with Ort::Unowned instance -/// so it does not destroy the pointer the kernel does not own. -/// -struct KernelInfo : detail::KernelInfoImpl { - explicit KernelInfo(std::nullptr_t) {} ///< Create an empty instance to initialize later - explicit KernelInfo(OrtKernelInfo* info); ///< Take ownership of the instance - ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; } -}; - -/// -/// Create and own custom defined operation. -/// -struct Op : detail::Base { - explicit Op(std::nullptr_t) {} ///< Create an empty Operator object, must be assigned a valid one to be used - - explicit Op(OrtOp*); ///< Take ownership of the OrtOp - - static Op Create(const OrtKernelInfo* info, const char* op_name, const char* domain, - int version, const char** type_constraint_names, - const ONNXTensorElementDataType* type_constraint_values, - size_t type_constraint_count, - const OpAttr* attr_values, - size_t attr_count, - size_t input_count, size_t output_count); - - void Invoke(const OrtKernelContext* context, - const Value* input_values, - size_t input_count, - Value* output_values, - size_t output_count); - - // For easier refactoring - void Invoke(const OrtKernelContext* context, - const OrtValue* const* input_values, - size_t input_count, - OrtValue* const* output_values, - size_t output_count); -}; - -/// -/// Provide access to per-node attributes and input shapes, so one could compute and set output shapes. -/// -struct ShapeInferContext { - struct SymbolicInteger { - SymbolicInteger(int64_t i) : i_(i), is_int_(true){}; - SymbolicInteger(const char* s) : s_(s), is_int_(false){}; - SymbolicInteger(const SymbolicInteger&) = default; - SymbolicInteger(SymbolicInteger&&) = default; - - SymbolicInteger& operator=(const SymbolicInteger&) = default; - SymbolicInteger& operator=(SymbolicInteger&&) = default; - - bool operator==(const SymbolicInteger& dim) const { - if (is_int_ == dim.is_int_) { - if (is_int_) { - return i_ == dim.i_; - } else { - return std::string{s_} == std::string{dim.s_}; - } - } - return false; - } - - bool IsInt() const { return is_int_; } - int64_t AsInt() const { return i_; } - const char* AsSym() const { return s_; } - - static constexpr int INVALID_INT_DIM = -2; - - private: - union { - int64_t i_; - const char* s_; - }; - bool is_int_; - }; - - using Shape = std::vector; - - ShapeInferContext(const OrtApi* ort_api, OrtShapeInferContext* ctx); - - const Shape& GetInputShape(size_t indice) const { return input_shapes_.at(indice); } - - size_t GetInputCount() const { return input_shapes_.size(); } - - Status SetOutputShape(size_t indice, const Shape& shape); - - int64_t GetAttrInt(const char* attr_name); - - using Ints = std::vector; - Ints GetAttrInts(const char* attr_name); - - float GetAttrFloat(const char* attr_name); - - using Floats = std::vector; - Floats GetAttrFloats(const char* attr_name); - - std::string GetAttrString(const char* attr_name); - - using Strings = std::vector; - Strings GetAttrStrings(const char* attr_name); - - private: - const OrtOpAttr* GetAttrHdl(const char* attr_name) const; - const OrtApi* ort_api_; - OrtShapeInferContext* ctx_; - std::vector input_shapes_; -}; - -using ShapeInferFn = Ort::Status (*)(Ort::ShapeInferContext&); - -#define MAX_CUSTOM_OP_END_VER (1UL << 31) - 1 - -template -struct CustomOpBase : OrtCustomOp { - CustomOpBase() { - OrtCustomOp::version = ORT_API_VERSION; - OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast(this_)->GetName(); }; - - OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast(this_)->GetExecutionProviderType(); }; - - OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetInputTypeCount(); }; - OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputType(index); }; - OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputMemoryType(index); }; - - OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetOutputTypeCount(); }; - OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputType(index); }; - -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(push) -#pragma warning(disable : 26409) -#endif - OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast(op_kernel); }; -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(pop) -#endif - OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputCharacteristic(index); }; - OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputCharacteristic(index); }; - - OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp* this_) { return static_cast(this_)->GetVariadicInputMinArity(); }; - OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp* this_) { return static_cast(static_cast(this_)->GetVariadicInputHomogeneity()); }; - OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp* this_) { return static_cast(this_)->GetVariadicOutputMinArity(); }; - OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp* this_) { return static_cast(static_cast(this_)->GetVariadicOutputHomogeneity()); }; -#ifdef __cpp_if_constexpr - if constexpr (WithStatus) { -#else - if (WithStatus) { -#endif - OrtCustomOp::CreateKernelV2 = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info, void** op_kernel) -> OrtStatusPtr { - return static_cast(this_)->CreateKernelV2(*api, info, op_kernel); - }; - OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { - return static_cast(op_kernel)->ComputeV2(context); - }; - } else { - OrtCustomOp::CreateKernelV2 = nullptr; - OrtCustomOp::KernelComputeV2 = nullptr; - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast(this_)->CreateKernel(*api, info); }; - OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { - static_cast(op_kernel)->Compute(context); - }; - } - - SetShapeInferFn(0); - - OrtCustomOp::GetStartVersion = [](const OrtCustomOp* this_) { - return static_cast(this_)->start_ver_; - }; - - OrtCustomOp::GetEndVersion = [](const OrtCustomOp* this_) { - return static_cast(this_)->end_ver_; - }; - - OrtCustomOp::GetMayInplace = nullptr; - OrtCustomOp::ReleaseMayInplace = nullptr; - OrtCustomOp::GetAliasMap = nullptr; - OrtCustomOp::ReleaseAliasMap = nullptr; - } - - // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider - const char* GetExecutionProviderType() const { return nullptr; } - - // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below - // (inputs and outputs are required by default) - OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t /*index*/) const { - return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; - } - - OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t /*index*/) const { - return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; - } - - // Default implemention of GetInputMemoryType() that returns OrtMemTypeDefault - OrtMemType GetInputMemoryType(size_t /*index*/) const { - return OrtMemTypeDefault; - } - - // Default implementation of GetVariadicInputMinArity() returns 1 to specify that a variadic input - // should expect at least 1 argument. - int GetVariadicInputMinArity() const { - return 1; - } - - // Default implementation of GetVariadicInputHomegeneity() returns true to specify that all arguments - // to a variadic input should be of the same type. - bool GetVariadicInputHomogeneity() const { - return true; - } - - // Default implementation of GetVariadicOutputMinArity() returns 1 to specify that a variadic output - // should produce at least 1 output value. - int GetVariadicOutputMinArity() const { - return 1; - } - - // Default implementation of GetVariadicOutputHomegeneity() returns true to specify that all output values - // produced by a variadic output should be of the same type. - bool GetVariadicOutputHomogeneity() const { - return true; - } - - // Declare list of session config entries used by this Custom Op. - // Implement this function in order to get configs from CustomOpBase::GetSessionConfigs(). - // This default implementation returns an empty vector of config entries. - std::vector GetSessionConfigKeys() const { - return std::vector{}; - } - - template - decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape)) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp*, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - ShapeInferContext ctx(&GetApi(), ort_ctx); - return C::InferOutputShape(ctx); - }; - return {}; - } - - template - void SetShapeInferFn(...) { - OrtCustomOp::InferOutputShapeFn = {}; - } - - protected: - // Helper function that returns a map of session config entries specified by CustomOpBase::GetSessionConfigKeys. - void GetSessionConfigs(std::unordered_map& out, ConstSessionOptions options) const; - - int start_ver_ = 1; - int end_ver_ = MAX_CUSTOM_OP_END_VER; -}; - -} // namespace Ort - -#include "onnxruntime_cxx_inline.h" diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_cxx_inline.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_cxx_inline.h deleted file mode 100644 index a732bf1..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_cxx_inline.h +++ /dev/null @@ -1,2125 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Do not include this file directly. Please include "onnxruntime_cxx_api.h" instead. -// If interested in trying out features of the new experimental C++ API, include "experimental_onnxruntime_cxx_api.h" instead. -// -// These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter -// the main C++ file with implementation details. - -#include -#include -#include -#include - -// Convert OrtStatus to Ort::Status and return -// instead of throwing -#define ORT_CXX_RETURN_ON_API_FAIL(expression) \ - { \ - auto ort_status = (expression); \ - if (ort_status) { \ - return Ort::Status(ort_status); \ - } \ - } - -#ifdef __cpp_if_constexpr -#define ORT_CXX_IF_CONSTEXPR if constexpr -#else -#define ORT_CXX_IF_CONSTEXPR if -#endif - -namespace Ort { - -namespace detail { -inline void ThrowStatus(const Status& st) { - std::string error_message = st.GetErrorMessage(); - OrtErrorCode error_code = st.GetErrorCode(); - ORT_CXX_API_THROW(std::move(error_message), error_code); -} -} // namespace detail - -inline void ThrowOnError(OrtStatus* ort_status) { - if (ort_status) { - Ort::Status st(ort_status); - detail::ThrowStatus(st); - } -} - -inline void ThrowOnError(const Status& st) { - if (st) { - detail::ThrowStatus(st); - } -} - -inline Status::Status(OrtStatus* status) noexcept : Base{status} { -} - -inline Status::Status(const std::exception& e) noexcept { - p_ = GetApi().CreateStatus(ORT_FAIL, e.what()); -} - -inline Status::Status(const Exception& e) noexcept { - p_ = GetApi().CreateStatus(e.GetOrtErrorCode(), e.what()); -} - -inline Status::Status(const char* message, OrtErrorCode code) noexcept { - p_ = GetApi().CreateStatus(code, message); -} - -inline std::string Status::GetErrorMessage() const { - std::string message(GetApi().GetErrorMessage(p_)); - return message; -} - -inline OrtErrorCode Status::GetErrorCode() const { - return GetApi().GetErrorCode(p_); -} - -inline bool Status::IsOK() const noexcept { - return (p_ == nullptr); -} - -// This template converts a C++ type into it's ONNXTensorElementDataType -template -struct TypeToTensorType; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; -}; - -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2; -}; -template <> -struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ; -}; - -inline bool BFloat16_t::operator==(const BFloat16_t& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is not equal to anything, including itself. - return false; - } - return val == rhs.val; -} - -inline bool BFloat16_t::operator<(const BFloat16_t& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is unordered with respect to everything, including itself. - return false; - } - - const bool left_is_negative = IsNegative(); - if (left_is_negative != rhs.IsNegative()) { - // When the signs of left and right differ, we know that left is less than right if it is - // the negative value. The exception to this is if both values are zero, in which case IEEE - // says they should be equal, even if the signs differ. - return left_is_negative && !AreZero(*this, rhs); - } - return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); -} - -inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size) - : allocator_(allocator), p_(p), size_(size) { -} - -inline MemoryAllocation::~MemoryAllocation() { - if (p_ != nullptr) { - // We do not throw out of destructor - auto ret = GetApi().AllocatorFree(allocator_, p_); - static_cast(ret); - } -} - -inline MemoryAllocation::MemoryAllocation(MemoryAllocation&& o) noexcept : allocator_(nullptr), p_(nullptr), size_(0) { - *this = std::move(o); -} - -inline MemoryAllocation& MemoryAllocation::operator=(MemoryAllocation&& o) noexcept { - OrtAllocator* alloc = nullptr; - void* p = nullptr; - size_t sz = 0; - - // Swap out this - std::swap(alloc, allocator_); - std::swap(p, p_); - std::swap(sz, size_); - - // Swap with incoming - std::swap(allocator_, o.allocator_); - std::swap(p_, o.p_); - std::swap(size_, o.size_); - - // Destroy this instance if needed - MemoryAllocation this_alloc(alloc, p, sz); - return *this; -} - -namespace detail { - -template -inline void* AllocatorImpl::Alloc(size_t size) { - void* out; - ThrowOnError(GetApi().AllocatorAlloc(this->p_, size, &out)); - return out; -} - -template -inline MemoryAllocation AllocatorImpl::GetAllocation(size_t size) { - void* out; - ThrowOnError(GetApi().AllocatorAlloc(this->p_, size, &out)); - MemoryAllocation result(this->p_, out, size); - return result; -} - -template -inline void AllocatorImpl::Free(void* p) { - ThrowOnError(GetApi().AllocatorFree(this->p_, p)); -} - -template -inline ConstMemoryInfo AllocatorImpl::GetInfo() const { - const OrtMemoryInfo* out; - ThrowOnError(GetApi().AllocatorGetInfo(this->p_, &out)); - return ConstMemoryInfo{out}; -} - -} // namespace detail - -inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { - ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&this->p_)); -} - -inline Allocator::Allocator(const Session& sess, const OrtMemoryInfo* mem_info) { - ThrowOnError(GetApi().CreateAllocator(sess, mem_info, &this->p_)); -} - -namespace detail { - -template -inline std::string MemoryInfoImpl::GetAllocatorName() const { - const char* name = nullptr; - ThrowOnError(GetApi().MemoryInfoGetName(this->p_, &name)); - return std::string(name); -} - -template -inline OrtAllocatorType MemoryInfoImpl::GetAllocatorType() const { - OrtAllocatorType type; - ThrowOnError(GetApi().MemoryInfoGetType(this->p_, &type)); - return type; -} - -template -inline int MemoryInfoImpl::GetDeviceId() const { - int id = 0; - ThrowOnError(GetApi().MemoryInfoGetId(this->p_, &id)); - return id; -} - -template -inline OrtMemoryInfoDeviceType MemoryInfoImpl::GetDeviceType() const { - OrtMemoryInfoDeviceType type; - GetApi().MemoryInfoGetDeviceType(this->p_, &type); - return type; -} - -template -inline OrtMemType MemoryInfoImpl::GetMemoryType() const { - OrtMemType type; - ThrowOnError(GetApi().MemoryInfoGetMemType(this->p_, &type)); - return type; -} - -template -template -inline bool MemoryInfoImpl::operator==(const MemoryInfoImpl& o) const { - int comp_result = 0; - ThrowOnError(Ort::GetApi().CompareMemoryInfo(this->p_, o, &comp_result)); - return comp_result == 0; -} - -} // namespace detail - -inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) { - OrtMemoryInfo* p; - ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p)); - return MemoryInfo(p); -} - -inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type) { - ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &this->p_)); -} - -namespace detail { -template -inline std::vector ConstIoBindingImpl::GetOutputNames() const { - AllocatorWithDefaultOptions allocator; - return binding_utils::GetOutputNamesHelper(this->p_, allocator); -} - -template -inline std::vector ConstIoBindingImpl::GetOutputNames(OrtAllocator* allocator) const { - return binding_utils::GetOutputNamesHelper(this->p_, allocator); -} - -template -inline std::vector ConstIoBindingImpl::GetOutputValues() const { - AllocatorWithDefaultOptions allocator; - return binding_utils::GetOutputValuesHelper(this->p_, allocator); -} - -template -inline std::vector ConstIoBindingImpl::GetOutputValues(OrtAllocator* allocator) const { - return binding_utils::GetOutputValuesHelper(this->p_, allocator); -} - -template -inline void IoBindingImpl::BindInput(const char* name, const Value& value) { - ThrowOnError(GetApi().BindInput(this->p_, name, value)); -} - -template -inline void IoBindingImpl::BindOutput(const char* name, const Value& value) { - ThrowOnError(GetApi().BindOutput(this->p_, name, value)); -} - -template -inline void IoBindingImpl::BindOutput(const char* name, const OrtMemoryInfo* mem_info) { - ThrowOnError(GetApi().BindOutputToDevice(this->p_, name, mem_info)); -} - -template -inline void IoBindingImpl::ClearBoundInputs() { - GetApi().ClearBoundInputs(this->p_); -} - -template -inline void IoBindingImpl::ClearBoundOutputs() { - GetApi().ClearBoundOutputs(this->p_); -} - -template -inline void IoBindingImpl::SynchronizeInputs() { - ThrowOnError(GetApi().SynchronizeBoundInputs(this->p_)); -} - -template -inline void IoBindingImpl::SynchronizeOutputs() { - ThrowOnError(GetApi().SynchronizeBoundOutputs(this->p_)); -} - -namespace binding_utils { -inline std::vector GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator* allocator) { - std::vector result; - auto free_fn = detail::AllocatedFree(allocator); - using Ptr = std::unique_ptr; - - char* buffer = nullptr; - size_t* lengths = nullptr; - size_t count = 0; - ThrowOnError(GetApi().GetBoundOutputNames(binding, allocator, &buffer, &lengths, &count)); - - if (count == 0) { - return result; - } - - Ptr buffer_g(buffer, free_fn); - Ptr lengths_g(lengths, free_fn); - - result.reserve(count); - for (size_t i = 0; i < count; ++i) { - auto sz = *lengths; - result.emplace_back(buffer, sz); - buffer += sz; - ++lengths; - } - return result; -} - -inline std::vector GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator* allocator) { - std::vector result; - size_t owned = 0; - size_t output_count = 0; - // Lambda to release the buffer when no longer needed and - // make sure that we destroy all instances on exception - auto free_fn = [&owned, &output_count, allocator](OrtValue** buffer) { - if (buffer) { - while (owned < output_count) { - auto* p = buffer + owned++; - GetApi().ReleaseValue(*p); - } - allocator->Free(allocator, buffer); - } - }; - using Ptr = std::unique_ptr; - - OrtValue** output_buffer = nullptr; - ThrowOnError(GetApi().GetBoundOutputValues(binding, allocator, &output_buffer, &output_count)); - if (output_count == 0) { - return result; - } - - Ptr buffer_g(output_buffer, free_fn); - - result.reserve(output_count); - for (size_t i = 0; i < output_count; ++i) { - result.emplace_back(output_buffer[i]); - ++owned; - } - return result; -} - -} // namespace binding_utils -} // namespace detail - -inline IoBinding::IoBinding(Session& session) { - ThrowOnError(GetApi().CreateIoBinding(session, &this->p_)); -} - -inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk) { - ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_)); -} - -inline ThreadingOptions::ThreadingOptions() { - ThrowOnError(GetApi().CreateThreadingOptions(&p_)); -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalIntraOpNumThreads(int intra_op_num_threads) { - ThrowOnError(GetApi().SetGlobalIntraOpNumThreads(p_, intra_op_num_threads)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalInterOpNumThreads(int inter_op_num_threads) { - ThrowOnError(GetApi().SetGlobalInterOpNumThreads(p_, inter_op_num_threads)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalSpinControl(int allow_spinning) { - ThrowOnError(GetApi().SetGlobalSpinControl(p_, allow_spinning)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalDenormalAsZero() { - ThrowOnError(GetApi().SetGlobalDenormalAsZero(p_)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { - ThrowOnError(GetApi().SetGlobalCustomCreateThreadFn(p_, ort_custom_create_thread_fn)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalCustomThreadCreationOptions(void* ort_custom_thread_creation_options) { - ThrowOnError(GetApi().SetGlobalCustomThreadCreationOptions(p_, ort_custom_thread_creation_options)); - return *this; -} - -inline ThreadingOptions& ThreadingOptions::SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn) { - ThrowOnError(GetApi().SetGlobalCustomJoinThreadFn(p_, ort_custom_join_thread_fn)); - return *this; -} - -inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param) { - ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(logging_level, logid, tp_options, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, - OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env& Env::EnableTelemetryEvents() { - ThrowOnError(GetApi().EnableTelemetryEvents(p_)); - return *this; -} - -inline Env& Env::DisableTelemetryEvents() { - ThrowOnError(GetApi().DisableTelemetryEvents(p_)); - return *this; -} - -inline Env& Env::UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level) { - ThrowOnError(GetApi().UpdateEnvWithCustomLogLevel(p_, log_severity_level)); - return *this; -} - -inline Env& Env::CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg) { - ThrowOnError(GetApi().CreateAndRegisterAllocator(p_, mem_info, arena_cfg)); - return *this; -} - -inline Env& Env::CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, const std::unordered_map& options, const OrtArenaCfg* arena_cfg) { - std::vector keys, values; - auto num_entries = options.size(); - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - for (const auto& entry : options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - ThrowOnError(GetApi().CreateAndRegisterAllocatorV2(p_, provider_type.c_str(), mem_info, arena_cfg, keys.data(), values.data(), num_entries)); - return *this; -} - -inline CustomOpDomain::CustomOpDomain(const char* domain) { - ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_)); -} - -inline void CustomOpDomain::Add(const OrtCustomOp* op) { - ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); -} - -inline RunOptions::RunOptions() { - ThrowOnError(GetApi().CreateRunOptions(&p_)); -} - -inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level)); - return *this; -} - -inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level)); - return *this; -} - -inline int RunOptions::GetRunLogVerbosityLevel() const { - int out; - ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out)); - return out; -} - -inline int RunOptions::GetRunLogSeverityLevel() const { - int out; - ThrowOnError(GetApi().RunOptionsGetRunLogSeverityLevel(p_, &out)); - return out; -} - -inline RunOptions& RunOptions::SetRunTag(const char* run_tag) { - ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag)); - return *this; -} - -inline const char* RunOptions::GetRunTag() const { - const char* out; - ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out)); - return out; -} - -inline RunOptions& RunOptions::AddConfigEntry(const char* config_key, const char* config_value) { - ThrowOnError(GetApi().AddRunConfigEntry(p_, config_key, config_value)); - return *this; -} - -inline RunOptions& RunOptions::SetTerminate() { - ThrowOnError(GetApi().RunOptionsSetTerminate(p_)); - return *this; -} - -inline RunOptions& RunOptions::UnsetTerminate() { - ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_)); - return *this; -} - -namespace detail { - -template -inline Ort::SessionOptions ConstSessionOptionsImpl::Clone() const { - OrtSessionOptions* out; - ThrowOnError(GetApi().CloneSessionOptions(this->p_, &out)); - return SessionOptions{out}; -} - -template -inline std::string ConstSessionOptionsImpl::GetConfigEntry(const char* config_key) const { - size_t size = 0; - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline bool ConstSessionOptionsImpl::HasConfigEntry(const char* config_key) const { - int out = 0; - Ort::ThrowOnError(GetApi().HasSessionConfigEntry(this->p_, config_key, &out)); - return static_cast(out); -} - -template -inline std::string ConstSessionOptionsImpl::GetConfigEntryOrDefault(const char* config_key, const std::string& def) { - if (!this->HasConfigEntry(config_key)) { - return def; - } - - return this->GetConfigEntry(config_key); -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetIntraOpNumThreads(int intra_op_num_threads) { - ThrowOnError(GetApi().SetIntraOpNumThreads(this->p_, intra_op_num_threads)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetInterOpNumThreads(int inter_op_num_threads) { - ThrowOnError(GetApi().SetInterOpNumThreads(this->p_, inter_op_num_threads)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) { - ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(this->p_, graph_optimization_level)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetDeterministicCompute(bool value) { - ThrowOnError(GetApi().SetDeterministicCompute(this->p_, value)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_filepath) { - ThrowOnError(GetApi().SetOptimizedModelFilePath(this->p_, optimized_model_filepath)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableProfiling(const ORTCHAR_T* profile_file_prefix) { - ThrowOnError(GetApi().EnableProfiling(this->p_, profile_file_prefix)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisableProfiling() { - ThrowOnError(GetApi().DisableProfiling(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableOrtCustomOps() { - ThrowOnError(GetApi().EnableOrtCustomOps(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableMemPattern() { - ThrowOnError(GetApi().EnableMemPattern(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisableMemPattern() { - ThrowOnError(GetApi().DisableMemPattern(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::EnableCpuMemArena() { - ThrowOnError(GetApi().EnableCpuMemArena(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisableCpuMemArena() { - ThrowOnError(GetApi().DisableCpuMemArena(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetExecutionMode(ExecutionMode execution_mode) { - ThrowOnError(GetApi().SetSessionExecutionMode(this->p_, execution_mode)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetLogId(const char* logid) { - ThrowOnError(GetApi().SetSessionLogId(this->p_, logid)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetLogSeverityLevel(int level) { - ThrowOnError(GetApi().SetSessionLogSeverityLevel(this->p_, level)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::Add(OrtCustomOpDomain* custom_op_domain) { - ThrowOnError(GetApi().AddCustomOpDomain(this->p_, custom_op_domain)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddConfigEntry(const char* config_key, const char* config_value) { - ThrowOnError(GetApi().AddSessionConfigEntry(this->p_, config_key, config_value)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddInitializer(const char* name, const OrtValue* ort_val) { - ThrowOnError(GetApi().AddInitializer(this->p_, name, ort_val)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::DisablePerSessionThreads() { - ThrowOnError(GetApi().DisablePerSessionThreads(this->p_)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddExternalInitializers(const std::vector& names, - const std::vector& ort_values) { - const size_t inputs_num = names.size(); - if (inputs_num != ort_values.size()) { - ORT_CXX_API_THROW("Expecting names and ort_values to have the same length", ORT_INVALID_ARGUMENT); - } - std::vector names_ptr; - std::vector ort_values_ptrs; - names_ptr.reserve(inputs_num); - ort_values_ptrs.reserve(inputs_num); - for (size_t i = 0; i < inputs_num; ++i) { - names_ptr.push_back(names[i].c_str()); - ort_values_ptrs.push_back(ort_values[i]); - } - ThrowOnError(GetApi().AddExternalInitializers(this->p_, names_ptr.data(), ort_values_ptrs.data(), inputs_num)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AddExternalInitializersFromFilesInMemory(const std::vector>& file_names, - const std::vector& buffer_array, - const std::vector& file_lengths) { - const size_t inputs_num = file_names.size(); - if (inputs_num != buffer_array.size()) { - ORT_CXX_API_THROW("Expecting names and buffer_array to have the same length", ORT_INVALID_ARGUMENT); - } - if (inputs_num != file_lengths.size()) { - ORT_CXX_API_THROW("Expecting names and file_lengths to have the same length", ORT_INVALID_ARGUMENT); - } - std::vector names_ptr; - names_ptr.reserve(inputs_num); - for (size_t i = 0; i < inputs_num; ++i) { - names_ptr.push_back(file_names[i].c_str()); - } - ThrowOnError(GetApi().AddExternalInitializersFromFilesInMemory(this->p_, names_ptr.data(), buffer_array.data(), - file_lengths.data(), inputs_num)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA_V2(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_ROCM(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT_V2(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_MIGraphX(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CANN(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_Dnnl(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider( - const std::string& provider_name, - const std::unordered_map& provider_options) { - auto num_entries = provider_options.size(); - std::vector keys, values; - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - - for (const auto& entry : provider_options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider(this->p_, provider_name.c_str(), - keys.data(), values.data(), num_entries)); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { - ThrowOnError(GetApi().SessionOptionsSetCustomCreateThreadFn(this->p_, ort_custom_create_thread_fn)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options) { - ThrowOnError(GetApi().SessionOptionsSetCustomThreadCreationOptions(this->p_, ort_custom_thread_creation_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn) { - ThrowOnError(GetApi().SessionOptionsSetCustomJoinThreadFn(this->p_, ort_custom_join_thread_fn)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(this->p_, &provider_options)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_OpenVINO_V2(const std::unordered_map& provider_options) { - auto num_entries = provider_options.size(); - std::vector keys, values; - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - - for (const auto& entry : provider_options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO_V2(this->p_, - keys.data(), values.data(), num_entries)); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_VitisAI(const std::unordered_map& provider_options) { - auto num_entries = provider_options.size(); - std::vector keys, values; - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - - for (const auto& entry : provider_options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_VitisAI(this->p_, keys.data(), values.data(), num_entries)); - - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, - const CustomOpConfigs& custom_op_configs) { - // Add custom op config entries before registering the custom op library. Otherwise, the config entries _may_ be ignored by - // the custom op library. - for (const auto& config_iter : custom_op_configs.GetFlattenedConfigs()) { - AddConfigEntry(config_iter.first.c_str(), config_iter.second.c_str()); - } - - ThrowOnError(GetApi().RegisterCustomOpsLibrary_V2(this->p_, library_name)); - return *this; -} - -template -inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsUsingFunction(const char* registration_function_name) { - ThrowOnError(GetApi().RegisterCustomOpsUsingFunction(this->p_, registration_function_name)); - return *this; -} - -/// Session -template -inline size_t ConstSessionImpl::GetInputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetInputCount(this->p_, &out)); - return out; -} - -template -inline size_t ConstSessionImpl::GetOutputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOutputCount(this->p_, &out)); - return out; -} - -template -inline size_t ConstSessionImpl::GetOverridableInitializerCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOverridableInitializerCount(this->p_, &out)); - return out; -} - -template -inline AllocatedStringPtr ConstSessionImpl::GetInputNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetInputName(this->p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -template -inline AllocatedStringPtr ConstSessionImpl::GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOutputName(this->p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -template -inline AllocatedStringPtr ConstSessionImpl::GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOverridableInitializerName(this->p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -template -inline uint64_t ConstSessionImpl::GetProfilingStartTimeNs() const { - uint64_t out; - ThrowOnError(GetApi().SessionGetProfilingStartTimeNs(this->p_, &out)); - return out; -} - -template -inline ModelMetadata ConstSessionImpl::GetModelMetadata() const { - OrtModelMetadata* out; - ThrowOnError(GetApi().SessionGetModelMetadata(this->p_, &out)); - return ModelMetadata{out}; -} - -template -inline TypeInfo ConstSessionImpl::GetInputTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetInputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline TypeInfo ConstSessionImpl::GetOutputTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetOutputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline TypeInfo ConstSessionImpl::GetOverridableInitializerTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline std::vector SessionImpl::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, size_t output_count) { - std::vector output_values; - output_values.reserve(output_count); - for (size_t i = 0; i < output_count; i++) - output_values.emplace_back(nullptr); - Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_count); - return output_values; -} - -template -inline void SessionImpl::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count) { - static_assert(sizeof(Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); - auto ort_input_values = reinterpret_cast(input_values); - auto ort_output_values = reinterpret_cast(output_values); - ThrowOnError(GetApi().Run(this->p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values)); -} - -template -inline void SessionImpl::Run(const RunOptions& run_options, const IoBinding& io_binding) { - ThrowOnError(GetApi().RunWithBinding(this->p_, run_options, io_binding)); -} - -template -inline void SessionImpl::RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data) { - auto ort_input_values = reinterpret_cast(input_values); - auto ort_output_values = reinterpret_cast(output_values); - ThrowOnError(GetApi().RunAsync(this->p_, run_options, input_names, - ort_input_values, input_count, output_names, output_count, - ort_output_values, callback, user_data)); -} - -template -inline AllocatedStringPtr SessionImpl::EndProfilingAllocated(OrtAllocator* allocator) { - char* out = nullptr; - ThrowOnError(GetApi().SessionEndProfiling(this->p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -} // namespace detail - -inline SessionOptions::SessionOptions() { - ThrowOnError(GetApi().CreateSessionOptions(&this->p_)); -} - -/// CustomOpConfigs -inline std::string detail::MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config) { - std::string config_key = "custom_op."; - - config_key += custom_op_name; - config_key += "."; - config_key += config; - - return config_key; -} - -inline CustomOpConfigs& CustomOpConfigs::AddConfig(const char* custom_op_name, const char* config_key, const char* config_value) { - const std::string full_flat_key = detail::MakeCustomOpConfigEntryKey(custom_op_name, config_key); - flat_configs_[full_flat_key] = config_value; - return *this; -} - -inline const std::unordered_map& CustomOpConfigs::GetFlattenedConfigs() const { - return flat_configs_; -} - -inline Session::Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) { - ThrowOnError(GetApi().CreateSession(env, model_path, options, &this->p_)); -} - -inline Session::Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container) { - ThrowOnError(GetApi().CreateSessionWithPrepackedWeightsContainer(env, model_path, options, prepacked_weights_container, &this->p_)); -} - -inline Session::Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options) { - ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &this->p_)); -} - -inline Session::Session(const Env& env, const void* model_data, size_t model_data_length, - const SessionOptions& options, OrtPrepackedWeightsContainer* prepacked_weights_container) { - ThrowOnError(GetApi().CreateSessionFromArrayWithPrepackedWeightsContainer(env, model_data, model_data_length, options, - prepacked_weights_container, &this->p_)); -} - -inline AllocatedStringPtr ModelMetadata::GetProducerNameAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::GetGraphNameAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::GetDomainAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr Ort::ModelMetadata::GetDescriptionAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::GetGraphDescriptionAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphDescription(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr ModelMetadata::LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline std::vector ModelMetadata::GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const { - auto deletor = detail::AllocatedFree(allocator); - std::vector result; - - char** out = nullptr; - int64_t num_keys = 0; - ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys)); - if (num_keys <= 0) { - return result; - } - - // array of pointers will be freed - std::unique_ptr array_guard(out, deletor); - // reserve may throw - auto strings_deletor = [&deletor, num_keys](char** out) { for(int64_t i = 0; i < num_keys; ++i) deletor(out[i]); }; - std::unique_ptr strings_guard(out, strings_deletor); - result.reserve(static_cast(num_keys)); - strings_guard.release(); - for (int64_t i = 0; i < num_keys; ++i) { - result.push_back(AllocatedStringPtr(out[i], deletor)); - } - - return result; -} - -inline int64_t ModelMetadata::GetVersion() const { - int64_t out; - ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out)); - return out; -} - -namespace detail { - -template -inline ONNXTensorElementDataType TensorTypeAndShapeInfoImpl::GetElementType() const { - ONNXTensorElementDataType out; - ThrowOnError(GetApi().GetTensorElementType(this->p_, &out)); - return out; -} - -template -inline size_t TensorTypeAndShapeInfoImpl::GetElementCount() const { - size_t out; - ThrowOnError(GetApi().GetTensorShapeElementCount(this->p_, &out)); - return static_cast(out); -} - -template -inline size_t TensorTypeAndShapeInfoImpl::GetDimensionsCount() const { - size_t out; - ThrowOnError(GetApi().GetDimensionsCount(this->p_, &out)); - return out; -} - -template -inline void TensorTypeAndShapeInfoImpl::GetDimensions(int64_t* values, size_t values_count) const { - ThrowOnError(GetApi().GetDimensions(this->p_, values, values_count)); -} - -template -inline void TensorTypeAndShapeInfoImpl::GetSymbolicDimensions(const char** values, size_t values_count) const { - ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, values, values_count)); -} - -template -inline std::vector TensorTypeAndShapeInfoImpl::GetShape() const { - std::vector out(GetDimensionsCount(), 0); - ThrowOnError(GetApi().GetDimensions(this->p_, out.data(), out.size())); - return out; -} - -template -inline ConstTensorTypeAndShapeInfo TypeInfoImpl::GetTensorTypeAndShapeInfo() const { - const OrtTensorTypeAndShapeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToTensorInfo(this->p_, &out)); - return ConstTensorTypeAndShapeInfo{out}; -} - -template -inline ConstSequenceTypeInfo TypeInfoImpl::GetSequenceTypeInfo() const { - const OrtSequenceTypeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(this->p_, &out)); - return ConstSequenceTypeInfo{out}; -} - -template -inline ConstMapTypeInfo TypeInfoImpl::GetMapTypeInfo() const { - const OrtMapTypeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(this->p_, &out)); - return ConstMapTypeInfo{out}; -} - -template -inline ONNXType TypeInfoImpl::GetONNXType() const { - ONNXType out; - ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(this->p_, &out)); - return out; -} - -template -inline TypeInfo SequenceTypeInfoImpl::GetSequenceElementType() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetSequenceElementType(this->p_, &output)); - return TypeInfo{output}; -} - -template -inline TypeInfo OptionalTypeInfoImpl::GetOptionalElementType() const { - OrtTypeInfo* info; - ThrowOnError(GetApi().GetOptionalContainedTypeInfo(this->p_, &info)); - return TypeInfo{info}; -} - -template -inline ONNXTensorElementDataType MapTypeInfoImpl::GetMapKeyType() const { - ONNXTensorElementDataType out; - ThrowOnError(GetApi().GetMapKeyType(this->p_, &out)); - return out; -} - -template -inline TypeInfo MapTypeInfoImpl::GetMapValueType() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetMapValueType(this->p_, &output)); - return TypeInfo{output}; -} - -template -inline ConstOptionalTypeInfo TypeInfoImpl::GetOptionalTypeInfo() const { - const OrtOptionalTypeInfo* info; - ThrowOnError(GetApi().CastTypeInfoToOptionalTypeInfo(this->p_, &info)); - return ConstOptionalTypeInfo{info}; -} - -} // namespace detail - -namespace detail { - -template -template -inline void ConstValueImpl::GetOpaqueData(const char* domain, const char* type_name, R& out) const { - ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, this->p_, &out, sizeof(R))); -} - -template -inline bool ConstValueImpl::IsTensor() const { - int out; - ThrowOnError(GetApi().IsTensor(this->p_, &out)); - return out != 0; -} - -template -inline bool ConstValueImpl::HasValue() const { - int out; - ThrowOnError(GetApi().HasValue(this->p_, &out)); - return out != 0; -} - -template -inline size_t ConstValueImpl::GetCount() const { - size_t out; - ThrowOnError(GetApi().GetValueCount(this->p_, &out)); - return out; -} - -template -inline Value ConstValueImpl::GetValue(int index, OrtAllocator* allocator) const { - OrtValue* out; - ThrowOnError(GetApi().GetValue(this->p_, index, allocator, &out)); - return Value{out}; -} - -template -inline size_t ConstValueImpl::GetStringTensorDataLength() const { - size_t out; - ThrowOnError(GetApi().GetStringTensorDataLength(this->p_, &out)); - return out; -} - -template -inline size_t ConstValueImpl::GetStringTensorElementLength(size_t element_index) const { - size_t out; - ThrowOnError(GetApi().GetStringTensorElementLength(this->p_, element_index, &out)); - return out; -} - -template -template -inline const R* ConstValueImpl::GetTensorData() const { - R* out; - ThrowOnError(GetApi().GetTensorMutableData(const_cast(this->p_), (void**)&out)); - return out; -} - -template -inline const void* ConstValueImpl::GetTensorRawData() const { - void* out; - ThrowOnError(GetApi().GetTensorMutableData(const_cast(this->p_), &out)); - return out; -} - -template -inline TypeInfo ConstValueImpl::GetTypeInfo() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetTypeInfo(this->p_, &output)); - return TypeInfo{output}; -} - -template -inline TensorTypeAndShapeInfo ConstValueImpl::GetTensorTypeAndShapeInfo() const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetTensorTypeAndShape(this->p_, &output)); - return TensorTypeAndShapeInfo{output}; -} - -template -inline ConstMemoryInfo ConstValueImpl::GetTensorMemoryInfo() const { - const OrtMemoryInfo* mem_info; - ThrowOnError(GetApi().GetTensorMemoryInfo(this->p_, &mem_info)); - return ConstMemoryInfo(mem_info); -} - -template -inline void ConstValueImpl::GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const { - ThrowOnError(GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, buffer)); -} - -template -inline std::string ConstValueImpl::GetStringTensorElement(size_t element_index) const { - size_t buffer_length; - ThrowOnError(GetApi().GetStringTensorElementLength(this->p_, element_index, &buffer_length)); - - std::string s; - s.resize(buffer_length); - ThrowOnError(GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, &s[0])); - return s; -} - -template -inline void ConstValueImpl::GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const { - ThrowOnError(GetApi().GetStringTensorContent(this->p_, buffer, buffer_length, offsets, offsets_count)); -} - -#if !defined(DISABLE_SPARSE_TENSORS) -template -inline OrtSparseFormat ConstValueImpl::GetSparseFormat() const { - OrtSparseFormat format; - ThrowOnError(GetApi().GetSparseTensorFormat(this->p_, &format)); - return format; -} - -template -inline TensorTypeAndShapeInfo ConstValueImpl::GetSparseTensorValuesTypeAndShapeInfo() const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetSparseTensorValuesTypeAndShape(this->p_, &output)); - return TensorTypeAndShapeInfo{output}; -} - -template -inline TensorTypeAndShapeInfo ConstValueImpl::GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat indices_format) const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetSparseTensorIndicesTypeShape(this->p_, indices_format, &output)); - return TensorTypeAndShapeInfo{output}; -} - -template -template -inline const R* ConstValueImpl::GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const { - const void* out; - ThrowOnError(GetApi().GetSparseTensorIndices(this->p_, indices_format, &num_indices, &out)); - return reinterpret_cast(out); -} - -template -inline bool ConstValueImpl::IsSparseTensor() const { - int out; - ThrowOnError(GetApi().IsSparseTensor(this->p_, &out)); - return out != 0; -} - -template -template -inline const R* ConstValueImpl::GetSparseTensorValues() const { - const void* out; - ThrowOnError(GetApi().GetSparseTensorValues(this->p_, &out)); - return reinterpret_cast(out); -} - -#endif - -template -void ValueImpl::FillStringTensor(const char* const* s, size_t s_len) { - ThrowOnError(GetApi().FillStringTensor(this->p_, s, s_len)); -} - -template -void ValueImpl::FillStringTensorElement(const char* s, size_t index) { - ThrowOnError(GetApi().FillStringTensorElement(this->p_, s, index)); -} - -template -inline char* ValueImpl::GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length) { - char* result; - ThrowOnError(GetApi().GetResizedStringTensorElementBuffer(this->p_, index, buffer_length, &result)); - return result; -} - -template -void* ValueImpl::GetTensorMutableRawData() { - void* out; - ThrowOnError(GetApi().GetTensorMutableData(this->p_, &out)); - return out; -} - -template -template -R* ValueImpl::GetTensorMutableData() { - R* out; - ThrowOnError(GetApi().GetTensorMutableData(this->p_, (void**)&out)); - return out; -} - -template -template -R& ValueImpl::At(const std::vector& location) { - static_assert(!std::is_same::value, "this api does not support std::string"); - R* out; - ThrowOnError(GetApi().TensorAt(this->p_, location.data(), location.size(), (void**)&out)); - return *out; -} - -#if !defined(DISABLE_SPARSE_TENSORS) -template -void ValueImpl::UseCooIndices(int64_t* indices_data, size_t indices_num) { - ThrowOnError(GetApi().UseCooIndices(this->p_, indices_data, indices_num)); -} - -template -void ValueImpl::UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num) { - ThrowOnError(GetApi().UseCsrIndices(this->p_, inner_data, inner_num, outer_data, outer_num)); -} - -template -void ValueImpl::UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data) { - ThrowOnError(GetApi().UseBlockSparseIndices(this->p_, indices_shape.shape, indices_shape.shape_len, indices_data)); -} - -template -void ValueImpl::FillSparseTensorCoo(const OrtMemoryInfo* mem_info, const OrtSparseValuesParam& values_param, - const int64_t* indices_data, size_t indices_num) { - ThrowOnError(GetApi().FillSparseTensorCoo(this->p_, mem_info, values_param.values_shape, - values_param.values_shape_len, values_param.data.p_data, - indices_data, indices_num)); -} - -template -void ValueImpl::FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const int64_t* inner_indices_data, size_t inner_indices_num, - const int64_t* outer_indices_data, size_t outer_indices_num) { - ThrowOnError(GetApi().FillSparseTensorCsr(this->p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, - inner_indices_data, inner_indices_num, - outer_indices_data, outer_indices_num)); -} - -template -void ValueImpl::FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const Shape& indices_shape, - const int32_t* indices_data) { - ThrowOnError(GetApi().FillSparseTensorBlockSparse(this->p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, - indices_shape.shape, indices_shape.shape_len, - indices_data)); -} - -#endif // !defined(DISABLE_SPARSE_TENSORS) - -} // namespace detail - -template -inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len) { - return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out)); - return Value{out}; -} - -template -inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) { - return CreateTensor(allocator, shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out)); - return Value{out}; -} - -#if !defined(DISABLE_SPARSE_TENSORS) - -template -inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, - const Shape& values_shape) { - return CreateSparseTensor(info, p_data, dense_shape, values_shape, TypeToTensorType::type); -} - -inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, - const Shape& values_shape, ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateSparseTensorWithValuesAsOrtValue(info, p_data, dense_shape.shape, dense_shape.shape_len, - values_shape.shape, values_shape.shape_len, type, &out)); - return Value{out}; -} - -template -inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape) { - return CreateSparseTensor(allocator, dense_shape, TypeToTensorType::type); -} - -inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, - ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateSparseTensorAsOrtValue(allocator, dense_shape.shape, dense_shape.shape_len, type, &out)); - return Value{out}; -} -#endif // !defined(DISABLE_SPARSE_TENSORS) - -inline Value Value::CreateMap(const Value& keys, const Value& values) { - OrtValue* out; - const OrtValue* inputs[2] = {keys, values}; - ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out)); - return Value{out}; -} - -inline Value Value::CreateSequence(const std::vector& values) { - OrtValue* out; - std::vector values_ort{values.data(), values.data() + values.size()}; - ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out)); - return Value{out}; -} - -template -inline Value Value::CreateOpaque(const char* domain, const char* type_name, const T& data_container) { - OrtValue* out; - ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out)); - return Value{out}; -} - -// -// Custom OP Inlines -// -inline Logger::Logger(const OrtLogger* logger) : logger_(logger) { - Ort::ThrowOnError(GetApi().Logger_GetLoggingSeverityLevel(this->logger_, &this->cached_severity_level_)); -} - -inline OrtLoggingLevel Logger::GetLoggingSeverityLevel() const noexcept { - return cached_severity_level_; -} - -inline Status Logger::LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, - const char* func_name, const char* message) const noexcept { - OrtStatus* status = GetApi().Logger_LogMessage(logger_, log_severity_level, message, file_path, line_number, - func_name); - return Status{status}; -} - -// Disable warnings about the format string not being a literal (-Wformat-nonliteral and -Wformat-security) -// for gcc and clang. The alternative is to use actual C-style variadic parameters and apply -// __attribute__(format(printf...)), which does not work with variadic templates. -#if defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat-nonliteral" -#pragma GCC diagnostic ignored "-Wformat-security" -#elif defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wformat-nonliteral" -#pragma clang diagnostic ignored "-Wformat-security" -#endif -template -inline Status Logger::LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, - int line_number, const char* func_name, const char* format, - Args&&... args) const noexcept { - int msg_len = std::snprintf(nullptr, 0U, format, std::forward(args)...); - - if (msg_len < 0) { // Formatting error - return Status("Failed to log message due to formatting error", OrtErrorCode::ORT_FAIL); - } - - OrtStatus* status = nullptr; - const size_t buffer_size = static_cast(msg_len) + 1U; - - constexpr size_t kStackBufferSize = 1024; - - if (buffer_size < kStackBufferSize) { - char buffer[kStackBufferSize]; - snprintf(buffer, kStackBufferSize, format, std::forward(args)...); - status = GetApi().Logger_LogMessage(logger_, log_severity_level, buffer, file_path, line_number, func_name); - } else { - // std::make_unique is only supported starting at C++14. -#if (__cplusplus >= 201402L) || (_MSC_VER >= 1900) - auto buffer = std::make_unique(buffer_size); -#else - std::unique_ptr buffer(new char[buffer_size]); -#endif - std::snprintf(buffer.get(), buffer_size, format, std::forward(args)...); - status = GetApi().Logger_LogMessage(logger_, log_severity_level, buffer.get(), file_path, line_number, func_name); - } - - return Status{status}; -} -// Re-enable -Wformat-nonliteral and -Wformat-security -#if defined(__GNUC__) -#pragma GCC diagnostic pop -#elif defined(__clang__) -#pragma clang diagnostic pop -#endif - -inline KernelContext::KernelContext(OrtKernelContext* context) : ctx_(context) { -} - -inline size_t KernelContext::GetInputCount() const { - size_t out = 0; - Ort::ThrowOnError(GetApi().KernelContext_GetInputCount(ctx_, &out)); - return out; -} - -inline size_t KernelContext::GetOutputCount() const { - size_t out = 0; - Ort::ThrowOnError(GetApi().KernelContext_GetOutputCount(ctx_, &out)); - return out; -} - -inline ConstValue KernelContext::GetInput(size_t index) const { - const OrtValue* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetInput(ctx_, index, &out)); - return ConstValue{out}; -} - -inline UnownedValue KernelContext::GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const { - OrtValue* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dim_values, dim_count, &out)); - return UnownedValue(out); -} - -inline UnownedValue KernelContext::GetOutput(size_t index, const std::vector& dims) const { - OrtValue* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dims.data(), dims.size(), &out)); - return UnownedValue(out); -} - -inline void* KernelContext::GetGPUComputeStream() const { - void* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetGPUComputeStream(ctx_, &out)); - return out; -} - -inline OrtAllocator* KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const { - OrtAllocator* out = nullptr; - Ort::ThrowOnError(GetApi().KernelContext_GetAllocator(ctx_, &memory_info, &out)); - return out; -} - -inline Logger KernelContext::GetLogger() const { - const OrtLogger* out = nullptr; - ThrowOnError(GetApi().KernelContext_GetLogger(this->ctx_, &out)); - return Logger{out}; -} - -inline void KernelContext::ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const { - ThrowOnError(GetApi().KernelContext_ParallelFor(ctx_, fn, total, num_batch, usr_data)); -} - -inline OpAttr::OpAttr(const char* name, const void* data, int len, OrtOpAttrType type) { - Ort::ThrowOnError(GetApi().CreateOpAttr(name, data, len, type, &p_)); -} - -namespace detail { -template -inline KernelInfo KernelInfoImpl::Copy() const { - OrtKernelInfo* info_copy = nullptr; - Ort::ThrowOnError(GetApi().CopyKernelInfo(this->p_, &info_copy)); - return KernelInfo{info_copy}; -} - -template -inline size_t KernelInfoImpl::GetInputCount() const { - size_t out = 0; - ThrowOnError(GetApi().KernelInfo_GetInputCount(this->p_, &out)); - return out; -} - -template -inline size_t KernelInfoImpl::GetOutputCount() const { - size_t out = 0; - ThrowOnError(GetApi().KernelInfo_GetOutputCount(this->p_, &out)); - return out; -} - -template -inline std::string KernelInfoImpl::GetInputName(size_t index) const { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline std::string KernelInfoImpl::GetOutputName(size_t index) const { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline TypeInfo KernelInfoImpl::GetInputTypeInfo(size_t index) const { - OrtTypeInfo* out = nullptr; - ThrowOnError(GetApi().KernelInfo_GetInputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline TypeInfo KernelInfoImpl::GetOutputTypeInfo(size_t index) const { - OrtTypeInfo* out = nullptr; - ThrowOnError(GetApi().KernelInfo_GetOutputTypeInfo(this->p_, index, &out)); - return TypeInfo{out}; -} - -template -inline Value KernelInfoImpl::GetTensorAttribute(const char* name, OrtAllocator* allocator) const { - OrtValue* out = nullptr; - ThrowOnError(GetApi().KernelInfoGetAttribute_tensor(this->p_, name, allocator, &out)); - return Value{out}; -} - -template -inline ConstValue KernelInfoImpl::GetTensorConstantInput(size_t index, int* is_constant) const { - const OrtValue* out = nullptr; - ThrowOnError(GetApi().KernelInfoGetConstantInput_tensor(this->p_, index, is_constant, &out)); - return ConstValue{out}; -} - -template -inline std::string KernelInfoImpl::GetNodeName() const { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the string value - Ort::ThrowOnError(GetApi().KernelInfo_GetNodeName(this->p_, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfo_GetNodeName(this->p_, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - - return out; -} - -template -inline Logger KernelInfoImpl::GetLogger() const { - const OrtLogger* out = nullptr; - ThrowOnError(GetApi().KernelInfo_GetLogger(this->p_, &out)); - return Logger{out}; -} - -inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, float& out) { - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_float(p, name, &out)); -} - -inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, int64_t& out) { - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_int64(p, name, &out)); -} - -inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, std::string& result) { - size_t size = 0; - // Feed nullptr for the data buffer to query the true size of the string attribute - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_string(p, name, nullptr, &size)); - - std::string out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_string(p, name, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - out.swap(result); -} - -inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std::vector& result) { - size_t size = 0; - // Feed nullptr for the data buffer to query the true size of the attribute - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_float(p, name, nullptr, &size)); - - std::vector out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_float(p, name, out.data(), &size)); - out.swap(result); -} - -inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std::vector& result) { - size_t size = 0; - - // Feed nullptr for the data buffer to query the true size of the attribute - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_int64(p, name, nullptr, &size)); - - std::vector out; - out.resize(size); - Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_int64(p, name, out.data(), &size)); - out.swap(result); -} -} // namespace detail - -inline KernelInfo::KernelInfo(OrtKernelInfo* info) : detail::KernelInfoImpl{info} {} - -inline Op::Op(OrtOp* p) : Base(p) {} - -inline Op Op::Create(const OrtKernelInfo* info, const char* op_name, const char* domain, int version, - const char** type_constraint_names, - const ONNXTensorElementDataType* type_constraint_values, - size_t type_constraint_count, - const OpAttr* attr_values, size_t attr_count, - size_t input_count, size_t output_count) { - static_assert(sizeof(OpAttr) == sizeof(OrtOpAttr*), - "OpAttr's is expected to be just an array of OrtOpAttr in memory so we can reinterpret safely"); - auto attr_input_values = reinterpret_cast(attr_values); - OrtOp* op; - Ort::ThrowOnError(GetApi().CreateOp(info, op_name, domain, version, type_constraint_names, type_constraint_values, - static_cast(type_constraint_count), - attr_input_values, - static_cast(attr_count), - static_cast(input_count), - static_cast(output_count), &op)); - return Op{op}; -} - -inline void Op::Invoke(const OrtKernelContext* context, - const Value* input_values, - size_t input_count, - Value* output_values, - size_t output_count) { - static_assert(sizeof(Value) == sizeof(OrtValue*), - "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); - auto ort_input_values = reinterpret_cast(input_values); - auto ort_output_values = reinterpret_cast(output_values); - Ort::ThrowOnError(GetApi().InvokeOp(context, p_, ort_input_values, static_cast(input_count), - ort_output_values, static_cast(output_count))); -} - -inline void Op::Invoke(const OrtKernelContext* context, - const OrtValue* const* input_values, - size_t input_count, - OrtValue* const* output_values, - size_t output_count) { - Ort::ThrowOnError(GetApi().InvokeOp(context, p_, input_values, static_cast(input_count), - output_values, static_cast(output_count))); -} - -inline std::string GetVersionString() { - return OrtGetApiBase()->GetVersionString(); -} - -inline std::string GetBuildInfoString() { - return GetApi().GetBuildInfoString(); -} - -inline std::vector GetAvailableProviders() { - char** providers; - int len; - - auto release_fn = [&len](char** providers) { - // This should always return nullptr. - ThrowOnError(GetApi().ReleaseAvailableProviders(providers, len)); - }; - - ThrowOnError(GetApi().GetAvailableProviders(&providers, &len)); - std::unique_ptr guard(providers, release_fn); - std::vector available_providers; - available_providers.reserve(static_cast(len)); - for (int i = 0; i < len; ++i) { - available_providers.emplace_back(providers[i]); - } - return available_providers; -} - -template -void CustomOpBase::GetSessionConfigs(std::unordered_map& out, - ConstSessionOptions options) const { - const TOp* derived = static_cast(this); - std::vector keys = derived->GetSessionConfigKeys(); - - out.reserve(keys.size()); - - std::string config_entry_key = detail::MakeCustomOpConfigEntryKey(derived->GetName(), ""); - const size_t prefix_size = config_entry_key.length(); - - for (const auto& key : keys) { - config_entry_key.resize(prefix_size); - config_entry_key.append(key); - out[key] = options.GetConfigEntryOrDefault(config_entry_key.c_str(), ""); - } -} - -inline ShapeInferContext::ShapeInferContext(const OrtApi* ort_api, - OrtShapeInferContext* ctx) : ort_api_(ort_api), ctx_(ctx) { - size_t input_count = 0; - Ort::ThrowOnError(ort_api_->ShapeInferContext_GetInputCount(ctx_, &input_count)); - for (size_t ith_input = 0; ith_input < input_count; ++ith_input) { - OrtTensorTypeAndShapeInfo* info{}; - Ort::ThrowOnError(ort_api_->ShapeInferContext_GetInputTypeShape(ctx, ith_input, &info)); - TensorTypeAndShapeInfo type_shape_info(info); - auto integer_shape = type_shape_info.GetShape(); - std::vector symbolic_shape(integer_shape.size(), {}); - type_shape_info.GetSymbolicDimensions(&symbolic_shape[0], integer_shape.size()); - Shape shape; - for (size_t ith = 0; ith < integer_shape.size(); ++ith) { - if (symbolic_shape[ith] && std::string{symbolic_shape[ith]}.size() > 0) { - shape.emplace_back(symbolic_shape[ith]); - } else { - shape.emplace_back(integer_shape[ith]); - } - } - input_shapes_.push_back(std::move(shape)); - type_shape_info.release(); - } -} - -inline Status ShapeInferContext::SetOutputShape(size_t indice, const Shape& shape) { - OrtTensorTypeAndShapeInfo* info = {}; - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->CreateTensorTypeAndShapeInfo(&info)); - - using InfoPtr = std::unique_ptr>; - - InfoPtr info_ptr(info, [this](OrtTensorTypeAndShapeInfo* obj) { - ort_api_->ReleaseTensorTypeAndShapeInfo(obj); - }); - - std::vector integer_dims; - std::vector symbolic_dims; - - for (const auto dim : shape) { - if (dim.IsInt()) { - integer_dims.push_back(dim.IsInt()); - symbolic_dims.push_back(""); - } else { - if (!dim.AsSym() || std::string{dim.AsSym()}.empty()) { - ORT_CXX_API_THROW("Symbolic dim must not be an empty string", ORT_INVALID_ARGUMENT); - } - integer_dims.push_back(SymbolicInteger::INVALID_INT_DIM); - symbolic_dims.push_back(dim.AsSym()); - } - } - - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetDimensions(info, integer_dims.data(), integer_dims.size())); - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetSymbolicDimensions(info, symbolic_dims.data(), symbolic_dims.size())); - ORT_CXX_RETURN_ON_API_FAIL(ort_api_->ShapeInferContext_SetOutputTypeShape(ctx_, indice, info)); - return Status{nullptr}; -} - -inline int64_t ShapeInferContext::GetAttrInt(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - int64_t i = {}; - size_t out = {}; - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INT, &i, sizeof(i), &out)); - return i; -} - -inline ShapeInferContext::Ints ShapeInferContext::GetAttrInts(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - int64_t i = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, &i, sizeof(i), &out); - if (status) { - size_t num_i = out / sizeof(int64_t); - ShapeInferContext::Ints ints(num_i, 0); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, ints.data(), out, &out)); - return ints; - } else { - return {i}; - } -} - -inline float ShapeInferContext::GetAttrFloat(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - float f = {}; - size_t out = {}; - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOAT, &f, sizeof(f), &out)); - return f; -} - -inline ShapeInferContext::Floats ShapeInferContext::GetAttrFloats(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - float f = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, &f, sizeof(f), &out); - if (status) { - size_t num_f = out / sizeof(float); - ShapeInferContext::Floats floats(num_f, 0); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, floats.data(), out, &out)); - return floats; - } else { - return {f}; - } -} - -inline std::string ShapeInferContext::GetAttrString(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - char c = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, &c, sizeof(char), &out); - if (status) { - std::vector chars(out, '\0'); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, chars.data(), out, &out)); - return {chars.data()}; - } else { - return {c}; - } -} - -inline ShapeInferContext::Strings ShapeInferContext::GetAttrStrings(const char* attr_name) { - const auto* attr = GetAttrHdl(attr_name); - char c = {}; - size_t out = {}; - // first call to get the bytes needed - auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, &c, sizeof(char), &out); - if (status) { - std::vector chars(out, '\0'); - Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, chars.data(), out, &out)); - ShapeInferContext::Strings strings; - char* char_st = chars.data(); - char* char_ed = char_st + out; - while (char_st < char_ed) { - strings.emplace_back(char_st); - while (*char_st != '\0') { - char_st++; - } - char_st++; - } - return strings; - } else { - return {std::string{c}}; - } -} - -inline const OrtOpAttr* ShapeInferContext::GetAttrHdl(const char* attr_name) const { - const OrtOpAttr* attr_hdl = {}; - Ort::ThrowOnError(ort_api_->ShapeInferContext_GetAttribute(ctx_, attr_name, &attr_hdl)); - return attr_hdl; -} - -} // namespace Ort diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_float16.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_float16.h deleted file mode 100644 index 0b066a9..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_float16.h +++ /dev/null @@ -1,540 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include -#include -#include -#include - -namespace onnxruntime_float16 { - -namespace detail { - -enum class endian { -#if defined(_WIN32) - little = 0, - big = 1, - native = little, -#elif defined(__GNUC__) || defined(__clang__) - little = __ORDER_LITTLE_ENDIAN__, - big = __ORDER_BIG_ENDIAN__, - native = __BYTE_ORDER__, -#else -#error onnxruntime_float16::detail::endian is not implemented in this environment. -#endif -}; - -static_assert( - endian::native == endian::little || endian::native == endian::big, - "Only little-endian or big-endian native byte orders are supported."); - -} // namespace detail - -/// -/// Shared implementation between public and internal classes. CRTP pattern. -/// -template -struct Float16Impl { - protected: - /// - /// Converts from float to uint16_t float16 representation - /// - /// - /// - constexpr static uint16_t ToUint16Impl(float v) noexcept; - - /// - /// Converts float16 to float - /// - /// float representation of float16 value - float ToFloatImpl() const noexcept; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - uint16_t AbsImpl() const noexcept { - return static_cast(val & ~kSignMask); - } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - uint16_t NegateImpl() const noexcept { - return IsNaN() ? val : static_cast(val ^ kSignMask); - } - - public: - // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7C00U; - static constexpr uint16_t kPositiveInfinityBits = 0x7C00U; - static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; - static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; - static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; - static constexpr uint16_t kEpsilonBits = 0x4170U; - static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number - static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number - static constexpr uint16_t kOneBits = 0x3C00U; - static constexpr uint16_t kMinusOneBits = 0xBC00U; - - uint16_t val{0}; - - Float16Impl() = default; - - /// - /// Checks if the value is negative - /// - /// true if negative - bool IsNegative() const noexcept { - return static_cast(val) < 0; - } - - /// - /// Tests if the value is NaN - /// - /// true if NaN - bool IsNaN() const noexcept { - return AbsImpl() > kPositiveInfinityBits; - } - - /// - /// Tests if the value is finite - /// - /// true if finite - bool IsFinite() const noexcept { - return AbsImpl() < kPositiveInfinityBits; - } - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - bool IsPositiveInfinity() const noexcept { - return val == kPositiveInfinityBits; - } - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - bool IsNegativeInfinity() const noexcept { - return val == kNegativeInfinityBits; - } - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - bool IsInfinity() const noexcept { - return AbsImpl() == kPositiveInfinityBits; - } - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - bool IsNaNOrZero() const noexcept { - auto abs = AbsImpl(); - return (abs == 0 || abs > kPositiveInfinityBits); - } - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - bool IsNormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) - } - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - bool IsSubnormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) - } - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) noexcept { - return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; - } - - bool operator==(const Float16Impl& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is not equal to anything, including itself. - return false; - } - return val == rhs.val; - } - - bool operator!=(const Float16Impl& rhs) const noexcept { return !(*this == rhs); } - - bool operator<(const Float16Impl& rhs) const noexcept { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is unordered with respect to everything, including itself. - return false; - } - - const bool left_is_negative = IsNegative(); - if (left_is_negative != rhs.IsNegative()) { - // When the signs of left and right differ, we know that left is less than right if it is - // the negative value. The exception to this is if both values are zero, in which case IEEE - // says they should be equal, even if the signs differ. - return left_is_negative && !AreZero(*this, rhs); - } - return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); - } -}; - -// The following Float16_t conversions are based on the code from -// Eigen library. - -// The conversion routines are Copyright (c) Fabian Giesen, 2016. -// The original license follows: -// -// Copyright (c) Fabian Giesen, 2016 -// All rights reserved. -// Redistribution and use in source and binary forms, with or without -// modification, are permitted. -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -namespace detail { -union float32_bits { - unsigned int u; - float f; -}; -} // namespace detail - -template -inline constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept { - detail::float32_bits f{}; - f.f = v; - - constexpr detail::float32_bits f32infty = {255 << 23}; - constexpr detail::float32_bits f16max = {(127 + 16) << 23}; - constexpr detail::float32_bits denorm_magic = {((127 - 15) + (23 - 10) + 1) << 23}; - constexpr unsigned int sign_mask = 0x80000000u; - uint16_t val = static_cast(0x0u); - - unsigned int sign = f.u & sign_mask; - f.u ^= sign; - - // NOTE all the integer compares in this function can be safely - // compiled into signed compares since all operands are below - // 0x80000000. Important if you want fast straight SSE2 code - // (since there's no unsigned PCMPGTD). - - if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) - val = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf - } else { // (De)normalized number or zero - if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero - // use a magic value to align our 10 mantissa bits at the bottom of - // the float. as long as FP addition is round-to-nearest-even this - // just works. - f.f += denorm_magic.f; - - // and one integer subtract of the bias later, we have our final float! - val = static_cast(f.u - denorm_magic.u); - } else { - unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd - - // update exponent, rounding bias part 1 - // Equivalent to `f.u += ((unsigned int)(15 - 127) << 23) + 0xfff`, but - // without arithmetic overflow. - f.u += 0xc8000fffU; - // rounding bias part 2 - f.u += mant_odd; - // take the bits! - val = static_cast(f.u >> 13); - } - } - - val |= static_cast(sign >> 16); - return val; -} - -template -inline float Float16Impl::ToFloatImpl() const noexcept { - constexpr detail::float32_bits magic = {113 << 23}; - constexpr unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift - detail::float32_bits o{}; - - o.u = (val & 0x7fff) << 13; // exponent/mantissa bits - unsigned int exp = shifted_exp & o.u; // just the exponent - o.u += (127 - 15) << 23; // exponent adjust - - // handle exponent special cases - if (exp == shifted_exp) { // Inf/NaN? - o.u += (128 - 16) << 23; // extra exp adjust - } else if (exp == 0) { // Zero/Denormal? - o.u += 1 << 23; // extra exp adjust - o.f -= magic.f; // re-normalize - } - - // Attempt to workaround the Internal Compiler Error on ARM64 - // for bitwise | operator, including std::bitset -#if (defined _MSC_VER) && (defined _M_ARM || defined _M_ARM64 || defined _M_ARM64EC) - if (IsNegative()) { - return -o.f; - } -#else - // original code: - o.u |= (val & 0x8000U) << 16U; // sign bit -#endif - return o.f; -} - -/// Shared implementation between public and internal classes. CRTP pattern. -template -struct BFloat16Impl { - protected: - /// - /// Converts from float to uint16_t float16 representation - /// - /// - /// - static uint16_t ToUint16Impl(float v) noexcept; - - /// - /// Converts bfloat16 to float - /// - /// float representation of bfloat16 value - float ToFloatImpl() const noexcept; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - uint16_t AbsImpl() const noexcept { - return static_cast(val & ~kSignMask); - } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - uint16_t NegateImpl() const noexcept { - return IsNaN() ? val : static_cast(val ^ kSignMask); - } - - public: - // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7F80U; - static constexpr uint16_t kPositiveInfinityBits = 0x7F80U; - static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; - static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; - static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; - static constexpr uint16_t kSignaling_NaNBits = 0x7F80U; - static constexpr uint16_t kEpsilonBits = 0x0080U; - static constexpr uint16_t kMinValueBits = 0xFF7FU; - static constexpr uint16_t kMaxValueBits = 0x7F7FU; - static constexpr uint16_t kRoundToNearest = 0x7FFFU; - static constexpr uint16_t kOneBits = 0x3F80U; - static constexpr uint16_t kMinusOneBits = 0xBF80U; - - uint16_t val{0}; - - BFloat16Impl() = default; - - /// - /// Checks if the value is negative - /// - /// true if negative - bool IsNegative() const noexcept { - return static_cast(val) < 0; - } - - /// - /// Tests if the value is NaN - /// - /// true if NaN - bool IsNaN() const noexcept { - return AbsImpl() > kPositiveInfinityBits; - } - - /// - /// Tests if the value is finite - /// - /// true if finite - bool IsFinite() const noexcept { - return AbsImpl() < kPositiveInfinityBits; - } - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - bool IsPositiveInfinity() const noexcept { - return val == kPositiveInfinityBits; - } - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - bool IsNegativeInfinity() const noexcept { - return val == kNegativeInfinityBits; - } - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - bool IsInfinity() const noexcept { - return AbsImpl() == kPositiveInfinityBits; - } - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - bool IsNaNOrZero() const noexcept { - auto abs = AbsImpl(); - return (abs == 0 || abs > kPositiveInfinityBits); - } - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - bool IsNormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) - } - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - bool IsSubnormal() const noexcept { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) - } - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) noexcept { - // IEEE defines that positive and negative zero are equal, this gives us a quick equality check - // for two values by or'ing the private bits together and stripping the sign. They are both zero, - // and therefore equivalent, if the resulting value is still zero. - return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; - } -}; - -template -inline uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept { - uint16_t result; - if (std::isnan(v)) { - result = kPositiveQNaNBits; - } else { - auto get_msb_half = [](float fl) { - uint16_t result; -#ifdef __cpp_if_constexpr - if constexpr (detail::endian::native == detail::endian::little) { -#else - if (detail::endian::native == detail::endian::little) { -#endif - std::memcpy(&result, reinterpret_cast(&fl) + sizeof(uint16_t), sizeof(uint16_t)); - } else { - std::memcpy(&result, &fl, sizeof(uint16_t)); - } - return result; - }; - - uint16_t upper_bits = get_msb_half(v); - union { - uint32_t U32; - float F32; - }; - F32 = v; - U32 += (upper_bits & 1) + kRoundToNearest; - result = get_msb_half(F32); - } - return result; -} - -template -inline float BFloat16Impl::ToFloatImpl() const noexcept { - if (IsNaN()) { - return std::numeric_limits::quiet_NaN(); - } - float result; - char* const first = reinterpret_cast(&result); - char* const second = first + sizeof(uint16_t); -#ifdef __cpp_if_constexpr - if constexpr (detail::endian::native == detail::endian::little) { -#else - if (detail::endian::native == detail::endian::little) { -#endif - std::memset(first, 0, sizeof(uint16_t)); - std::memcpy(second, &val, sizeof(uint16_t)); - } else { - std::memcpy(first, &val, sizeof(uint16_t)); - std::memset(second, 0, sizeof(uint16_t)); - } - return result; -} - -} // namespace onnxruntime_float16 diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_lite_custom_op.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_lite_custom_op.h deleted file mode 100644 index ee60f25..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_lite_custom_op.h +++ /dev/null @@ -1,1119 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Summary -// The header has APIs to save custom op authors the trouble of defining schemas, -// which will be inferred by functions' signature, as long as their argument list has types supported here. -// Input could be: -// 1. Tensor of onnx data types. -// 2. Span of onnx data types. -// 3. Scalar of onnx data types. -// A input could be optional if indicated as std::optional<...>. -// For an output, it must be a tensor of onnx data types. -// Further, the header also has utility for a simple custom struct, where resources could be kept, to be registered as a custom op. -// For concrete examples, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". -// Note - all APIs in this header are ABI. - -#pragma once -#include "onnxruntime_cxx_api.h" -#include -#include -#include -#include - -namespace Ort { -namespace Custom { - -class ArgBase { - public: - ArgBase(OrtKernelContext* ctx, - size_t indice, - bool is_input) : ctx_(ctx), indice_(indice), is_input_(is_input) {} - virtual ~ArgBase(){}; - - protected: - struct KernelContext ctx_; - size_t indice_; - bool is_input_; -}; - -using ArgPtr = std::unique_ptr; -using ArgPtrs = std::vector; - -class TensorBase : public ArgBase { - public: - TensorBase(OrtKernelContext* ctx, - size_t indice, - bool is_input) : ArgBase(ctx, indice, is_input) {} - - operator bool() const { - return shape_.has_value(); - } - - const std::vector& Shape() const { - if (!shape_.has_value()) { - ORT_CXX_API_THROW("tensor shape is not yet initialized", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return shape_.value(); - } - - ONNXTensorElementDataType Type() const { - return type_; - } - - int64_t NumberOfElement() const { - if (shape_.has_value()) { - return std::accumulate(shape_->begin(), shape_->end(), 1LL, std::multiplies()); - } else { - return 0; - } - } - - std::string Shape2Str() const { - if (shape_.has_value()) { - std::string shape_str; - for (const auto& dim : *shape_) { - shape_str.append(std::to_string(dim)); - shape_str.append(", "); - } - return shape_str; - } else { - return "empty"; - } - } - - bool IsCpuTensor() const { - return strcmp("Cpu", mem_type_) == 0; - } - - virtual const void* DataRaw() const = 0; - virtual size_t SizeInBytes() const = 0; - - protected: - std::optional> shape_; - ONNXTensorElementDataType type_ = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; - const char* mem_type_ = "Cpu"; -}; - -template -struct Span { - const T* data_ = {}; - size_t size_ = {}; - void Assign(const T* data, size_t size) { - data_ = data; - size_ = size; - } - size_t size() const { return size_; } - T operator[](size_t indice) const { - return data_[indice]; - } - const T* data() const { return data_; } -}; - -template -class Tensor : public TensorBase { - public: - using TT = typename std::remove_reference::type; - Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { - if (is_input_) { - if (indice >= ctx_.GetInputCount()) { - ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - const_value_ = ctx_.GetInput(indice); - auto type_shape_info = const_value_.GetTensorTypeAndShapeInfo(); - shape_ = type_shape_info.GetShape(); - } - } - const TT* Data() const { - return reinterpret_cast(const_value_.GetTensorRawData()); - } - TT* Allocate(const std::vector& shape) { - shape_ = shape; - if (!data_) { - shape_ = shape; - data_ = ctx_.GetOutput(indice_, shape).template GetTensorMutableData(); - } - return data_; - } - static TT GetT() { return (TT)0; } - const Span& AsSpan() { - if (!shape_.has_value() || shape_->size() != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a span out of Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - span_.Assign(Data(), static_cast((*shape_)[0])); - return span_; - } - const T& AsScalar() { - if (!shape_.has_value() || shape_->size() != 1 || (*shape_)[0] != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a scalar from Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return *Data(); - } - const void* DataRaw() const override { - return reinterpret_cast(Data()); - } - - size_t SizeInBytes() const override { - return sizeof(TT) * static_cast(NumberOfElement()); - } - - private: - ConstValue const_value_; // for input - TT* data_{}; // for output - Span span_; -}; - -template <> -class Tensor : public TensorBase { - public: - using strings = std::vector; - - Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { - if (is_input_) { - if (indice >= ctx_.GetInputCount()) { - ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - auto const_value = ctx_.GetInput(indice); - auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); - shape_ = type_shape_info.GetShape(); - auto num_chars = const_value.GetStringTensorDataLength(); - // note - there will be copy ... - auto num_strings = static_cast(NumberOfElement()); - if (num_strings) { - std::vector chars(num_chars + 1, '\0'); - std::vector offsets(num_strings); - const_value.GetStringTensorContent(static_cast(chars.data()), num_chars, offsets.data(), offsets.size()); - auto upper_bound = num_strings - 1; - input_strings_.resize(num_strings); - for (size_t i = upper_bound;; --i) { - if (i < upper_bound) { - chars[offsets[i + 1]] = '\0'; - } - input_strings_[i] = chars.data() + offsets[i]; - if (0 == i) { - break; - } - } - } - } - } - const strings& Data() const { - return input_strings_; - } - const void* DataRaw() const override { - if (input_strings_.size() != 1) { - ORT_CXX_API_THROW("DataRaw() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return reinterpret_cast(input_strings_[0].c_str()); - } - size_t SizeInBytes() const override { - if (input_strings_.size() != 1) { - ORT_CXX_API_THROW("SizeInBytes() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return input_strings_[0].size(); - } - void SetStringOutput(const strings& ss, const std::vector& dims) { - shape_ = dims; - std::vector raw; - for (const auto& s : ss) { - raw.push_back(s.data()); - } - auto output = ctx_.GetOutput(indice_, dims.data(), dims.size()); - // note - there will be copy ... - output.FillStringTensor(raw.data(), raw.size()); - } - const Span& AsSpan() { - ORT_CXX_API_THROW("span for TensorT of string not implemented", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - const std::string& AsScalar() { - if (input_strings_.size() != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a scalar string from Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return input_strings_[0]; - } - - private: - std::vector input_strings_; // for input -}; - -template <> -class Tensor : public TensorBase { - public: - using strings = std::vector; - using string_views = std::vector; - - Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { - if (is_input_) { - if (indice >= ctx_.GetInputCount()) { - ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); - } - auto const_value = ctx_.GetInput(indice); - auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); - shape_ = type_shape_info.GetShape(); - auto num_chars = const_value.GetStringTensorDataLength(); - chars_.resize(num_chars + 1, '\0'); - auto num_strings = static_cast(NumberOfElement()); - if (num_strings) { - std::vector offsets(num_strings); - const_value.GetStringTensorContent(static_cast(chars_.data()), num_chars, offsets.data(), offsets.size()); - offsets.push_back(num_chars); - for (size_t i = 0; i < num_strings; ++i) { - input_string_views_.emplace_back(chars_.data() + offsets[i], offsets[i + 1] - offsets[i]); - } - } - } - } - const string_views& Data() const { - return input_string_views_; - } - const void* DataRaw() const override { - if (input_string_views_.size() != 1) { - ORT_CXX_API_THROW("DataRaw() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return reinterpret_cast(input_string_views_[0].data()); - } - size_t SizeInBytes() const override { - if (input_string_views_.size() != 1) { - ORT_CXX_API_THROW("SizeInBytes() only applies to string scalar", ORT_RUNTIME_EXCEPTION); - } - return input_string_views_[0].size(); - } - void SetStringOutput(const strings& ss, const std::vector& dims) { - shape_ = dims; - std::vector raw; - for (const auto& s : ss) { - raw.push_back(s.data()); - } - auto output = ctx_.GetOutput(indice_, dims.data(), dims.size()); - // note - there will be copy ... - output.FillStringTensor(raw.data(), raw.size()); - } - const Span& AsSpan() { - ORT_CXX_API_THROW("span for TensorT of string view not implemented", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - std::string_view AsScalar() { - if (input_string_views_.size() != 1) { - ORT_CXX_API_THROW("invalid shape while trying to get a scalar string view from Ort::Custom::Tensor", - OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - return input_string_views_[0]; - } - - private: - std::vector chars_; // for input - std::vector input_string_views_; // for input -}; - -using TensorPtr = std::unique_ptr; -using TensorPtrs = std::vector; - -struct TensorArray : public ArgBase { - TensorArray(OrtKernelContext* ctx, - size_t start_indice, - bool is_input) : ArgBase(ctx, - start_indice, - is_input) { - if (is_input) { - auto input_count = ctx_.GetInputCount(); - for (size_t ith_input = start_indice; ith_input < input_count; ++ith_input) { - auto const_value = ctx_.GetInput(start_indice); - auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); - auto type = type_shape_info.GetElementType(); - TensorPtr tensor; - switch (type) { - case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: - tensor = std::make_unique>(ctx, ith_input, true); - break; - case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: - tensor = std::make_unique>(ctx, ith_input, true); - break; - default: - ORT_CXX_API_THROW("unknow input type", ORT_RUNTIME_EXCEPTION); - break; - } - tensors_.emplace_back(tensor.release()); - } // for - } - } - template - T* AllocateOutput(size_t ith_output, const std::vector& shape) { - // ith_output is the indice of output relative to the tensor array - // indice_ + ith_output is the indice relative to context - auto tensor = std::make_unique>(ctx_.GetOrtKernelContext(), indice_ + ith_output, false); - auto raw_output = tensor.get()->Allocate(shape); - tensors_.emplace_back(tensor.release()); - return raw_output; - } - Tensor& AllocateStringTensor(size_t ith_output) { - // ith_output is the indice of output relative to the tensor array - // indice_ + ith_output is the indice relative to context - auto tensor = std::make_unique>(ctx_.GetOrtKernelContext(), indice_ + ith_output, false); - Tensor& output = *tensor; - tensors_.emplace_back(tensor.release()); - return output; - } - size_t Size() const { - return tensors_.size(); - } - const TensorPtr& operator[](size_t ith_input) const { - // ith_input is the indice of output relative to the tensor array - return tensors_.at(ith_input); - } - - private: - TensorPtrs tensors_; -}; - -using Variadic = TensorArray; - -/* -Note: -OrtLiteCustomOp inherits from OrtCustomOp to bridge tween a custom func/struct and ort core. -The lifetime of an OrtLiteCustomOp instance is managed by customer code, not ort, so: -1. DO NOT cast OrtLiteCustomOp to OrtCustomOp and release since there is no virtual destructor in the hierachy. -2. OrtLiteCustomFunc and OrtLiteCustomStruct, as two sub-structs, can be released in form of OrtLiteCustomOp since all members are kept in the OrtLiteCustomOp, - hence memory could still be recycled properly. -Further, OrtCustomOp is a c struct bearing no v-table, so offspring structs are by design to be of zero virtual functions to maintain cast safety. -*/ -struct OrtLiteCustomOp : public OrtCustomOp { - using ConstOptionalFloatTensor = std::optional&>; - using OptionalFloatTensor = std::optional>; - - // CreateTuple - template - static typename std::enable_if>::type - CreateTuple(OrtKernelContext*, ArgPtrs&, size_t, size_t, const std::string&) { - return std::make_tuple(); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - std::tuple current = std::tuple{context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - std::tuple current = std::tuple{*context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - -#ifdef ORT_CUDA_CTX - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - thread_local CudaContext cuda_context; - cuda_context.Init(*context); - std::tuple current = std::tuple{cuda_context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } -#endif - -#ifdef ORT_ROCM_CTX - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - thread_local RocmContext rocm_context; - rocm_context.Init(*context); - std::tuple current = std::tuple{rocm_context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } -#endif - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_input, true)); - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_input, true)); - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_output, false)); - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - args.push_back(std::make_unique(context, ith_output, false)); - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } - -#define CREATE_TUPLE_INPUT(data_type) \ - template \ - static typename std::enable_if*>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if&>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if*>>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_input < num_input) { \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } \ - template \ - static typename std::enable_if*>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{&reinterpret_cast*>(args.back().get())->AsSpan()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if&>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsSpan()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if*>>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_input < num_input) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{&reinterpret_cast*>(args.back().get())->AsSpan()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } \ - template \ - static typename std::enable_if::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("scalar input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsScalar()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_input < num_input) { \ - if ("CPUExecutionProvider" != ep) { \ - ORT_CXX_API_THROW("scalar input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ - } \ - args.push_back(std::make_unique>(context, ith_input, true)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsScalar()}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } -#define CREATE_TUPLE_OUTPUT(data_type) \ - template \ - static typename std::enable_if*>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_output, false)); \ - std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if&>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - args.push_back(std::make_unique>(context, ith_output, false)); \ - std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - template \ - static typename std::enable_if*>>::value, std::tuple>::type \ - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ - if (ith_output < num_output) { \ - args.push_back(std::make_unique>(context, ith_output, false)); \ - std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } else { \ - std::tuple current = std::tuple{}; \ - auto next = CreateTuple(context, args, num_input, num_output, ep); \ - return std::tuple_cat(current, next); \ - } \ - } -#define CREATE_TUPLE(data_type) \ - CREATE_TUPLE_INPUT(data_type) \ - CREATE_TUPLE_OUTPUT(data_type) - - CREATE_TUPLE(bool) - CREATE_TUPLE(float) - CREATE_TUPLE(Ort::Float16_t) - CREATE_TUPLE(Ort::BFloat16_t) - CREATE_TUPLE(double) - CREATE_TUPLE(int8_t) - CREATE_TUPLE(int16_t) - CREATE_TUPLE(int32_t) - CREATE_TUPLE(int64_t) - CREATE_TUPLE(uint8_t) - CREATE_TUPLE(uint16_t) - CREATE_TUPLE(uint32_t) - CREATE_TUPLE(uint64_t) - CREATE_TUPLE(std::string) - CREATE_TUPLE_INPUT(std::string_view) - CREATE_TUPLE(Ort::Float8E4M3FN_t) - CREATE_TUPLE(Ort::Float8E4M3FNUZ_t) - CREATE_TUPLE(Ort::Float8E5M2_t) - CREATE_TUPLE(Ort::Float8E5M2FNUZ_t) - - // ParseArgs ... - template - static typename std::enable_if<0 == sizeof...(Ts)>::type - ParseArgs(std::vector&, std::vector&) { - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } - -#ifdef ORT_CUDA_CTX - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } -#endif - -#ifdef ORT_ROCM_CTX - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } -#endif - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - input_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - input_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - output_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - output_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); - ParseArgs(input_types, output_types); - } - -#define PARSE_INPUT_BASE(pack_type, onnx_type) \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - input_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - input_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - input_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } - -#define PARSE_INPUT(data_type, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Tensor*, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Tensor&, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Span*, onnx_type) \ - PARSE_INPUT_BASE(const Custom::Span&, onnx_type) \ - PARSE_INPUT_BASE(data_type, onnx_type) - -#define PARSE_OUTPUT(data_type, onnx_type) \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same*>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - output_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same&>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - output_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } \ - template \ - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same*>>::value>::type \ - ParseArgs(std::vector& input_types, std::vector& output_types) { \ - output_types.push_back(onnx_type); \ - ParseArgs(input_types, output_types); \ - } - -#define PARSE_ARGS(data_type, onnx_type) \ - PARSE_INPUT(data_type, onnx_type) \ - PARSE_OUTPUT(data_type, onnx_type) - - PARSE_ARGS(bool, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL) - PARSE_ARGS(float, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) - PARSE_ARGS(Ort::Float16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) - PARSE_ARGS(Ort::BFloat16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16) - PARSE_ARGS(double, ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE) - PARSE_ARGS(int8_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) - PARSE_ARGS(int16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16) - PARSE_ARGS(int32_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) - PARSE_ARGS(int64_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) - PARSE_ARGS(uint8_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8) - PARSE_ARGS(uint16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16) - PARSE_ARGS(uint32_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32) - PARSE_ARGS(uint64_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64) - PARSE_ARGS(std::string, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) - PARSE_ARGS(std::string_view, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) // todo - remove string_view output - PARSE_ARGS(Ort::Float8E4M3FN_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN) - PARSE_ARGS(Ort::Float8E4M3FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ) - PARSE_ARGS(Ort::Float8E5M2_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2) - PARSE_ARGS(Ort::Float8E5M2FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ) - - OrtLiteCustomOp(const char* op_name, - const char* execution_provider, - ShapeInferFn shape_infer_fn, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : op_name_(op_name), - execution_provider_(execution_provider), - shape_infer_fn_(shape_infer_fn), - start_ver_(start_ver), - end_ver_(end_ver) { - OrtCustomOp::version = ORT_API_VERSION; - - OrtCustomOp::GetName = [](const OrtCustomOp* op) { return static_cast(op)->op_name_.c_str(); }; - OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* op) { return ((OrtLiteCustomOp*)op)->execution_provider_.c_str(); }; - OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp*, size_t) { return OrtMemTypeDefault; }; - - OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->input_types_.size(); - }; - - OrtCustomOp::GetInputType = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->input_types_[indice]; - }; - - OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->output_types_.size(); - }; - - OrtCustomOp::GetOutputType = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->output_types_[indice]; - }; - - OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->input_types_[indice] == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED ? INPUT_OUTPUT_VARIADIC : INPUT_OUTPUT_OPTIONAL; - }; - - OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* op, size_t indice) { - auto self = reinterpret_cast(op); - return self->output_types_[indice] == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED ? INPUT_OUTPUT_VARIADIC : INPUT_OUTPUT_OPTIONAL; - }; - - OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp*) { - return 1; - }; - - OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp*) { - return 0; - }; - - OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp*) { - return 1; - }; - - OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp*) { - return 0; - }; - - OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp*) { return 0; }; - OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp*) { return 0; }; - OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp*) { return 0; }; - OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp*) { return 0; }; - - OrtCustomOp::CreateKernelV2 = {}; - OrtCustomOp::KernelComputeV2 = {}; - OrtCustomOp::KernelCompute = {}; - - OrtCustomOp::InferOutputShapeFn = {}; - - OrtCustomOp::GetStartVersion = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->start_ver_; - }; - - OrtCustomOp::GetEndVersion = [](const OrtCustomOp* op) { - auto self = reinterpret_cast(op); - return self->end_ver_; - }; - - OrtCustomOp::GetMayInplace = {}; - OrtCustomOp::ReleaseMayInplace = {}; - OrtCustomOp::GetAliasMap = {}; - OrtCustomOp::ReleaseAliasMap = {}; - } - - const std::string op_name_; - const std::string execution_provider_; - - std::vector input_types_; - std::vector output_types_; - - ShapeInferFn shape_infer_fn_ = {}; - - int start_ver_ = 1; - int end_ver_ = MAX_CUSTOM_OP_END_VER; - - void* compute_fn_ = {}; - void* compute_fn_return_status_ = {}; -}; - -//////////////////////////// OrtLiteCustomFunc //////////////////////////////// -// The struct is to implement function-as-op. -// E.g. a function might be defined as: -// void Filter(const Ort::Custom::Tensor& floats_in, Ort::Custom::Tensor& floats_out) { ... } -// It could be registered this way: -// Ort::CustomOpDomain v2_domain{"v2"}; -// std::unique_ptr fil_op_ptr{Ort::Custom::CreateLiteCustomOp("Filter", "CPUExecutionProvider", Filter)}; -// v2_domain.Add(fil_op_ptr.get()); -// session_options.Add(v2_domain); -// For the complete example, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". -template -struct OrtLiteCustomFunc : public OrtLiteCustomOp { - using ComputeFn = void (*)(Args...); - using ComputeFnReturnStatus = Status (*)(Args...); - using MyType = OrtLiteCustomFunc; - - struct Kernel { - size_t num_input_{}; - size_t num_output_{}; - ComputeFn compute_fn_{}; - ComputeFnReturnStatus compute_fn_return_status_{}; - std::string ep_{}; - }; - - OrtLiteCustomFunc(const char* op_name, - const char* execution_provider, - ComputeFn compute_fn, - ShapeInferFn shape_infer_fn = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, shape_infer_fn, start_ver, end_ver) { - compute_fn_ = reinterpret_cast(compute_fn); - ParseArgs(input_types_, output_types_); - - OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { - auto kernel = reinterpret_cast(op_kernel); - std::vector args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - std::apply([kernel](Args const&... t_args) { kernel->compute_fn_(t_args...); }, t); - }; - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { - auto kernel = std::make_unique(); - auto me = static_cast(this_); - kernel->compute_fn_ = reinterpret_cast(me->compute_fn_); - Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); - Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); - auto self = static_cast(this_); - kernel->ep_ = self->execution_provider_; - return reinterpret_cast(kernel.release()); - }; - - OrtCustomOp::KernelDestroy = [](void* op_kernel) { - delete reinterpret_cast(op_kernel); - }; - - if (shape_infer_fn_) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp* op, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - auto shape_info_fn = static_cast(op)->shape_infer_fn_; - ShapeInferContext ctx(&GetApi(), ort_ctx); - return shape_info_fn(ctx); - }; - } - } - - OrtLiteCustomFunc(const char* op_name, - const char* execution_provider, - ComputeFnReturnStatus compute_fn_return_status, - ShapeInferFn shape_infer_fn = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, shape_infer_fn, start_ver, end_ver) { - compute_fn_return_status_ = reinterpret_cast(compute_fn_return_status); - ParseArgs(input_types_, output_types_); - - OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { - auto kernel = reinterpret_cast(op_kernel); - std::vector args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - return std::apply([kernel](Args const&... t_args) { Status status = kernel->compute_fn_return_status_(t_args...); return status.release(); }, t); - }; - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { - auto kernel = std::make_unique(); - auto me = static_cast(this_); - kernel->compute_fn_return_status_ = reinterpret_cast(me->compute_fn_return_status_); - Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); - Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); - auto self = static_cast(this_); - kernel->ep_ = self->execution_provider_; - return reinterpret_cast(kernel.release()); - }; - - OrtCustomOp::KernelDestroy = [](void* op_kernel) { - delete reinterpret_cast(op_kernel); - }; - - if (shape_infer_fn_) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp* op, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - auto shape_info_fn = static_cast(op)->shape_infer_fn_; - ShapeInferContext ctx(&GetApi(), ort_ctx); - return shape_info_fn(ctx); - }; - } - } -}; // struct OrtLiteCustomFunc - -/////////////////////////// OrtLiteCustomStruct /////////////////////////// -// The struct is to implement struct-as-op. -// E.g. a struct might be defined as: -// struct Merge { -// Merge(const OrtApi* ort_api, const OrtKernelInfo* info) {...} -// void Compute(const Ort::Custom::Tensor& strings_in, -// std::string_view string_in, -// Ort::Custom::Tensor* strings_out) {...} -// bool reverse_ = false; -// }; -// It could be registered this way: -// Ort::CustomOpDomain v2_domain{"v2"}; -// std::unique_ptr mrg_op_ptr{Ort::Custom::CreateLiteCustomOp("Merge", "CPUExecutionProvider")}; -// v2_domain.Add(mrg_op_ptr.get()); -// session_options.Add(v2_domain); -// For the complete example, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". -template -struct OrtLiteCustomStruct : public OrtLiteCustomOp { - template - using CustomComputeFn = void (CustomOp::*)(Args...); - - template - using CustomComputeFnReturnStatus = Status (CustomOp::*)(Args...); - - using MyType = OrtLiteCustomStruct; - - struct Kernel { - size_t num_input_{}; - size_t num_output_{}; - std::unique_ptr custom_op_; - std::string ep_{}; - }; - - OrtLiteCustomStruct(const char* op_name, - const char* execution_provider, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, {}, start_ver, end_ver) { - SetCompute(&CustomOp::Compute); - - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { - auto kernel = std::make_unique(); - Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); - Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); - kernel->custom_op_ = std::make_unique(ort_api, info); - auto self = static_cast(this_); - kernel->ep_ = self->execution_provider_; - return reinterpret_cast(kernel.release()); - }; - - OrtCustomOp::KernelDestroy = [](void* op_kernel) { - delete reinterpret_cast(op_kernel); - }; - - SetShapeInfer(0); - } - - template - void SetCompute(CustomComputeFn) { - ParseArgs(input_types_, output_types_); - OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { - auto kernel = reinterpret_cast(op_kernel); - ArgPtrs args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - std::apply([kernel](Args const&... t_args) { kernel->custom_op_->Compute(t_args...); }, t); - }; - } - - template - void SetCompute(CustomComputeFnReturnStatus) { - ParseArgs(input_types_, output_types_); - OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { - auto kernel = reinterpret_cast(op_kernel); - ArgPtrs args; - auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); - return std::apply([kernel](Args const&... t_args) { Status status = kernel->custom_op_->Compute(t_args...); return status.release(); }, t); - }; - } - - template - decltype(&C::InferOutputShape) SetShapeInfer(decltype(&C::InferOutputShape)) { - OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp*, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { - ShapeInferContext ctx(&GetApi(), ort_ctx); - return C::InferOutputShape(ctx); - }; - return {}; - } - - template - void SetShapeInfer(...) { - OrtCustomOp::InferOutputShapeFn = {}; - } -}; // struct OrtLiteCustomStruct - -/////////////////////////// CreateLiteCustomOp //////////////////////////// - -template -OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, - const char* execution_provider, - void (*custom_compute_fn)(Args...), - Status (*shape_infer_fn)(ShapeInferContext&) = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) { - using LiteOp = OrtLiteCustomFunc; - return std::make_unique(op_name, execution_provider, custom_compute_fn, shape_infer_fn, start_ver, end_ver).release(); -} - -template -OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, - const char* execution_provider, - Status (*custom_compute_fn_v2)(Args...), - Status (*shape_infer_fn)(ShapeInferContext&) = {}, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) { - using LiteOp = OrtLiteCustomFunc; - return std::make_unique(op_name, execution_provider, custom_compute_fn_v2, shape_infer_fn, start_ver, end_ver).release(); -} - -template -OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, - const char* execution_provider, - int start_ver = 1, - int end_ver = MAX_CUSTOM_OP_END_VER) { - using LiteOp = OrtLiteCustomStruct; - return std::make_unique(op_name, execution_provider, start_ver, end_ver).release(); -} - -} // namespace Custom -} // namespace Ort diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_run_options_config_keys.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_run_options_config_keys.h deleted file mode 100644 index c80b8c0..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_run_options_config_keys.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -/* - * This file defines RunOptions Config Keys and format of the Config Values. - * - * The Naming Convention for a RunOptions Config Key, - * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" - * Such as "ep.cuda.use_arena" - * The Config Key cannot be empty - * The maximum length of the Config Key is 128 - * - * The string format of a RunOptions Config Value is defined individually for each Config. - * The maximum length of the Config Value is 1024 - */ - -// Key for enabling shrinkages of user listed device memory arenas. -// Expects a list of semi-colon separated key value pairs separated by colon in the following format: -// "device_0:device_id_0;device_1:device_id_1" -// No white-spaces allowed in the provided list string. -// Currently, the only supported devices are : "cpu", "gpu" (case sensitive). -// If "cpu" is included in the list, DisableCpuMemArena() API must not be called (i.e.) arena for cpu should be enabled. -// Example usage: "cpu:0;gpu:0" (or) "gpu:0" -// By default, the value for this key is empty (i.e.) no memory arenas are shrunk -static const char* const kOrtRunOptionsConfigEnableMemoryArenaShrinkage = "memory.enable_memory_arena_shrinkage"; - -// Set to '1' to not synchronize execution providers with CPU at the end of session run. -// Per default it will be set to '0' -// Taking CUDA EP as an example, it omit triggering cudaStreamSynchronize on the compute stream. -static const char* const kOrtRunOptionsConfigDisableSynchronizeExecutionProviders = "disable_synchronize_execution_providers"; - -// Set HTP performance mode for QNN HTP backend before session run. -// options for HTP performance mode: "burst", "balanced", "default", "high_performance", -// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", -// "sustained_high_performance". Default to "default". -static const char* const kOrtRunOptionsConfigQnnPerfMode = "qnn.htp_perf_mode"; - -// Set HTP performance mode for QNN HTP backend post session run. -static const char* const kOrtRunOptionsConfigQnnPerfModePostRun = "qnn.htp_perf_mode_post_run"; - -// Set RPC control latency for QNN HTP backend -static const char* const kOrtRunOptionsConfigQnnRpcControlLatency = "qnn.rpc_control_latency"; - -// Set graph annotation id for CUDA EP. Use with enable_cuda_graph=true. -// The value should be an integer. If the value is not set, the default value is 0 and -// ORT session only captures one cuda graph before another capture is requested. -// If the value is set to -1, cuda graph capture/replay is disabled in that run. -// User are not expected to set the value to 0 as it is reserved for internal use. -static const char* const kOrtRunOptionsConfigCudaGraphAnnotation = "gpu_graph_id"; diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_session_options_config_keys.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_session_options_config_keys.h deleted file mode 100644 index bb5e034..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_session_options_config_keys.h +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -/* - * This file defines SessionOptions Config Keys and format of the Config Values. - * - * The Naming Convention for a SessionOptions Config Key, - * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" - * Such as "ep.cuda.use_arena" - * The Config Key cannot be empty - * The maximum length of the Config Key is 128 - * - * The string format of a SessionOptions Config Value is defined individually for each Config. - * The maximum length of the Config Value is 1024 - */ - -// Key for disable PrePacking, -// If the config value is set to "1" then the prepacking is disabled, otherwise prepacking is enabled (default value) -static const char* const kOrtSessionOptionsConfigDisablePrepacking = "session.disable_prepacking"; - -// A value of "1" means allocators registered in the env will be used. "0" means the allocators created in the session -// will be used. Use this to override the usage of env allocators on a per session level. -static const char* const kOrtSessionOptionsConfigUseEnvAllocators = "session.use_env_allocators"; - -// Set to 'ORT' (case sensitive) to load an ORT format model. -// If unset, model type will default to ONNX unless inferred from filename ('.ort' == ORT format) or bytes to be ORT -static const char* const kOrtSessionOptionsConfigLoadModelFormat = "session.load_model_format"; - -// Set to 'ORT' (case sensitive) to save optimized model in ORT format when SessionOptions.optimized_model_path is set. -// If unset, format will default to ONNX unless optimized_model_filepath ends in '.ort'. -static const char* const kOrtSessionOptionsConfigSaveModelFormat = "session.save_model_format"; - -// If a value is "1", flush-to-zero and denormal-as-zero are applied. The default is "0". -// When multiple sessions are created, a main thread doesn't override changes from succeeding session options, -// but threads in session thread pools follow option changes. -// When ORT runs with OpenMP, the same rule is applied, i.e. the first session option to flush-to-zero and -// denormal-as-zero is only applied to global OpenMP thread pool, which doesn't support per-session thread pool. -// Note that an alternative way not using this option at runtime is to train and export a model without denormals -// and that's recommended because turning this option on may hurt model accuracy. -static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.set_denormal_as_zero"; - -// It controls to run quantization model in QDQ (QuantizelinearDeQuantizelinear) format or not. -// "0": enable. ORT does fusion logic for QDQ format. -// "1": disable. ORT doesn't do fusion logic for QDQ format. -// Its default value is "0" unless the DirectML execution provider is registered, in which case it defaults to "1". -static const char* const kOrtSessionOptionsDisableQuantQDQ = "session.disable_quant_qdq"; - -// It controls whether to enable Double QDQ remover and Identical Children Consolidation -// "0": not to disable. ORT does remove the middle 2 Nodes from a Q->(QD->Q)->QD pairs -// "1": disable. ORT doesn't remove the middle 2 Nodes from a Q->(QD->Q)->QD pairs -// Its default value is "0" -static const char* const kOrtSessionOptionsDisableDoubleQDQRemover = "session.disable_double_qdq_remover"; - -// If set to "1", enables the removal of QuantizeLinear/DequantizeLinear node pairs once all QDQ handling has been -// completed. e.g. If after all QDQ handling has completed and we have -> FloatOp -> Q -> DQ -> FloatOp -> the -// Q -> DQ could potentially be removed. This will provide a performance benefit by avoiding going from float to -// 8-bit and back to float, but could impact accuracy. The impact on accuracy will be model specific and depend on -// other factors like whether the model was created using Quantization Aware Training or Post Training Quantization. -// As such, it's best to test to determine if enabling this works well for your scenario. -// The default value is "0" -// Available since version 1.11. -static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enable_quant_qdq_cleanup"; - -// Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0". -// GeluApproximation has side effects which may change the inference results. It is disabled by default due to this. -static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation"; - -// This setting controls whether to enable AheadOfTime function inlining. -// AOT function inlining examines the graph and attempts to inline as many locally defined functions in the model -// as possible with the help of enabled execution providers. -// This can reduce the number of function calls and improve performance because it is done before -// Level1 optimizers and constant folding. However, under some circumstances, when the EPs are not available, -// one can disable the AOT inlining, produce an optimized model and postpone AOT until run time. -// "0": enable; "1": disable. -// Its default value is "0". -static const char* const kOrtSessionOptionsDisableAheadOfTimeFunctionInlining = "session.disable_aot_function_inlining"; - -#ifdef ENABLE_TRAINING -// Specifies a list of op types for memory footprint reduction. -// The value should be a ","-delimited list of pair of -// . -// For example, "Gelu+Cast+:1:0,Dropout+:1:1". -// A valid "subgraph string" should be one subgraph representation output by ORT graph transformations. -// "optimization strategy" currently has valid values: 0 - disabled, 1 - recompute. -// "number of subgraph to apply" is used to control how many subgraphs to apply optimization, to avoid "oversaving" -// the memory. -static const char* const kOrtSessionOptionsMemoryOptimizerEnabler = "optimization.memory_optimizer_config"; - -// Specifies the config for detecting subgraphs for memory footprint reduction. -// The value should be a string contains int separated using commas. The default value is "0:0". -static const char* const kOrtSessionOptionsMemoryOptimizerProbeConfig = "optimization.enable_memory_probe_recompute_config"; -#endif - -// This setting if set should contain a comma separated list of optimizers names that should be disabled. -// Optimizers may take time to execute and affect model loading time. If you feel that a specific optimizer -// does not provider runtime benefits, but affects your model loading time you may disable it using this config -// entry. This option is not enabled in ORT_MINIMAL_BUILD build. -// A list of optimizes is available in onnxruntime/core/optimizer/graph_transformer_utils.cc -// -// Default is an empty string which means no optimizers are disabled. -static const char* const kOrtSessionOptionsDisableSpecifiedOptimizers = "optimization.disable_specified_optimizers"; - -// Enable or disable using device allocator for allocating initialized tensor memory. "1": enable; "0": disable. The default is "0". -// Using device allocators means the memory allocation is made using malloc/new. -static const char* const kOrtSessionOptionsUseDeviceAllocatorForInitializers = "session.use_device_allocator_for_initializers"; - -// Configure whether to allow the inter_op/intra_op threads spinning a number of times before blocking -// "0": thread will block if found no job to run -// "1": default, thread will spin a number of times before blocking -static const char* const kOrtSessionOptionsConfigAllowInterOpSpinning = "session.inter_op.allow_spinning"; -static const char* const kOrtSessionOptionsConfigAllowIntraOpSpinning = "session.intra_op.allow_spinning"; - -// Key for using model bytes directly for ORT format -// If a session is created using an input byte array contains the ORT format model data, -// By default we will copy the model bytes at the time of session creation to ensure the model bytes -// buffer is valid. -// Setting this option to "1" will disable copy the model bytes, and use the model bytes directly. The caller -// has to guarantee that the model bytes are valid until the ORT session using the model bytes is destroyed. -static const char* const kOrtSessionOptionsConfigUseORTModelBytesDirectly = "session.use_ort_model_bytes_directly"; - -/// -/// Key for using the ORT format model flatbuffer bytes directly for initializers. -/// This avoids copying the bytes and reduces peak memory usage during model loading and initialization. -/// Requires `session.use_ort_model_bytes_directly` to be true. -/// If set, the flatbuffer bytes provided when creating the InferenceSession MUST remain valid for the entire -/// duration of the InferenceSession. -/// -static const char* const kOrtSessionOptionsConfigUseORTModelBytesForInitializers = - "session.use_ort_model_bytes_for_initializers"; - -// This should only be specified when exporting an ORT format model for use on a different platform. -// If the ORT format model will be used on ARM platforms set to "1". For other platforms set to "0" -// Available since version 1.11. -static const char* const kOrtSessionOptionsQDQIsInt8Allowed = "session.qdqisint8allowed"; - -// x64 SSE4.1/AVX2/AVX512(with no VNNI) has overflow problem with quantizied matrix multiplication with U8S8. -// To avoid this we need to use slower U8U8 matrix multiplication instead. This option, if -// turned on, use slower U8U8 matrix multiplications. Only effective with AVX2 or AVX512 -// platforms. -static const char* const kOrtSessionOptionsAvx2PrecisionMode = "session.x64quantprecision"; - -// Specifies how minimal build graph optimizations are handled in a full build. -// These optimizations are at the extended level or higher. -// Possible values and their effects are: -// "save": Save runtime optimizations when saving an ORT format model. -// "apply": Only apply optimizations available in a minimal build. -// ""/: Apply optimizations available in a full build. -// Available since version 1.11. -static const char* const kOrtSessionOptionsConfigMinimalBuildOptimizations = - "optimization.minimal_build_optimizations"; - -// Note: The options specific to an EP should be specified prior to appending that EP to the session options object in -// order for them to take effect. - -// Specifies a list of stop op types. Nodes of a type in the stop op types and nodes downstream from them will not be -// run by the NNAPI EP. -// The value should be a ","-delimited list of op types. For example, "Add,Sub". -// If not specified, the default set of stop ops is used. To specify an empty stop ops types list and disable stop op -// exclusion, set the value to "". -static const char* const kOrtSessionOptionsConfigNnapiEpPartitioningStopOps = "ep.nnapi.partitioning_stop_ops"; - -// Enabling dynamic block-sizing for multithreading. -// With a positive value, thread pool will split a task of N iterations to blocks of size starting from: -// N / (num_of_threads * dynamic_block_base) -// As execution progresses, the size will decrease according to the diminishing residual of N, -// meaning the task will be distributed in smaller granularity for better parallelism. -// For some models, it helps to reduce the variance of E2E inference latency and boost performance. -// The feature will not function by default, specify any positive integer, e.g. "4", to enable it. -// Available since version 1.11. -static const char* const kOrtSessionOptionsConfigDynamicBlockBase = "session.dynamic_block_base"; - -// This option allows to decrease CPU usage between infrequent -// requests and forces any TP threads spinning stop immediately when the last of -// concurrent Run() call returns. -// Spinning is restarted on the next Run() call. -// Applies only to internal thread-pools -static const char* const kOrtSessionOptionsConfigForceSpinningStop = "session.force_spinning_stop"; - -// "1": all inconsistencies encountered during shape and type inference -// will result in failures. -// "0": in some cases warnings will be logged but processing will continue. The default. -// May be useful to expose bugs in models. -static const char* const kOrtSessionOptionsConfigStrictShapeTypeInference = "session.strict_shape_type_inference"; - -// "1": every model using a more recent opset than the latest released one will fail -// "0": the model may or may not work if onnxruntime cannot find an implementation, this option -// is used for development purpose. -static const char* const kOrtSessionOptionsConfigStrictAllowReleasedOpsetsOnly = "session.allow_released_opsets_only"; - -// The file saves configuration for partitioning node among logic streams -static const char* const kNodePartitionConfigFile = "session.node_partition_config_file"; - -// This Option allows setting affinities for intra op threads. -// Affinity string follows format: -// logical_processor_id,logical_processor_id;logical_processor_id,logical_processor_id -// Semicolon isolates configurations among threads, while comma split processors where ith thread expected to attach to. -// e.g.1,2,3;4,5 -// specifies affinities for two threads, with the 1st thread attach to the 1st, 2nd, and 3rd processor, and 2nd thread to the 4th and 5th. -// To ease the configuration, an "interval" is also allowed: -// e.g. 1-8;8-16;17-24 -// orders that the 1st thread runs on first eight processors, 2nd thread runs on next eight processors, and so forth. -// Note: -// 1. Once set, the number of thread affinities must equal to intra_op_num_threads - 1, since ort does not set affinity on the main thread which -// is started and managed by the calling app; -// 2. For windows, ort will infer the group id from a logical processor id, for example, assuming there are two groups with each has 64 logical processors, -// an id of 64 will be inferred as the last processor of the 1st group, while 65 will be interpreted as the 1st processor of the second group. -// Hence 64-65 is an invalid configuration, because a windows thread cannot be attached to processors across group boundary. -static const char* const kOrtSessionOptionsConfigIntraOpThreadAffinities = "session.intra_op_thread_affinities"; - -// This option will dump out the model to assist debugging any issues with layout transformation, -// and is primarily intended for developer usage. It is only relevant if an execution provider that requests -// NHWC layout is enabled such as NNAPI, XNNPACK or QNN. -// -// Default is off. Set to "1" to enable. -// -// If modified by layout transformation the model will be dumped after these steps: -// 1) insertion of the layout transformation Transpose nodes -// 2) after those are optimized using the transpose optimizer, -// 3) after the L1 transformers are applied to the updated graph. -// The model will be saved to filename post_layout_transform_step_.onnx. -static const char* const kDebugLayoutTransformation = "session.debug_layout_transformation"; - -// Graph nodes that are not supported by the execution providers (EPs) explicitly added to the session are -// assigned (i.e., "fallback") to the CPU EP by default. -// -// This option allows the user to disable the fallback of unsupported graph nodes to the CPU EP. -// If this option is set to "1", session creation will fail if the execution providers other than the CPU EP cannot -// fully support all of the nodes in the graph. -// -// It is invalid to set this option and explicitly add the CPU EP to the session. In this case, session creation -// will also fail with an error. -// -// Option values: -// - "0": CPU EP fallback is not disabled. [DEFAULT] -// - "1": CPU EP fallback is disabled. -static const char* const kOrtSessionOptionsDisableCPUEPFallback = "session.disable_cpu_ep_fallback"; - -// Use this config when serializing a large model after optimization to specify an external initializers file -static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersFileName = - "session.optimized_model_external_initializers_file_name"; - -// Use this config to control the minimum size of the initializer when externalizing it during serialization -static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersMinSizeInBytes = - "session.optimized_model_external_initializers_min_size_in_bytes"; - -// Enable EP context feature to dump the partitioned graph which includes the EP context into Onnx file. -// The dumped Onnx model with EP context can be used for future inference to avoid the EP graph partitioning/compile overhead. -// "0": disable. (default) -// "1": enable. -static const char* const kOrtSessionOptionEpContextEnable = "ep.context_enable"; - -// Specify the file path for the Onnx model which has EP context. -// Default to original_file_name_ctx.onnx if not specified -static const char* const kOrtSessionOptionEpContextFilePath = "ep.context_file_path"; - -// Flag to specify whether to dump the EP context into the Onnx model. -// "0": dump the EP context into separate file, keep the file name in the Onnx model. -// "1": dump the EP context into the Onnx model. (default). -static const char* const kOrtSessionOptionEpContextEmbedMode = "ep.context_embed_mode"; - -// Gemm fastmath mode provides fp32 gemm acceleration with bfloat16 based matmul. -// Option values: -// - "0": Gemm FastMath mode is not enabled. [DEFAULT] -// - "1": Gemm FastMath mode is enabled. -static const char* const kOrtSessionOptionsMlasGemmFastMathArm64Bfloat16 = "mlas.enable_gemm_fastmath_arm64_bfloat16"; diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_c_api.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_c_api.h deleted file mode 100644 index ed6d151..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_c_api.h +++ /dev/null @@ -1,731 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// This file contains the training c apis. - -#pragma once -#include -#include "onnxruntime_c_api.h" - -/** \page training_c_cpp_api Training C & C++ APIs - * - * Training C and C++ APIs are an extension of the \ref c_cpp_api "onnxruntime core C and C++ APIs" and should be used in conjunction with them. - * - * In order to train a model with onnxruntime, the following training artifacts must be generated: - * - The training onnx model - * - The checkpoint file - * - The optimizer onnx model - * - The eval onnx model model (optional) - * - * These training artifacts can be generated as part of an offline step using the python [utilities](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md) made available in the `onnxruntime-training` python package. - * - * After these artifacts have been generated, the C and C++ utilities listed in this documentation can be leveraged to perform training. - * - * If any problem is encountered, please create an [issue](https://github.com/microsoft/onnxruntime/issues/new) with your scenario and requirements, and we will be sure to respond and follow up on the request. - * - *

Training C API

- * - * ::OrtTrainingApi - Training C API functions. - * - * This C structure contains functions that enable users to perform training with onnxruntime. - * - * _Sample Code_: - * - * ```c - * #include - * - * OrtApi* g_ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION); - * OrtTrainingApi* g_ort_training_api = g_ort_api->GetTrainingApi(ORT_API_VERSION); - * - * OrtEnv* env = NULL; - * g_ort_api->CreateEnv(logging_level, logid, &env); - * OrtSessionOptions* session_options = NULL; - * g_ort_api->CreateSessionOptions(&session_options); - * - * OrtCheckpointState* state = NULL; - * g_ort_training_api->LoadCheckpoint(path_to_checkpoint, &state); - * - * OrtTrainingSession* training_session = NULL; - * g_ort_training_api->CreateTrainingSession(env, session_options, training_model_path, - * state, eval_model_path, optimizer_model_path, - * &training_session); - * // Training loop - * { - * g_ort_training_api->TrainStep(...); - * g_ort_training_api->OptimizerStep(...); - * g_ort_training_api->LazyResetGrad(...); - * } - * - * g_ort_training_api->ExportModelForInferencing(training_session, inference_model_path, ...); - * g_ort_training_api->SaveCheckpoint(state, path_to_checkpoint, false); - * - * g_ort_training_api->ReleaseTrainingSession(training_session); - * g_ort_training_api->ReleaseCheckpointState(state); - * ``` - * - * > **Note** - * > The ::OrtCheckpointState contains the entire training state that the ::OrtTrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::OrtCheckpointState instance must outlive the lifetime of the ::OrtTrainingSession instance. - * - *

Training C++ API

- * - * @ref TrainingCpp - Training C++ API classes and functions. - * - * These C++ classes and functions enable users to perform training with onnxruntime. - * - * _Sample Code_: - * - * ```cc - * #include - * - * Ort::Env env; - * Ort::SessionOptions session_options; - * - * auto state = Ort::CheckpointState::LoadCheckpoint(path_to_checkpoint); - * auto training_session = Ort::TrainingSession(env, session_options, state, training_model_path, - * eval_model_path, optimizer_model_path); - * - * // Training Loop - * { - * training_session.TrainStep(...); - * training_session.OptimizerStep(...); - * training_session.LazyResetGrad(...); - * } - * - * training_session->ExportModelForInferencing(inference_model_path, ...); - * Ort::CheckpointState::SaveCheckpoint(state, path_to_checkpoint, false); - * ``` - * > **Note** - * > The ::Ort::CheckpointState contains the entire training state that the ::Ort::TrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::Ort::CheckpointState instance must outlive the lifetime of the ::Ort::TrainingSession instance. - */ - -/** @defgroup TrainingC Ort Training C API - * @{ - */ -ORT_RUNTIME_CLASS(TrainingSession); // Type that enables performing training for the given user models. -ORT_RUNTIME_CLASS(CheckpointState); // Type that holds the training states for the training session. - -/** \brief Type of property to be added to or returned from the ::OrtCheckpointState. - */ -typedef enum OrtPropertyType { - OrtIntProperty = 0, - OrtFloatProperty = 1, - OrtStringProperty = 2, -} OrtPropertyType; - -/** \brief The Training C API that holds onnxruntime training function pointers - * - * All the Training C API functions are defined inside this structure as pointers to functions. - * Call OrtApi::GetTrainingApi to get a pointer to this struct. - * - * \nosubgrouping - */ -struct OrtTrainingApi { - /// \name Accessing The Training Session State - /// @{ - - /** \brief Load a checkpoint state from a file on disk into checkpoint_state. - * - * This function will parse a checkpoint file, pull relevant data and load the training - * state into the checkpoint_state. This checkpoint state can then be used to create the - * training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training - * session will resume training from the given checkpoint state. - * \note Note that the training session created with a checkpoint state uses this state to store the entire - * training state (including model parameters, its gradients, the optimizer states and the properties). - * As a result, it is required that the checkpoint state outlive the lifetime of the training session. - * \note Note that the checkpoint file can be either the complete checkpoint or the nominal checkpoint. - * - * \param[in] checkpoint_path Path to the checkpoint file - * \param[out] checkpoint_state Checkpoint state that contains the states of the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(LoadCheckpoint, _In_ const ORTCHAR_T* checkpoint_path, - _Outptr_ OrtCheckpointState** checkpoint_state); - - /** \brief Save the given state to a checkpoint file on disk. - * - * This function serializes the provided checkpoint state to a file on disk. - * This checkpoint can later be loaded by invoking OrtTrainingApi::LoadCheckpoint to resume - * training from this snapshot of the state. - * - * \param[in] checkpoint_state The checkpoint state to save. - * \param[in] checkpoint_path Path to the checkpoint file. - * \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SaveCheckpoint, _In_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* checkpoint_path, - const bool include_optimizer_state); - - /// @} - - /// \name Implementing The Training Loop - /// @{ - /** \brief Create a training session that can be used to begin or resume training. - * - * This function creates a training session based on the env and session options provided that can - * begin or resume training from a given checkpoint state for the given onnx models. - * The checkpoint state represents the parameters of the training session which will be moved - * to the device specified by the user through the session options (if necessary). - * The training session requires four training artifacts - * - The training onnx model - * - The evaluation onnx model (optional) - * - The optimizer onnx model - * - The checkpoint file - * - * These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md). - * - * \param[in] env Environment to be used for the training session. - * \param[in] options Session options that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_path Model to be used to perform training. - * \param[in] eval_model_path Model to be used to perform evaluation. - * \param[in] optimizer_model_path Model to be used to perform gradient descent. - * \param[out] out Created training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(CreateTrainingSession, _In_ const OrtEnv* env, _In_ const OrtSessionOptions* options, - _Inout_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* train_model_path, - _In_ const ORTCHAR_T* eval_model_path, _In_ const ORTCHAR_T* optimizer_model_path, - _Outptr_result_maybenull_ OrtTrainingSession** out); - - /** \brief Create a training session that can be used to begin or resume training. - * This api provides a way to load all the training artifacts from buffers instead of files. - * - * \param[in] env Environment to be used for the training session. - * \param[in] options Session options that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_data Buffer containing the model data to be used to perform training - * \param[in] train_data_length Length of the buffer containing train_model_data - * \param[in] eval_model_data Buffer containing the model data to be used to perform evaluation - * \param[in] eval_data_length Length of the buffer containing eval_model_data - * \param[in] optim_model_data Buffer containing the model data to be used to perform weight update - * \param[in] optim_data_length Length of the buffer containing optim_model_data - * \param[out] out Created training session. - * - */ - ORT_API2_STATUS(CreateTrainingSessionFromBuffer, _In_ const OrtEnv* env, - _In_ const OrtSessionOptions* options, _Inout_ OrtCheckpointState* checkpoint_state, - _In_ const void* train_model_data, size_t train_data_length, - _In_ const void* eval_model_data, size_t eval_data_length, - _In_ const void* optim_model_data, size_t optim_data_length, - _Outptr_result_maybenull_ OrtTrainingSession** out); - - /// @} - - /// \name Model IO Information - /// @{ - - /** \brief Retrieves the number of user outputs in the training model. - * - * This function returns the number of outputs of the training model so that the user can - * allocate space for the number of outputs when OrtTrainingApi::TrainStep is invoked. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user outputs in the training model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the number of user outputs in the eval model. - * - * This function returns the number of outputs of the eval model so that the user can - * allocate space for the number of outputs when OrtTrainingApi::EvalStep is invoked. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user outputs in the eval model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the names of user outputs in the training model. - * - * This function returns the names of outputs of the training model that can be associated with the OrtValue(s) - * returned by the OrtTrainingApi::TrainStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index Index of the output name requested. - * \param[in] allocator Allocator to use to allocate the memory for the name. - * \param[out] output Name of the training model output at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output); - - /** \brief Retrieves the names of user outputs in the eval model. - * - * This function returns the names of outputs of the eval model that can be associated with the OrtValue(s) returned - * by the OrtTrainingApi::EvalStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index Index of the output name requested. - * \param[in] allocator Allocator to use to allocate the memory for the name. - * \param[out] output Name of the eval model output at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output); - - /// @} - - /// \name Implementing The Training Loop - /// @{ - - /** \brief Reset the gradients of all trainable parameters to zero lazily. - * - * This function sets the internal state of the training session such that the gradients of the trainable - * parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are - * computed on the next invocation of the next OrtTrainingApi::TrainStep. - * - * \param[in] session The `this` pointer to the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(LazyResetGrad, _Inout_ OrtTrainingSession* session); - - /** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs - * - * This function performs a training step that computes the outputs of the training model and the gradients - * of the trainable parameters for the given inputs. The train step is performed based on the training model - * that was provided to the training session. - * The OrtTrainingApi::TrainStep is equivalent of running forward propagation and backward propagation in a single - * step. - * The gradients computed are stored inside the training session state so they can be later consumed - * by the OrtTrainingApi::OptimizerStep function. - * The gradients can be lazily reset by invoking the OrtTrainingApi::LazyResetGrad function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] run_options Run options for this training step. - * \param[in] inputs_len Number of user inputs to the training model. - * \param[in] inputs The user inputs to the training model. - * \param[in] outputs_len Number of user outputs expected from this training step. - * \param[out] outputs User outputs computed by train step. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainStep, _Inout_ OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options, - _In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs, - _In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs); - - /** \brief Computes the outputs for the eval model for the given inputs - * - * This function performs an eval step that computes the outputs of the eval model for the given inputs. - * The eval step is performed based on the eval model that was provided to the training session. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] run_options Run options for this eval step. - * \param[in] inputs_len Number of user inputs to the eval model. - * \param[in] inputs The user inputs to the eval model. - * \param[in] outputs_len Number of user outputs expected from this eval step. - * \param[out] outputs User outputs computed by eval step. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(EvalStep, _In_ const OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options, - _In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs, - _In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs); - - /** \brief Sets the learning rate for this training session. - * - * This function allows users to set the learning rate for the training session. The current - * learning rate is maintained by the training session and can be overwritten by invoking - * this function with the desired learning rate. This function should not be used when a valid - * learning rate scheduler is registered. It should be used either to set the learning rate - * derived from a custom learning rate scheduler or to set a constant learning rate to be used - * throughout the training session. - * \note Please note that this function does not set the initial learning rate that may be needed - * by the predefined learning rate schedulers. To set the initial learning rate for learning - * rate schedulers, please look at the function OrtTrainingApi::RegisterLinearLRScheduler. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] learning_rate Desired learning rate to be set. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SetLearningRate, _Inout_ OrtTrainingSession* sess, _In_ float learning_rate); - - /** \brief Gets the current learning rate for this training session. - * - * This function allows users to get the learning rate for the training session. The current - * learning rate is maintained by the training session, and users can query it for the purpose - * of implementing their own learning rate schedulers. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] learning_rate Learning rate currently in use by the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetLearningRate, _Inout_ OrtTrainingSession* sess, _Out_ float* learning_rate); - - /** \brief Performs the weight updates for the trainable parameters using the optimizer model. - * - * This function performs the weight update step that updates the trainable parameters such that they - * take a step in the direction of their gradients (gradient descent). The optimizer step is performed - * based on the optimizer model that was provided to the training session. - * The updated parameters are stored inside the training state so that they can be used by the next - * OrtTrainingApi::TrainStep function call. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] run_options Run options for this optimizer step. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(OptimizerStep, _Inout_ OrtTrainingSession* sess, - _In_opt_ const OrtRunOptions* run_options); - - /** \brief Registers a linear learning rate scheduler for the training session. - * - * Register a linear learning rate scheduler that decays the learning rate by linearly updated - * multiplicative factor from the initial learning rate set on the training session to 0. The decay - * is performed after the initial warm up phase where the learning rate is linearly incremented - * from 0 to the initial learning rate provided. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] warmup_step_count Warmup steps for LR warmup. - * \param[in] total_step_count Total step count. - * \param[in] initial_lr The initial learning rate to be used by the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(RegisterLinearLRScheduler, _Inout_ OrtTrainingSession* sess, _In_ const int64_t warmup_step_count, - _In_ const int64_t total_step_count, _In_ const float initial_lr); - - /** \brief Update the learning rate based on the registered learing rate scheduler. - * - * Takes a scheduler step that updates the learning rate that is being used by the training session. - * This function should typically be called before invoking the optimizer step for each round, - * or as determined necessary to update the learning rate being used by the training session. - * \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this - * function. - * - * \param[in] sess The `this` pointer to the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SchedulerStep, _Inout_ OrtTrainingSession* sess); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - /** \brief Retrieves the size of all the parameters. - * - * Calculates the total number of primitive (datatype of the parameters) elements of all the parameters in the - * training state. - * When trainable_only argument is true, the size is calculated for trainable params only. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Size of all parameter elements. - * \param[in] trainable_only Whether to skip non-trainable parameters - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetParametersSize, _Inout_ OrtTrainingSession* sess, _Out_ size_t* out, bool trainable_only); - - /** \brief Copy all parameters to a contiguous buffer held by the argument parameters_buffer - * - * The parameters_buffer has to be of the size given by GetParametersSize api call, - * with matching setting for the argument trainable_only. All the target parameters must be of the same - * datatype. The OrtValue must be pre-allocated onto - * the desired device. This is a complementary function to OrtTrainingApi::CopyBufferToParameters. - * Parameter ordering is preserved. - * User is responsible for allocating and freeing the resources used by the parameters_buffer. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] trainable_only Whether to skip non-trainable parameters - * \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy onto. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(CopyParametersToBuffer, _Inout_ OrtTrainingSession* sess, - _Inout_ OrtValue* parameters_buffer, bool trainable_only); - - /** \brief Copy parameter values from the given contiguous buffer held by parameters_buffer to the training state - * - * The parameters_buffer argument has to be of the size given by OrtTrainingApi::GetParametersSize api call, - * with matching setting for trainable_only argument. All the target parameters must be of the same - * datatype. This is a complementary function to OrtTrainingApi::CopyParametersToBuffer - * and can be used to load updated buffer values onto the training state. - * Parameter ordering is preserved. - * User is responsible for allocating and freeing the resources used by the parameters_buffer. - * In case the training session was created with a nominal checkpoint, invoking this function is required - * to load the updated parameters onto the checkpoint to complete it. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] trainable_only Whether to skip non-trainable parameters - * \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy from. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(CopyBufferToParameters, _Inout_ OrtTrainingSession* sess, - _Inout_ OrtValue* parameters_buffer, bool trainable_only); - - /// @} - - /// \name Release Training Resources - /// @{ - - /** \brief Frees up the memory used up by the training session. - * - * This function frees up any memory that was allocated in the training session. The training - * session can no longer be used after this call. - * - */ - ORT_CLASS_RELEASE(TrainingSession); - - /** \brief Frees up the memory used up by the checkpoint state. - * - * This function frees up any memory that was allocated in the checkpoint state. The checkpoint - * state can no longer be used after this call. - * \note Note that the checkpoint state must be released only after the training session has been released. - * - */ - ORT_CLASS_RELEASE(CheckpointState); - - /// @} - - /// \name Prepare For Inferencing - /// @{ - /** \brief Export a model that can be used for inferencing. - * - * If the training session was provided with an eval model, the training session can generate - * an inference model if it knows the inference graph outputs. The input inference graph outputs - * are used to prune the eval model so that the inference model's outputs align with the provided outputs. - * The exported model is saved at the path provided and can be used for inferencing with InferenceSession. - * \note Note that the function re-loads the eval model from the path provided to OrtTrainingApi::CreateTrainingSession - * and expects that this path still be valid. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] inference_model_path Path where the inference model should be serialized to. - * \param[in] graph_outputs_len Size of the graph output names array. - * \param[in] graph_output_names Names of the outputs that are needed in the inference model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(ExportModelForInferencing, _Inout_ OrtTrainingSession* sess, - _In_ const ORTCHAR_T* inference_model_path, size_t graph_outputs_len, - _In_reads_(graph_outputs_len) const char* const* graph_output_names); - - /// @} - - /// \name Training Utilities - /// @{ - /** \brief Sets the seed used for random number generation in Onnxruntime. - * - * Use this function to generate reproducible results. It should be noted that completely reproducible - * results are not guaranteed. - * - * \param[in] seed The seed to be set. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(SetSeed, _In_ const int64_t seed); - - /// @} - - /// \name Model IO Information - /// @{ - /** \brief Retrieves the number of user inputs in the training model. - * - * This function returns the number of inputs of the training model so that the user can accordingly - * allocate the OrtValue(s) provided to the OrtTrainingApi::TrainStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user inputs in the training model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the number of user inputs in the eval model. - * - * This function returns the number of inputs of the eval model so that the user can accordingly - * allocate the OrtValue(s) provided to the OrtTrainingApi::EvalStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[out] out Number of user inputs in the eval model. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out); - - /** \brief Retrieves the name of the user input at given index in the training model. - * - * This function returns the names of inputs of the training model that can be associated with the - * OrtValue(s) provided to the OrtTrainingApi::TrainStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index The index of the training model input name requested. - * \param[in] allocator The allocator to use to allocate the memory for the requested name. - * \param[out] output Name of the user input for the training model at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetTrainingModelInputName, _In_ const OrtTrainingSession* sess, size_t index, - _In_ OrtAllocator* allocator, _Outptr_ char** output); - - /** \brief Retrieves the name of the user input at given index in the eval model. - * - * This function returns the names of inputs of the eval model that can be associated with the OrtValue(s) provided - * to the OrtTrainingApi::EvalStep function. - * - * \param[in] sess The `this` pointer to the training session. - * \param[in] index The index of the eval model input name requested. - * \param[in] allocator The allocator to use to allocate the memory for the requested name. - * \param[out] output Name of the user input for the eval model at the given index. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(TrainingSessionGetEvalModelInputName, _In_ const OrtTrainingSession* sess, size_t index, - _In_ OrtAllocator* allocator, _Outptr_ char** output); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Adds or updates the given property to/in the checkpoint state. - * - * Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint - * state by the user by calling this function with the corresponding property name and value. - * The given property name must be unique to be able to successfully add the property. - * - * \param[in] checkpoint_state The checkpoint state which should hold the property. - * \param[in] property_name Name of the property being added or updated. - * \param[in] property_type Type of the property associated with the given name. - * \param[in] property_value Property value associated with the given name. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(AddProperty, _Inout_ OrtCheckpointState* checkpoint_state, - _In_ const char* property_name, _In_ enum OrtPropertyType property_type, - _In_ void* property_value); - - /** \brief Gets the property value associated with the given name from the checkpoint state. - * - * Gets the property value from an existing entry in the checkpoint state. The property must - * exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] checkpoint_state The checkpoint state that is currently holding the property. - * \param[in] property_name Name of the property being retrieved. - * \param[in] allocator Allocator used to allocate the memory for the property_value. - * \param[out] property_type Type of the property associated with the given name. - * \param[out] property_value Property value associated with the given name. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetProperty, _In_ const OrtCheckpointState* checkpoint_state, - _In_ const char* property_name, _Inout_ OrtAllocator* allocator, - _Out_ enum OrtPropertyType* property_type, _Outptr_ void** property_value); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Load a checkpoint state from a buffer into checkpoint_state. - * - * This function will parse a checkpoint bytes buffer, pull relevant data and load the training - * state into the checkpoint_state. This checkpoint state can then be used to create the - * training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training - * session will resume training from the given checkpoint state. - * \note Note that the training session created with a checkpoint state uses this state to store the entire - * training state (including model parameters, its gradients, the optimizer states and the properties). - * As a result, it is required that the checkpoint state outlive the lifetime of the training session. - * - * \param[in] checkpoint_buffer Path to the checkpoint bytes buffer. - * \param[in] num_bytes Number of bytes in the checkpoint buffer. - * \param[out] checkpoint_state Checkpoint state that contains the states of the training session. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(LoadCheckpointFromBuffer, _In_ const void* checkpoint_buffer, - _In_ const size_t num_bytes, _Outptr_ OrtCheckpointState** checkpoint_state); - - /** \brief Retrieves the type and shape information of the parameter associated with the given parameter name. - * - * This function retrieves the type and shape of the parameter associated with the given parameter name. - * The parameter must exist in the checkpoint state to be able to retrieve its type and shape information successfully. - * - * \param[in] checkpoint_state The checkpoint state. - * \param[in] parameter_name Name of the parameter being retrieved. - * \param[out] parameter_type_and_shape The type and shape of the parameter being retrieved. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetParameterTypeAndShape, _In_ const OrtCheckpointState* checkpoint_state, - _In_ const char* parameter_name, _Outptr_ OrtTensorTypeAndShapeInfo** parameter_type_and_shape); - - /** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name. - * - * This function updates a model parameter in the checkpoint state with the given parameter data. - * The training session must be already created with the checkpoint state that contains the parameter - * being updated. The given parameter is copied over to the registered device for the training session. - * The parameter must exist in the checkpoint state to be able to update it successfully. - * - * \param[in] checkpoint_state The checkpoint state. - * \param[in] parameter_name Name of the parameter being updated. - * \param[in] parameter The parameter data that should replace the existing parameter data. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(UpdateParameter, _Inout_ OrtCheckpointState* checkpoint_state, - _In_ const char* parameter_name, _In_ OrtValue* parameter); - - /** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name. - * - * This function retrieves the model parameter data from the checkpoint state for the given parameter name. - * The parameter is copied over and returned as an OrtValue. The training session must be already created - * with the checkpoint state that contains the parameter being retrieved. - * The parameter must exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] checkpoint_state The checkpoint state. - * \param[in] parameter_name Name of the parameter being retrieved. - * \param[in] allocator Allocator used to allocate the memory for the parameter. - * \param[out] parameter The parameter data that is retrieved from the checkpoint state. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - */ - ORT_API2_STATUS(GetParameter, _In_ const OrtCheckpointState* checkpoint_state, - _In_ const char* parameter_name, _Inout_ OrtAllocator* allocator, - _Outptr_ OrtValue** parameter); - - /// @} -}; - -typedef struct OrtTrainingApi OrtTrainingApi; - -/// @} diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_cxx_api.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_cxx_api.h deleted file mode 100644 index e78c161..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_cxx_api.h +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "onnxruntime_training_c_api.h" -#include -#include - -namespace Ort::detail { - -#define ORT_DECLARE_TRAINING_RELEASE(NAME) \ - void OrtRelease(Ort##NAME* ptr); - -// These release methods must be forward declared before including onnxruntime_cxx_api.h -// otherwise class Base won't be aware of them -ORT_DECLARE_TRAINING_RELEASE(CheckpointState); -ORT_DECLARE_TRAINING_RELEASE(TrainingSession); - -} // namespace Ort::detail - -#include "onnxruntime_cxx_api.h" - -namespace Ort { - -/// -/// This function returns the C training api struct with the pointers to the ort training C functions. -/// If using C++, please use the class instances instead of invoking the C functions directly. -/// -/// OrtTrainingApi struct with ort training C function pointers. -inline const OrtTrainingApi& GetTrainingApi() { return *GetApi().GetTrainingApi(ORT_API_VERSION); } - -namespace detail { - -#define ORT_DEFINE_TRAINING_RELEASE(NAME) \ - inline void OrtRelease(Ort##NAME* ptr) { GetTrainingApi().Release##NAME(ptr); } - -ORT_DEFINE_TRAINING_RELEASE(CheckpointState); -ORT_DEFINE_TRAINING_RELEASE(TrainingSession); - -#undef ORT_DECLARE_TRAINING_RELEASE -#undef ORT_DEFINE_TRAINING_RELEASE - -} // namespace detail - -using Property = std::variant; - -/** - * \defgroup TrainingCpp Ort Training C++ API - * @{ - */ - -/** \brief Holds the state of the training session. - * - * This class holds the entire training session state that includes model parameters, their gradients, - * optimizer parameters, and user properties. The Ort::TrainingSession leverages the Ort::CheckpointState - * by accessing and updating the contained training state. - * \note Note that the training session created with a checkpoint state uses this state to store the entire - * training state (including model parameters, its gradients, the optimizer states and the properties). - * The Ort::TrainingSession does not hold a copy of the Ort::CheckpointState and as a result, it is required - * that the checkpoint state outlive the lifetime of the training session. - * \note Note that the checkpoint state can be either the complete checkpoint state or the nominal checkpoint - * state depending on the version provided while loading the checkpoint. - * - */ -class CheckpointState : public detail::Base { - private: - CheckpointState(OrtCheckpointState* checkpoint_state) { p_ = checkpoint_state; } - - public: - // Construct the checkpoint state by loading the checkpoint by calling LoadCheckpoint - CheckpointState() = delete; - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Load a checkpoint state from a file on disk into checkpoint_state. - * - * This function will parse a checkpoint file, pull relevant data and load the training - * state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the - * training session by instantiating Ort::TrainingSession. By doing so, the training session will resume - * training from the given checkpoint state. - * - * \param[in] path_to_checkpoint Path to the checkpoint file - * \return Ort::CheckpointState object which holds the state of the training session parameters. - * - */ - static CheckpointState LoadCheckpoint(const std::basic_string& path_to_checkpoint); - - /** \brief Load a checkpoint state from a buffer. - * - * This function will parse a checkpoint buffer, pull relevant data and load the training - * state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the - * training session by instantiating Ort::TrainingSession. By doing so, the training session will resume - * training from the given checkpoint state. - * - * \param[in] buffer Buffer containing the checkpoint data. - * \return Ort::CheckpointState object which holds the state of the training session parameters. - * - */ - static CheckpointState LoadCheckpointFromBuffer(const std::vector& buffer); - - /** \brief Save the given state to a checkpoint file on disk. - * - * This function serializes the provided checkpoint state to a file on disk. - * This checkpoint can later be loaded by invoking Ort::CheckpointState::LoadCheckpoint to resume - * training from this snapshot of the state. - * - * \param[in] checkpoint_state The checkpoint state to save. - * \param[in] path_to_checkpoint Path to the checkpoint file. - * \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not. - * - */ - static void SaveCheckpoint(const CheckpointState& checkpoint_state, - const std::basic_string& path_to_checkpoint, - const bool include_optimizer_state = false); - - /** \brief Adds or updates the given property to/in the checkpoint state. - * - * Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint - * state by the user by calling this function with the corresponding property name and value. - * The given property name must be unique to be able to successfully add the property. - * - * \param[in] property_name Name of the property being added or updated. - * \param[in] property_value Property value associated with the given name. - * - */ - void AddProperty(const std::string& property_name, const Property& property_value); - - /** \brief Gets the property value associated with the given name from the checkpoint state. - * - * Gets the property value from an existing entry in the checkpoint state. The property must - * exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] property_name Name of the property being retrieved. - * \return Property value associated with the given property name. - * - */ - Property GetProperty(const std::string& property_name); - - /** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name. - * - * This function updates a model parameter in the checkpoint state with the given parameter data. - * The training session must be already created with the checkpoint state that contains the parameter - * being updated. The given parameter is copied over to the registered device for the training session. - * The parameter must exist in the checkpoint state to be able to update it successfully. - * - * \param[in] parameter_name Name of the parameter being updated. - * \param[in] parameter The parameter data that should replace the existing parameter data. - * - */ - void UpdateParameter(const std::string& parameter_name, const Value& parameter); - - /** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name. - * - * This function retrieves the model parameter data from the checkpoint state for the given parameter name. - * The parameter is copied over to the provided OrtValue. The training session must be already created - * with the checkpoint state that contains the parameter being retrieved. - * The parameter must exist in the checkpoint state to be able to retrieve it successfully. - * - * \param[in] parameter_name Name of the parameter being retrieved. - * \return The parameter data that is retrieved from the checkpoint state. - * - */ - Value GetParameter(const std::string& parameter_name); - - /// @} -}; - -/** \brief Trainer class that provides training, evaluation and optimizer methods for training an ONNX models. - * - * The training session requires four training artifacts - * - The training onnx model - * - The evaluation onnx model (optional) - * - The optimizer onnx model - * - The checkpoint file - * - * These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md). - * - */ -class TrainingSession : public detail::Base { - private: - size_t training_model_output_count_, eval_model_output_count_; - - public: - /// \name Constructing the Training Session - /// @{ - /** \brief Create a training session that can be used to begin or resume training. - * - * This constructor instantiates the training session based on the env and session options provided that can - * begin or resume training from a given checkpoint state for the given onnx models. - * The checkpoint state represents the parameters of the training session which will be moved - * to the device specified by the user through the session options (if necessary). - * - * \param[in] env Env to be used for the training session. - * \param[in] session_options SessionOptions that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_path Model to be used to perform training. - * \param[in] eval_model_path Model to be used to perform evaluation. - * \param[in] optimizer_model_path Model to be used to perform gradient descent. - * - */ - TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state, - const std::basic_string& train_model_path, - const std::optional>& eval_model_path = std::nullopt, - const std::optional>& optimizer_model_path = std::nullopt); - - /** \brief Create a training session that can be used to begin or resume training. - * This constructor allows the users to load the models from buffers instead of files. - * - * \param[in] env Env to be used for the training session. - * \param[in] session_options SessionOptions that the user can customize for this training session. - * \param[in] checkpoint_state Training states that the training session uses as a starting point for training. - * \param[in] train_model_data Buffer containing training model data. - * \param[in] eval_model_data Buffer containing evaluation model data. - * \param[in] optim_model_data Buffer containing optimizer model (used for performing weight/parameter update). - * - */ - TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state, - const std::vector& train_model_data, const std::vector& eval_model_data = {}, - const std::vector& optim_model_data = {}); - /// @} - - /// \name Implementing The Training Loop - /// @{ - /** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs - * - * This function performs a training step that computes the outputs of the training model and the gradients - * of the trainable parameters for the given inputs. The train step is performed based on the training model - * that was provided to the training session. - * The Ort::TrainingSession::TrainStep is equivalent of running forward propagation and backward propagation in a single - * step. - * The gradients computed are stored inside the training session state so they can be later consumed - * by the Ort::TrainingSession::OptimizerStep function. - * The gradients can be lazily reset by invoking the Ort::TrainingSession::LazyResetGrad function. - * - * \param[in] input_values The user inputs to the training model. - * \return A std::vector of Ort::Value objects that represents the output of the forward pass of the training model. - * - * - */ - std::vector TrainStep(const std::vector& input_values); - - /** \brief Reset the gradients of all trainable parameters to zero lazily. - * - * This function sets the internal state of the training session such that the gradients of the trainable - * parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are - * computed on the next invocation of the next Ort::TrainingSession::TrainStep. - * - */ - void LazyResetGrad(); - - /** \brief Computes the outputs for the eval model for the given inputs - * - * This function performs an eval step that computes the outputs of the eval model for the given inputs. - * The eval step is performed based on the eval model that was provided to the training session. - * - * \param[in] input_values The user inputs to the eval model. - * \return A std::vector of Ort::Value objects that represents the output of the eval pass. - * - */ - std::vector EvalStep(const std::vector& input_values); - - /** \brief Sets the learning rate for this training session. - * - * This function allows users to set the learning rate for the training session. The current - * learning rate is maintained by the training session and can be overwritten by invoking - * this function with the desired learning rate. This function should not be used when a valid - * learning rate scheduler is registered. It should be used either to set the learning rate - * derived from a custom learning rate scheduler or to set a constant learning rate to be used - * throughout the training session. - * \note Please note that this function does not set the initial learning rate that may be needed - * by the predefined learning rate schedulers. To set the initial learning rate for learning - * rate schedulers, please look at the function Ort::TrainingSession::RegisterLinearLRScheduler. - * - * \param[in] learning_rate Desired learning rate to be set. - * - */ - void SetLearningRate(float learning_rate); - - /** \brief Gets the current learning rate for this training session. - * - * This function allows users to get the learning rate for the training session. The current - * learning rate is maintained by the training session, and users can query it for the purpose - * of implementing their own learning rate schedulers. - * - * \return float representing the current learning rate. - * - */ - float GetLearningRate() const; - - /** \brief Registers a linear learning rate scheduler for the training session. - * - * Register a linear learning rate scheduler that decays the learning rate by linearly updated - * multiplicative factor from the initial learning rate set on the training session to 0. The decay - * is performed after the initial warm up phase where the learning rate is linearly incremented - * from 0 to the initial learning rate provided. - * - * \param[in] warmup_step_count Warmup steps for LR warmup. - * \param[in] total_step_count Total step count. - * \param[in] initial_lr The initial learning rate to be used by the training session. - * - */ - void RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count, - float initial_lr); - - /** \brief Update the learning rate based on the registered learing rate scheduler. - * - * Takes a scheduler step that updates the learning rate that is being used by the training session. - * This function should typically be called before invoking the optimizer step for each round, - * or as determined necessary to update the learning rate being used by the training session. - * \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this - * function. - * - */ - void SchedulerStep(); - - /** \brief Performs the weight updates for the trainable parameters using the optimizer model. - * - * This function performs the weight update step that updates the trainable parameters such that they - * take a step in the direction of their gradients (gradient descent). The optimizer step is performed - * based on the optimizer model that was provided to the training session. - * The updated parameters are stored inside the training state so that they can be used by the next - * Ort::TrainingSession::TrainStep function call. - * - */ - void OptimizerStep(); - - /// @} - - /// \name Prepare For Inferencing - /// @{ - - /** \brief Export a model that can be used for inferencing. - * - * If the training session was provided with an eval model, the training session can generate - * an inference model if it knows the inference graph outputs. The input inference graph outputs - * are used to prune the eval model so that the inference model's outputs align with the provided outputs. - * The exported model is saved at the path provided and can be used for inferencing with Ort::Session. - * \note Note that the function re-loads the eval model from the path provided to Ort::TrainingSession - * and expects that this path still be valid. - * - * \param[in] inference_model_path Path where the inference model should be serialized to. - * \param[in] graph_output_names Names of the outputs that are needed in the inference model. - * - */ - void ExportModelForInferencing(const std::basic_string& inference_model_path, - const std::vector& graph_output_names); - - /// @} - - /// \name Model IO Information - /// @{ - /** \brief Retrieves the names of the user inputs for the training and eval models. - * - * This function returns the names of inputs of the training or eval model that can be associated - * with the Ort::Value(s) provided to the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep - * function. - * - * \param[in] training Whether the training model input names are requested or eval model input names. - * \return Graph input names for either the training model or the eval model. - * - */ - std::vector InputNames(const bool training); - - /** \brief Retrieves the names of the user outputs for the training and eval models. - * - * This function returns the names of outputs of the training or eval model that can be associated - * with the Ort::Value(s) returned by the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep - * function. - * - * \param[in] training Whether the training model output names are requested or eval model output names. - * \return Graph output names for either the training model or the eval model. - * - */ - std::vector OutputNames(const bool training); - - /// @} - - /// \name Accessing The Training Session State - /// @{ - - /** \brief Returns a contiguous buffer that holds a copy of all training state parameters - * - * \param[in] only_trainable Whether to only copy trainable parameters or to copy all parameters. - * \return Contiguous buffer to the model parameters. - * - */ - Value ToBuffer(const bool only_trainable); - - /** \brief Loads the training session model parameters from a contiguous buffer - * - * In case the training session was created with a nominal checkpoint, invoking this function is required - * to load the updated parameters onto the checkpoint to complete it. - * - * \param[in] buffer Contiguous buffer to load the parameters from. - */ - void FromBuffer(Value& buffer); - - /// @} -}; - -/// \name Training Utilities -/// @{ -/** \brief This function sets the seed for generating random numbers. - * - * Use this function to generate reproducible results. It should be noted that completely - * reproducible results are not guaranteed. - * - * \param[in] seed Manual seed to use for random number generation. - */ -void SetSeed(const int64_t seed); -/// @} - -/// @} - -} // namespace Ort - -#include "onnxruntime_training_cxx_inline.h" diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_cxx_inline.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_cxx_inline.h deleted file mode 100644 index 397cba0..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/onnxruntime_training_cxx_inline.h +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "onnxruntime_training_c_api.h" -#include "onnxruntime_cxx_api.h" - -namespace Ort { - -inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options, - CheckpointState& checkpoint_state, - const std::basic_string& train_model_path, - const std::optional>& eval_model_path, - const std::optional>& optimizer_model_path) { - ThrowOnError(GetTrainingApi().CreateTrainingSession( - env, session_options, checkpoint_state, - train_model_path.c_str(), - eval_model_path.has_value() ? eval_model_path.value().c_str() : nullptr, - optimizer_model_path.has_value() ? optimizer_model_path.value().c_str() : nullptr, - &p_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_)); -} - -inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options, - CheckpointState& checkpoint_state, - const std::vector& train_model_data, - const std::vector& eval_model_data, - const std::vector& optim_model_data) { - ThrowOnError(GetTrainingApi().CreateTrainingSessionFromBuffer( - env, session_options, checkpoint_state, - train_model_data.data(), train_model_data.size(), - eval_model_data.data(), eval_model_data.size(), - optim_model_data.data(), optim_model_data.size(), - &p_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_)); - - ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_)); -} - -inline std::vector TrainingSession::TrainStep(const std::vector& input_values) { - std::vector output_values; - output_values.reserve(training_model_output_count_); - for (size_t i = 0; i < training_model_output_count_; i++) output_values.emplace_back(nullptr); - auto ort_input_values = reinterpret_cast(input_values.data()); - auto ort_output_values = reinterpret_cast(output_values.data()); - RunOptions run_options; - ThrowOnError(GetTrainingApi().TrainStep( - p_, run_options, input_values.size(), ort_input_values, - training_model_output_count_, ort_output_values)); - - return output_values; -} - -inline void TrainingSession::LazyResetGrad() { - ThrowOnError(GetTrainingApi().LazyResetGrad(p_)); -} - -inline std::vector TrainingSession::EvalStep(const std::vector& input_values) { - std::vector output_values; - output_values.reserve(eval_model_output_count_); - for (size_t i = 0; i < eval_model_output_count_; i++) output_values.emplace_back(nullptr); - auto ort_input_values = reinterpret_cast(input_values.data()); - auto ort_output_values = reinterpret_cast(output_values.data()); - RunOptions run_options; - ThrowOnError(GetTrainingApi().EvalStep( - p_, run_options, input_values.size(), ort_input_values, - eval_model_output_count_, ort_output_values)); - - return output_values; -} - -inline void TrainingSession::SetLearningRate(float learning_rate) { - ThrowOnError(GetTrainingApi().SetLearningRate(p_, learning_rate)); -} - -inline float TrainingSession::GetLearningRate() const { - float learning_rate = 0; - ThrowOnError(GetTrainingApi().GetLearningRate(p_, &learning_rate)); - return learning_rate; -} - -inline void TrainingSession::RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count, - float initial_lr) { - ThrowOnError(GetTrainingApi().RegisterLinearLRScheduler(p_, warmup_step_count, total_step_count, - initial_lr)); -} - -inline void TrainingSession::SchedulerStep() { - ThrowOnError(GetTrainingApi().SchedulerStep(p_)); -} - -inline void TrainingSession::OptimizerStep() { - RunOptions run_options; - ThrowOnError(GetTrainingApi().OptimizerStep(p_, run_options)); -} - -inline std::vector TrainingSession::InputNames(const bool training) { - auto& input_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputCount - : GetTrainingApi().TrainingSessionGetEvalModelInputCount; - auto& input_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputName - : GetTrainingApi().TrainingSessionGetEvalModelInputName; - - size_t input_count = 0; - ThrowOnError(input_count_function(p_, &input_count)); - std::vector input_names(input_count); - AllocatorWithDefaultOptions allocator; - for (size_t index = 0; index < input_count; ++index) { - char* input_name; - ThrowOnError(input_name_function(p_, index, allocator, &input_name)); - input_names[index] = std::string(input_name); - allocator.Free(input_name); - } - - return input_names; -} - -inline std::vector TrainingSession::OutputNames(const bool training) { - auto& output_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputCount - : GetTrainingApi().TrainingSessionGetEvalModelOutputCount; - auto& output_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputName - : GetTrainingApi().TrainingSessionGetEvalModelOutputName; - - size_t output_count = 0; - ThrowOnError(output_count_function(p_, &output_count)); - std::vector output_names(output_count); - AllocatorWithDefaultOptions allocator; - for (size_t index = 0; index < output_count; ++index) { - char* output_name; - ThrowOnError(output_name_function(p_, index, allocator, &output_name)); - output_names[index] = std::string(output_name); - allocator.Free(output_name); - } - - return output_names; -} - -inline Value TrainingSession::ToBuffer(const bool only_trainable) { - size_t buffer_size = 0U; - ThrowOnError(GetTrainingApi().GetParametersSize(p_, &buffer_size, only_trainable)); - - std::array buffer_shape{static_cast(buffer_size)}; - - AllocatorWithDefaultOptions allocator; - Value buffer = Value::CreateTensor(allocator, buffer_shape.data(), 1U, - ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); - - ThrowOnError(GetTrainingApi().CopyParametersToBuffer(p_, buffer, only_trainable)); - - return buffer; -} - -inline void TrainingSession::FromBuffer(Value& buffer) { - if (!buffer.IsTensor()) { - ThrowStatus(Status("Incorrect buffer received. Expected a tensor buffer.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - } - - auto tensor_info = buffer.GetTensorTypeAndShapeInfo(); - auto buffer_shape = tensor_info.GetShape(); - - if (buffer_shape.size() != 1U) { - ThrowStatus(Status("Incorrect buffer received. Expected a contiguous tensor buffer.", - OrtErrorCode::ORT_INVALID_ARGUMENT)); - } - - auto buffer_size = buffer_shape.front(); - - size_t session_buffer_size = 0U; - ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size, false)); - - if (buffer_size == static_cast(session_buffer_size)) { - ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, false)); - return; - } - - size_t session_buffer_size_trainable_only = 0U; - ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size_trainable_only, true)); - - if (buffer_size == static_cast(session_buffer_size_trainable_only)) { - ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, true)); - return; - } else { - ThrowStatus(Status("Incorrect buffer size received.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - } -} - -inline CheckpointState CheckpointState::LoadCheckpoint(const std::basic_string& path_to_checkpoint) { - OrtCheckpointState* checkpoint_state; - ThrowOnError(GetTrainingApi().LoadCheckpoint(path_to_checkpoint.c_str(), &checkpoint_state)); - return CheckpointState(checkpoint_state); -} - -inline CheckpointState CheckpointState::LoadCheckpointFromBuffer(const std::vector& buffer) { - OrtCheckpointState* checkpoint_state; - ThrowOnError(GetTrainingApi().LoadCheckpointFromBuffer(buffer.data(), buffer.size(), &checkpoint_state)); - return CheckpointState(checkpoint_state); -} - -inline void CheckpointState::SaveCheckpoint(const CheckpointState& checkpoint_states, - const std::basic_string& path_to_checkpoint, - const bool include_optimizer_state) { - ThrowOnError(GetTrainingApi().SaveCheckpoint(checkpoint_states, path_to_checkpoint.c_str(), - include_optimizer_state)); -} - -inline void TrainingSession::ExportModelForInferencing(const std::basic_string& inference_model_path, - const std::vector& graph_output_names) { - std::vector output_names; - output_names.reserve(graph_output_names.size()); - for (const auto& output_name : graph_output_names) { - output_names.push_back(output_name.c_str()); - } - ThrowOnError(GetTrainingApi().ExportModelForInferencing( - p_, inference_model_path.c_str(), graph_output_names.size(), output_names.data())); -} - -inline void SetSeed(const int64_t seed) { - ThrowOnError(GetTrainingApi().SetSeed(seed)); -} - -inline void CheckpointState::AddProperty(const std::string& property_name, const Property& property_value) { - if (std::holds_alternative(property_value)) { - int64_t value = std::get(property_value); - void* value_p = &value; - ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtIntProperty, value_p)); - } else if (std::holds_alternative(property_value)) { - float value = std::get(property_value); - void* value_p = &value; - ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtFloatProperty, value_p)); - } else if (std::holds_alternative(property_value)) { - std::string value = std::get(property_value); - auto buffer = std::make_unique(value.length() + 1); - memcpy(buffer.get(), value.c_str(), value.length()); - // AddProperty takes a char* and calls PropertyBag::AddProperty which takes a std::string. The data will be - // copied at that point so buffer can free the local allocation once the call is made. - ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtStringProperty, - buffer.get())); - } else { - ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - } -} - -inline Property CheckpointState::GetProperty(const std::string& property_name) { - void* property_value = nullptr; - OrtPropertyType property_type; - - AllocatorWithDefaultOptions allocator; - ThrowOnError(GetTrainingApi().GetProperty(p_, property_name.c_str(), allocator, &property_type, &property_value)); - - Property property; - - switch (property_type) { - case OrtPropertyType::OrtIntProperty: { - auto value_p = reinterpret_cast(property_value); - property = *value_p; - allocator.Free(property_value); - break; - } - case OrtPropertyType::OrtFloatProperty: { - auto value_p = reinterpret_cast(property_value); - property = *value_p; - allocator.Free(property_value); - break; - } - case OrtPropertyType::OrtStringProperty: { - auto value_p = reinterpret_cast(property_value); - property = std::string(value_p); - allocator.Free(property_value); - break; - } - default: { - ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT)); - break; - } - } - - return property; -} - -inline void CheckpointState::UpdateParameter(const std::string& parameter_name, const Value& parameter) { - ThrowOnError(GetTrainingApi().UpdateParameter(p_, parameter_name.c_str(), parameter)); -} - -inline Value CheckpointState::GetParameter(const std::string& parameter_name) { - AllocatorWithDefaultOptions allocator; - OrtValue* parameter; - ThrowOnError(GetTrainingApi().GetParameter(p_, parameter_name.c_str(), allocator, ¶meter)); - - return Value{parameter}; -} - -} // namespace Ort diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/include/provider_options.h b/inference_core/ort_core/onnxruntime-pkg-x86/include/provider_options.h deleted file mode 100644 index aab13e8..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/include/provider_options.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include -#include -#include - -namespace onnxruntime { - -// data types for execution provider options - -using ProviderOptions = std::unordered_map; -using ProviderOptionsVector = std::vector; -using ProviderOptionsMap = std::unordered_map; - -} // namespace onnxruntime diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/lib/libonnxruntime.so b/inference_core/ort_core/onnxruntime-pkg-x86/lib/libonnxruntime.so deleted file mode 120000 index cc5e169..0000000 --- a/inference_core/ort_core/onnxruntime-pkg-x86/lib/libonnxruntime.so +++ /dev/null @@ -1 +0,0 @@ -libonnxruntime.so.1.18.1 \ No newline at end of file diff --git a/inference_core/ort_core/onnxruntime-pkg-x86/lib/libonnxruntime.so.1.18.1 b/inference_core/ort_core/onnxruntime-pkg-x86/lib/libonnxruntime.so.1.18.1 deleted file mode 100755 index 492aede..0000000 Binary files a/inference_core/ort_core/onnxruntime-pkg-x86/lib/libonnxruntime.so.1.18.1 and /dev/null differ diff --git a/inference_core/ort_core/src/ort_core.cpp b/inference_core/ort_core/src/ort_core.cpp deleted file mode 100644 index e3806d3..0000000 --- a/inference_core/ort_core/src/ort_core.cpp +++ /dev/null @@ -1,450 +0,0 @@ -#include "ort_core/ort_core.h" - -#include -#include -#include - -namespace inference_core { - -enum BlobType { kINPUT = 0, kOUTPUT = 1 }; - -template -inline static std::string VisualVec(const std::vector &vec) -{ - std::string ret; - for (const auto &v : vec) - { - ret += std::to_string(v) + " "; - } - return ret; -} - -struct OrtBlobBuffer : public IBlobsBuffer { -public: - std::pair GetOuterBlobBuffer(const std::string &blob_name) noexcept override - { - if (outer_map_blob2ptr.find(blob_name) == outer_map_blob2ptr.end()) - { - LOG(ERROR) << "[OrtBlobBuffer] `GetOuterBlobBuffer` Got invalid `blob_name`: " << blob_name; - return {nullptr, UNKOWN}; - } - return outer_map_blob2ptr[blob_name]; - } - - bool SetBlobBuffer(const std::string &blob_name, - void *data_ptr, - DataLocation location) noexcept override - { - if (inner_map_blob2ptr.find(blob_name) == inner_map_blob2ptr.end()) - { - LOG(ERROR) << "[OrtBlobBuffer] `SetBlobBuffer` Got invalid `blob_name`: " << blob_name; - return false; - } - outer_map_blob2ptr[blob_name] = {data_ptr, location}; - return true; - } - - bool SetBlobBuffer(const std::string &blob_name, DataLocation location) noexcept override - { - if (inner_map_blob2ptr.find(blob_name) == inner_map_blob2ptr.end()) - { - LOG(ERROR) << "[OrtBlobBuffer] `SetBlobBuffer` Got invalid `blob_name`: " << blob_name; - return false; - } - outer_map_blob2ptr[blob_name] = {inner_map_blob2ptr[blob_name], location}; - return true; - } - - bool SetBlobShape(const std::string &blob_name, - const std::vector &shape) noexcept override - { - if (map_blob_name2shape.find(blob_name) == map_blob_name2shape.end()) - { - LOG(ERROR) << "[OrtBlobBuffer] `SetBlobShape` Got invalid `blob_name`: " << blob_name; - return false; - } - const auto &origin_shape = map_blob_name2shape[blob_name]; - if (origin_shape.size() != shape.size()) - { - const std::string origin_shape_in_str = VisualVec(origin_shape); - const std::string shape_in_str = VisualVec(shape); - LOG(ERROR) << "[OrtBlobBuffer] `SetBlobShape` Got invalid `shape` input. " - << "`shape`: " << shape_in_str << "\t" - << "`origin_shape`: " << origin_shape_in_str; - return false; - } - map_blob_name2shape[blob_name] = shape; - return true; - } - - const std::vector &GetBlobShape(const std::string &blob_name) const noexcept override - { - if (map_blob_name2shape.find(blob_name) == map_blob_name2shape.end()) - { - LOG(ERROR) << "[TrtBlobBuffer] `GetBlobShape` Got invalid `blob_name`: " << blob_name; - static std::vector empty_shape; - return empty_shape; - } - return map_blob_name2shape.at(blob_name); - } - - size_t Size() const noexcept override - { - return inner_map_blob2ptr.size(); - } - - void Release() noexcept override - { - Reset(); - for (float *&ptr : blobs_buffer) - { - if (ptr != nullptr) - delete[] ptr; - ptr = nullptr; - } - } - - void Reset() noexcept override - { - map_type2tensors[BlobType::kINPUT].clear(); - map_type2tensors[BlobType::kOUTPUT].clear(); - for (const auto &p_name_ptr : inner_map_blob2ptr) - { - outer_map_blob2ptr[p_name_ptr.first] = {p_name_ptr.second, DataLocation::HOST}; - } - } - - ~OrtBlobBuffer() override - { - Release(); - } - OrtBlobBuffer() = default; - OrtBlobBuffer(const OrtBlobBuffer &) = delete; - OrtBlobBuffer &operator=(const OrtBlobBuffer &) = delete; - - std::unordered_map> outer_map_blob2ptr; - std::unordered_map inner_map_blob2ptr; - - std::vector blobs_buffer; - - std::unordered_map> map_type2tensors; - - std::unordered_map> map_blob_name2shape; -}; - -class OrtInferCore : public BaseInferCore { -public: - ~OrtInferCore() override = default; - - OrtInferCore(const std::string onnx_path, - const std::unordered_map> &input_blobs_shape, - const std::unordered_map> &output_blobs_shape, - const int num_threads = 0); - - OrtInferCore(const std::string onnx_path, const int num_threads = 0); - - std::shared_ptr AllocBlobsBuffer() override; - - InferCoreType GetType() - { - return InferCoreType::ONNXRUNTIME; - } - - std::string GetName() - { - return "ort_core"; - } - -private: - bool PreProcess(std::shared_ptr buffer) override; - - bool Inference(std::shared_ptr buffer) override; - - bool PostProcess(std::shared_ptr buffer) override; - -private: - std::unordered_map> ResolveModelInputInformation(); - - std::unordered_map> ResolveModelOutputInformation(); - - std::unordered_map map_blob2ptr_; - - std::shared_ptr ort_env_; - - std::shared_ptr ort_session_; - - std::unordered_map> map_input_blob_name2shape_; - std::unordered_map> map_output_blob_name2shape_; -}; - -OrtInferCore::OrtInferCore( - const std::string onnx_path, - const std::unordered_map> &input_blobs_shape, - const std::unordered_map> &output_blobs_shape, - const int num_threads) -{ - // onnxruntime session initialization - LOG(INFO) << "start initializing onnxruntime session with onnx model {" << onnx_path << "} ..."; - ort_env_ = std::make_shared(ORT_LOGGING_LEVEL_ERROR, onnx_path.data()); - Ort::SessionOptions session_options; - session_options.SetIntraOpNumThreads(num_threads); - session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); - session_options.SetLogSeverityLevel(4); - ort_session_ = std::make_shared(*ort_env_, onnx_path.c_str(), session_options); - LOG(INFO) << "successfully created onnxruntime session!"; - - map_input_blob_name2shape_ = - input_blobs_shape.empty() ? ResolveModelInputInformation() : input_blobs_shape; - map_output_blob_name2shape_ = - output_blobs_shape.empty() ? ResolveModelOutputInformation() : output_blobs_shape; - - // show info - auto func_display_blobs_info = - [](const std::unordered_map> &blobs_shape) { - for (const auto &p_name_shape : blobs_shape) - { - std::string s_blob_shape; - for (const auto dim : p_name_shape.second) - { - s_blob_shape += std::to_string(dim) + "\t"; - } - LOG(INFO) << p_name_shape.first << "\tshape: " << s_blob_shape; - } - }; - - func_display_blobs_info(input_blobs_shape); - func_display_blobs_info(output_blobs_shape); - - BaseInferCore::Init(); -} - -std::unordered_map> OrtInferCore::ResolveModelInputInformation() -{ - std::unordered_map> ret; - - OrtAllocator *allocator = nullptr; - bool allocator_init_status = Ort::GetApi().GetAllocatorWithDefaultOptions(&allocator) == nullptr; - CHECK(allocator_init_status); - - const int input_blob_count = ort_session_->GetInputCount(); - - for (int i = 0; i < input_blob_count; ++i) - { - const auto blob_info = ort_session_->GetInputTypeInfo(i); - const auto blob_type_shape = blob_info.GetTensorTypeAndShapeInfo(); - const auto blob_shape = blob_type_shape.GetShape(); - const auto blob_name = ort_session_->GetInputNameAllocated(i, allocator); - const std::string s_blob_name = std::string(blob_name.get()); - - ret[s_blob_name] = std::vector(); - std::string s_blob_info = std::string(blob_name.get()) + ":\t"; - size_t blob_element_size = 1; - for (size_t i = 0; i < blob_shape.size(); ++i) - { - if (blob_shape[i] < 0) - { - throw std::runtime_error( - "auto resolve onnx model failed! \ - for blob shape < 0, please use explicit blob shape constructor!!"); - } - s_blob_info += "\t" + std::to_string(blob_shape[i]); - blob_element_size *= blob_shape[i]; - ret[s_blob_name].push_back(blob_shape[i]); - } - s_blob_info += "\ttotal elements: " + std::to_string(blob_element_size); - LOG(INFO) << s_blob_info; - } - - return ret; -} - -std::unordered_map> OrtInferCore::ResolveModelOutputInformation() -{ - std::unordered_map> ret; - - OrtAllocator *allocator = nullptr; - bool allocator_init_status = Ort::GetApi().GetAllocatorWithDefaultOptions(&allocator) == nullptr; - CHECK(allocator_init_status); - - const int output_blob_count = ort_session_->GetOutputCount(); - - for (int i = 0; i < output_blob_count; ++i) - { - const auto blob_info = ort_session_->GetOutputTypeInfo(i); - const auto blob_type_shape = blob_info.GetTensorTypeAndShapeInfo(); - const auto blob_shape = blob_type_shape.GetShape(); - const auto blob_name = ort_session_->GetOutputNameAllocated(i, allocator); - const std::string s_blob_name = std::string(blob_name.get()); - - ret[s_blob_name] = std::vector(); - std::string s_blob_info = std::string(blob_name.get()) + ":\t"; - size_t blob_element_size = 1; - for (size_t i = 0; i < blob_shape.size(); ++i) - { - if (blob_shape[i] < 0) - { - throw std::runtime_error( - "auto resolve onnx model failed! \ - for blob shape < 0, please use explicit blob shape constructor!!"); - } - s_blob_info += "\t" + std::to_string(blob_shape[i]); - blob_element_size *= blob_shape[i]; - ret[s_blob_name].push_back(blob_shape[i]); - } - s_blob_info += "\ttotal elements: " + std::to_string(blob_element_size); - LOG(INFO) << s_blob_info; - } - - return ret; -} - -std::shared_ptr OrtInferCore::AllocBlobsBuffer() -{ - OrtAllocator *allocator = nullptr; - bool allocator_init_status = Ort::GetApi().GetAllocatorWithDefaultOptions(&allocator) == nullptr; - CHECK(allocator_init_status); - - auto ret = std::make_shared(); - - // input blobs - const int input_blob_count = map_input_blob_name2shape_.size(); - for (int i = 0; i < input_blob_count; ++i) - { - const auto blob_name = ort_session_->GetInputNameAllocated(i, allocator); - const std::string s_blob_name = std::string(blob_name.get()); - const auto &blob_shape = map_input_blob_name2shape_[s_blob_name]; - int64_t element_size = 1; - for (auto s : blob_shape) - { - element_size *= s; - } - - ret->blobs_buffer.push_back(new float[element_size]); - ret->inner_map_blob2ptr.insert({s_blob_name, static_cast(ret->blobs_buffer.back())}); - ret->outer_map_blob2ptr.insert( - {s_blob_name, {static_cast(ret->blobs_buffer.back()), DataLocation::HOST}}); - ret->map_blob_name2shape.emplace(s_blob_name, blob_shape); - } - - // output blobs - const int output_blob_count = map_output_blob_name2shape_.size(); - for (int i = 0; i < output_blob_count; ++i) - { - const auto blob_name = ort_session_->GetOutputNameAllocated(i, allocator); - const std::string s_blob_name = std::string(blob_name.get()); - const auto &blob_shape = map_output_blob_name2shape_[s_blob_name]; - int64_t element_size = 1; - for (auto s : blob_shape) - { - element_size *= s; - } - - ret->blobs_buffer.push_back(new float[element_size]); - ret->inner_map_blob2ptr.insert({s_blob_name, static_cast(ret->blobs_buffer.back())}); - ret->outer_map_blob2ptr.insert( - {s_blob_name, {static_cast(ret->blobs_buffer.back()), DataLocation::HOST}}); - ret->map_blob_name2shape.emplace(s_blob_name, blob_shape); - } - - return ret; -} - -bool OrtInferCore::PreProcess(std::shared_ptr buffer) -{ - // 获取内存缓存 - CHECK_STATE(buffer != nullptr, "[ort core] PreProcess got WRONG input data format!"); - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[ort core] PreProcess got WRONG p_buf data format!"); - - OrtBlobBuffer &buf = *p_buf; - - for (const auto &p_name_shape : map_input_blob_name2shape_) - { - const auto &s_blob_name = p_name_shape.first; - const auto &max_blob_shape = p_name_shape.second; - const auto &dynamic_blob_shape = buf.map_blob_name2shape[s_blob_name]; - - auto mem_info = - Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtDeviceAllocator, OrtMemType::OrtMemTypeCPU); - - int64_t element_size = sizeof(float); - for (auto s : dynamic_blob_shape) - { - element_size *= s; - } - - buf.map_type2tensors[BlobType::kINPUT].push_back( - Ort::Value::CreateTensor(mem_info, buf.outer_map_blob2ptr[s_blob_name].first, element_size, - dynamic_blob_shape.data(), dynamic_blob_shape.size(), - ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)); - } - - for (const auto &p_name_shape : map_output_blob_name2shape_) - { - const auto &s_blob_name = p_name_shape.first; - const auto &max_blob_shape = p_name_shape.second; - const auto &dynamic_blob_shape = buf.map_blob_name2shape[s_blob_name]; - - auto mem_info = - Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtDeviceAllocator, OrtMemType::OrtMemTypeCPU); - - int64_t element_size = sizeof(float); - for (auto s : dynamic_blob_shape) - { - element_size *= s; - } - - buf.map_type2tensors[BlobType::kOUTPUT].push_back( - Ort::Value::CreateTensor(mem_info, buf.outer_map_blob2ptr[s_blob_name].first, element_size, - dynamic_blob_shape.data(), dynamic_blob_shape.size(), - ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)); - } - return true; -} - -bool OrtInferCore::Inference(std::shared_ptr buffer) -{ - // 获取内存缓存 - CHECK_STATE(buffer != nullptr, "[ort core] Inference got WRONG input data format!"); - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[ort core] Inference got WRONG p_buf data format!"); - - OrtBlobBuffer &buf = *p_buf; - - // 构造推理接口参数 - std::vector input_blobs_name; - std::vector output_blobs_name; - for (const auto &p_name_shape : map_input_blob_name2shape_) - { - input_blobs_name.push_back(p_name_shape.first.c_str()); - } - for (const auto &p_name_shape : map_output_blob_name2shape_) - { - output_blobs_name.push_back(p_name_shape.first.c_str()); - } - - // 执行推理 - ort_session_->Run(Ort::RunOptions{nullptr}, input_blobs_name.data(), - buf.map_type2tensors[BlobType::kINPUT].data(), input_blobs_name.size(), - output_blobs_name.data(), buf.map_type2tensors[BlobType::kOUTPUT].data(), - output_blobs_name.size()); - - return true; -} - -bool OrtInferCore::PostProcess(std::shared_ptr buffer) -{ - return true; -} - -std::shared_ptr CreateOrtInferCore( - const std::string onnx_path, - const std::unordered_map> &input_blobs_shape, - const std::unordered_map> &output_blobs_shape, - const int num_threads) -{ - return std::make_shared(onnx_path, input_blobs_shape, output_blobs_shape, - num_threads); -} - -} // namespace inference_core \ No newline at end of file diff --git a/inference_core/ort_core/src/ort_core_factory.cpp b/inference_core/ort_core/src/ort_core_factory.cpp deleted file mode 100644 index ccc901c..0000000 --- a/inference_core/ort_core/src/ort_core_factory.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-12-02 19:43:23 - * @LastEditTime: 2024-12-02 19:50:55 - * @FilePath: /easy_deploy/inference_core/rknn_core/src/rknn_core_factory.cpp - */ -#include "ort_core/ort_core.h" - -namespace inference_core { - -struct OrtInferCoreParams { - std::string onnx_path; - std::unordered_map> input_blobs_shape; - std::unordered_map> output_blobs_shape; - int num_threads; -}; - -class OrtInferCoreFactory : public BaseInferCoreFactory { -public: - OrtInferCoreFactory(const OrtInferCoreParams ¶ms) : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateOrtInferCore(params_.onnx_path, params_.input_blobs_shape, - params_.output_blobs_shape, params_.num_threads); - } - -private: - const OrtInferCoreParams params_; -}; - -std::shared_ptr CreateOrtInferCoreFactory( - const std::string onnx_path, - const std::unordered_map> &input_blobs_shape, - const std::unordered_map> &output_blobs_shape, - const int num_threads) -{ - OrtInferCoreParams params; - params.onnx_path = onnx_path; - params.input_blobs_shape = input_blobs_shape; - params.output_blobs_shape = output_blobs_shape; - params.num_threads = num_threads; - - return std::make_shared(params); -} - -} // namespace inference_core \ No newline at end of file diff --git a/inference_core/rknn_core/CMakeLists.txt b/inference_core/rknn_core/CMakeLists.txt deleted file mode 100644 index 7112bb9..0000000 --- a/inference_core/rknn_core/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -cmake_minimum_required(VERSION 3.0.2) -project(rknn_core) - -add_compile_options(-std=c++17) -add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) -set(CMAKE_CXX_STANDARD 17) - -find_package(glog REQUIRED) - -set(source_file src/rknn_core.cpp - src/rknn_core_factory.cpp) - -add_library(${PROJECT_NAME} SHARED ${source_file}) - -include_directories( - include -) - -target_link_libraries(${PROJECT_NAME} PUBLIC - glog::glog - deploy_core - rknnrt -) - -install(TARGETS ${PROJECT_NAME} - LIBRARY DESTINATION lib) - -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file diff --git a/inference_core/rknn_core/include/rknn_core/rknn_core.h b/inference_core/rknn_core/include/rknn_core/rknn_core.h deleted file mode 100644 index 66e93f5..0000000 --- a/inference_core/rknn_core/include/rknn_core/rknn_core.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:05 - * @LastEditTime: 2024-12-02 19:43:10 - * @FilePath: /easy_deploy/inference_core/rknn_core/include/rknn_core/rknn_core.h - */ -#ifndef __EASY_DEPLOY_INFERENCE_CORE_RKNN_CORE_H -#define __EASY_DEPLOY_INFERENCE_CORE_RKNN_CORE_H - -#include "deploy_core/base_infer_core.h" - -namespace inference_core { - -enum RknnInputTensorType { - RK_UINT8, - RK_INT8, - RK_FLOAT16, - RK_FLOAT32, - RK_UINT32, - RK_INT32, - RK_INT64 -}; - -std::shared_ptr CreateRknnInferCore( - std::string model_path, - const std::unordered_map &map_blob_type = {}, - const int mem_buf_size = 5, - const int parallel_ctx_num = 1); - -std::shared_ptr CreateRknnInferCoreFactory( - std::string model_path, - const std::unordered_map &map_blob_type = {}, - const int mem_buf_size = 5, - const int parallel_ctx_num = 1); - -} // namespace inference_core - -#endif \ No newline at end of file diff --git a/inference_core/rknn_core/src/rknn_core.cpp b/inference_core/rknn_core/src/rknn_core.cpp deleted file mode 100644 index 1c7554b..0000000 --- a/inference_core/rknn_core/src/rknn_core.cpp +++ /dev/null @@ -1,535 +0,0 @@ -#include "rknn_core/rknn_core.h" - -#include - -#include - -namespace inference_core { - -static std::unordered_map map_type_my2rk{ - {RknnInputTensorType::RK_UINT8, RKNN_TENSOR_UINT8}, - {RknnInputTensorType::RK_INT8, RKNN_TENSOR_INT8}, - {RknnInputTensorType::RK_FLOAT16, RKNN_TENSOR_FLOAT16}, - {RknnInputTensorType::RK_FLOAT32, RKNN_TENSOR_FLOAT32}, - {RknnInputTensorType::RK_UINT32, RKNN_TENSOR_UINT32}, - {RknnInputTensorType::RK_INT32, RKNN_TENSOR_INT32}, - {RknnInputTensorType::RK_INT64, RKNN_TENSOR_INT64}, -}; - -static std::unordered_map map_rknn_type2size_{ - {RKNN_TENSOR_INT8, 1}, {RKNN_TENSOR_UINT8, 1}, {RKNN_TENSOR_FLOAT16, 4}, - {RKNN_TENSOR_FLOAT32, 4}, {RKNN_TENSOR_INT32, 4}, {RKNN_TENSOR_UINT32, 4}, - {RKNN_TENSOR_INT64, 8}}; - -static std::unordered_map map_rknn_type2type{ - {RKNN_TENSOR_INT8, RKNN_TENSOR_UINT8}, {RKNN_TENSOR_UINT8, RKNN_TENSOR_UINT8}, - {RKNN_TENSOR_FLOAT16, RKNN_TENSOR_FLOAT32}, {RKNN_TENSOR_FLOAT32, RKNN_TENSOR_FLOAT32}, - {RKNN_TENSOR_INT32, RKNN_TENSOR_INT32}, {RKNN_TENSOR_UINT32, RKNN_TENSOR_UINT32}, - {RKNN_TENSOR_INT64, RKNN_TENSOR_INT64}}; - -class RknnBlobBuffer : public IBlobsBuffer { -public: - std::pair GetOuterBlobBuffer(const std::string &blob_name) noexcept override - { - if (outer_map_blob2ptr.find(blob_name) == outer_map_blob2ptr.end()) - { - LOG(ERROR) << "[RknnBlobBuffer] `GetOuterBlobBuffer` Got invalid `blob_name`: " << blob_name; - return {nullptr, UNKOWN}; - } - return outer_map_blob2ptr[blob_name]; - } - - bool SetBlobBuffer(const std::string &blob_name, - void *data_ptr, - DataLocation location) noexcept override - { - if (inner_map_blob2ptr.find(blob_name) == inner_map_blob2ptr.end()) - { - LOG(ERROR) << "[RknnBlobBuffer] `SetBlobBuffer` Got invalid `blob_name`: " << blob_name; - return false; - } - outer_map_blob2ptr[blob_name] = {data_ptr, location}; - return true; - } - - bool SetBlobBuffer(const std::string &blob_name, DataLocation location) noexcept override - { - if (inner_map_blob2ptr.find(blob_name) == inner_map_blob2ptr.end()) - { - LOG(ERROR) << "[RknnBlobBuffer] `SetBlobBuffer` Got invalid `blob_name`: " << blob_name; - return false; - } - outer_map_blob2ptr[blob_name] = {inner_map_blob2ptr[blob_name], location}; - return true; - } - - bool SetBlobShape(const std::string &blob_name, - const std::vector &shape) noexcept override - { - LOG(WARNING) << "[RknnBlobBuffer] `SetBlobShape` dynamic input shape not supported!!!"; - return false; - } - - const std::vector &GetBlobShape(const std::string &blob_name) const noexcept override - { - if (map_blob_name2shape.find(blob_name) == map_blob_name2shape.end()) - { - LOG(ERROR) << "[RknnBlobBuffer] `GetBlobShape` Got invalid `blob_name`: " << blob_name; - static std::vector empty_shape; - return empty_shape; - } - return map_blob_name2shape.at(blob_name); - } - - size_t Size() const noexcept override - { - return outer_map_blob2ptr.size(); - } - - void Release() noexcept override - { - for (const auto &p_name_ptr : input_blobs_ptr) - { - if (p_name_ptr.second != nullptr) - { - delete[] p_name_ptr.second; - } - } - for (const auto &p_name_ptr : output_blobs_ptr) - { - if (p_name_ptr.second != nullptr) - { - delete[] p_name_ptr.second; - } - } - outer_map_blob2ptr.clear(); - inner_map_blob2ptr.clear(); - input_blobs_ptr.clear(); - output_blobs_ptr.clear(); - } - - void Reset() noexcept override - { - for (const auto &p_name_ptr : inner_map_blob2ptr) - { - outer_map_blob2ptr[p_name_ptr.first] = {p_name_ptr.second, DataLocation::HOST}; - } - } - - ~RknnBlobBuffer() override - { - Release(); - } - // - RknnBlobBuffer() = default; - RknnBlobBuffer(const RknnBlobBuffer &) = delete; - RknnBlobBuffer &operator=(const RknnBlobBuffer &) = delete; - - // - std::unordered_map> outer_map_blob2ptr; - std::unordered_map inner_map_blob2ptr; - - // - std::unordered_map input_blobs_ptr; - std::unordered_map output_blobs_ptr; - - // - std::vector device_buffer_input; - std::vector device_buffer_output; - - // - std::unordered_map> map_blob_name2shape; - - // - std::future async_infer_handle_; -}; - -class RknnInferCore : public BaseInferCore { -public: - RknnInferCore(std::string model_path, - const std::unordered_map &map_blob_type, - const int mem_buf_size = 5, - const int parallel_ctx_num = 1); - - ~RknnInferCore() override; - - InferCoreType GetType() - { - return InferCoreType::RKNN; - } - - std::string GetName() - { - return "rknn_core"; - } - -private: - bool PreProcess(std::shared_ptr buffer) override; - - bool Inference(std::shared_ptr buffer) override; - - bool PostProcess(std::shared_ptr buffer) override; - -private: - std::shared_ptr AllocBlobsBuffer() override; - - std::shared_ptr _AllocBlobsBuffer(); - - size_t ReadModelFromFile(const std::string &model_path, void **model_data); - - void ResolveModelInformation( - const std::unordered_map &map_blob_type); - -private: - // - std::vector rknn_ctx_parallel_; - // - BlockQueue bq_ctx_; - - // - int blob_input_number_; - int blob_output_number_; - - std::vector blob_attr_input_; - std::vector blob_attr_output_; - - std::unordered_map> map_input_blob_name2shape_; - std::unordered_map> map_output_blob_name2shape_; - std::vector blob_element_size_input_; - std::vector blob_element_size_output_; - std::vector blob_tensor_type_input_; -}; - -RknnInferCore::RknnInferCore( - std::string model_path, - const std::unordered_map &map_blob_type, - const int mem_buf_size, - const int parallel_ctx_num) - : bq_ctx_(parallel_ctx_num) -{ - if (parallel_ctx_num <= 0) - { - throw std::invalid_argument("[rknn core] Got Invalid ctx_num: " + - std::to_string(parallel_ctx_num)); - } - - void *model_data = nullptr; - size_t model_data_byte_size = ReadModelFromFile(model_path, &model_data); - if (model_data == nullptr) - { - throw std::runtime_error("[rknn_core] Failed to read model from file: " + model_path); - } - LOG(INFO) << "[rknn core] initilize using " << parallel_ctx_num << " ctx instances"; - rknn_ctx_parallel_.resize(parallel_ctx_num); - for (int i = 0; i < parallel_ctx_num; ++i) - { - if (rknn_init(&rknn_ctx_parallel_[i], model_data, model_data_byte_size, 0, NULL) != RKNN_SUCC) - { - throw std::runtime_error("[rknn_core] Failed to init rknn_ctx [ " + std::to_string(i) + " ]"); - } - bq_ctx_.BlockPush(i); - } - - rknn_sdk_version version; - auto ret = - rknn_query(rknn_ctx_parallel_[0], RKNN_QUERY_SDK_VERSION, &version, sizeof(rknn_sdk_version)); - if (ret < 0) - { - LOG(ERROR) << "[rknn core] Failed to get rknn sdk version info!!!"; - } else - { - LOG(INFO) << "sdk version: " << version.api_version - << ", driver version: " << version.drv_version; - } - - free(model_data); - - ResolveModelInformation(map_blob_type); - - BaseInferCore::Init(mem_buf_size); -} - -size_t RknnInferCore::ReadModelFromFile(const std::string &model_path, void **model_data) -{ - FILE *fp = fopen(model_path.c_str(), "rb"); - if (fp == NULL) - { - printf("fopen %s fail!\n", model_path.c_str()); - return -1; - } - fseek(fp, 0, SEEK_END); - size_t file_size = ftell(fp); - char *data = (char *)malloc(file_size + 1); - data[file_size] = 0; - fseek(fp, 0, SEEK_SET); - if (file_size != fread(data, 1, file_size, fp)) - { - printf("fread %s fail!\n", model_path.c_str()); - free(data); - fclose(fp); - return -1; - } - if (fp) - { - fclose(fp); - } - *model_data = data; - return file_size; -} - -RknnInferCore::~RknnInferCore() -{ - //////////////////////////// IMPORTANT ///////////////////////////////// - for (size_t i = 0; i < rknn_ctx_parallel_.size(); ++i) - { - bq_ctx_.Take(); - } - - for (auto &rknn_context : rknn_ctx_parallel_) - { - if (rknn_destroy(rknn_context) != RKNN_SUCC) - { - LOG(ERROR) << "[rknn core] In deconstructor destroy rknn ctx failed!!!"; - } - } - rknn_ctx_parallel_.clear(); -} - -std::shared_ptr RknnInferCore::AllocBlobsBuffer() -{ - return _AllocBlobsBuffer(); -} - -void RknnInferCore::ResolveModelInformation( - const std::unordered_map &map_blob_type) -{ - rknn_input_output_num rknn_io_num; - if (rknn_query(rknn_ctx_parallel_[0], RKNN_QUERY_IN_OUT_NUM, &rknn_io_num, sizeof(rknn_io_num)) != - RKNN_SUCC) - { - throw std::runtime_error("[rknn core] Failed to execute in_out_num `rknn_query`"); - } - LOG(INFO) << "model input blob num: " << rknn_io_num.n_input - << "\toutput blob num: " << rknn_io_num.n_output; - - blob_input_number_ = rknn_io_num.n_input; - blob_output_number_ = rknn_io_num.n_output; - blob_element_size_input_.resize(blob_input_number_); - blob_element_size_output_.resize(blob_output_number_); - blob_attr_input_.resize(blob_input_number_); - blob_attr_output_.resize(blob_output_number_); - - // input blob - blob_tensor_type_input_.resize(blob_input_number_); - for (int i = 0; i < blob_input_number_; ++i) - { - blob_attr_input_[i].index = i; - if (rknn_query(rknn_ctx_parallel_[0], RKNN_QUERY_INPUT_ATTR, &(blob_attr_input_[i]), - sizeof(rknn_tensor_attr)) != RKNN_SUCC) - { - throw std::runtime_error("[rknn core] Failed to execute input `rknn_query`"); - } - const std::string s_blob_name = blob_attr_input_[i].name; - // - rknn_tensor_type blob_type; - if (map_blob_type.find(s_blob_name) != map_blob_type.end()) - { - blob_type = map_type_my2rk[map_blob_type.at(s_blob_name)]; - } else - { - blob_type = blob_attr_input_[i].type; - } - - if (map_rknn_type2size_.find(blob_type) == map_rknn_type2size_.end()) - { - LOG(ERROR) << "[rknn core] blob_name: " << s_blob_name << ", blob_type : " << blob_type - << " NOT FOUND in `map_rknn_type2size_`"; - throw std::runtime_error("[rknn core] Failed to resolve model information!!!"); - } - blob_tensor_type_input_[i] = map_rknn_type2type[blob_type]; - const int blob_type_byte_size = map_rknn_type2size_[blob_type]; - - std::vector blob_shape; - size_t blob_element_size = blob_type_byte_size; - std::string s_blob_info = s_blob_name; - for (size_t j = 0; j < blob_attr_input_[i].n_dims; ++j) - { - s_blob_info += "\t" + std::to_string(blob_attr_input_[i].dims[j]); - blob_element_size *= blob_attr_input_[i].dims[j]; - blob_shape.push_back(blob_attr_input_[i].dims[j]); - } - LOG(INFO) << s_blob_info; - LOG(INFO) << "blob fmt: " << get_format_string(blob_attr_input_[i].fmt) - << ", type: " << get_type_string(blob_tensor_type_input_[i]); - map_input_blob_name2shape_[s_blob_name] = blob_shape; - blob_element_size_input_[i] = blob_element_size; - } - - // output blob - for (int i = 0; i < blob_output_number_; ++i) - { - blob_attr_output_[i].index = i; - if (rknn_query(rknn_ctx_parallel_[0], RKNN_QUERY_OUTPUT_ATTR, &(blob_attr_output_[i]), - sizeof(rknn_tensor_attr)) != RKNN_SUCC) - { - throw std::runtime_error("[rknn core] Failed to execute output `rknn_query`"); - } - const std::string s_blob_name = blob_attr_output_[i].name; - std::vector blob_shape; - size_t blob_element_size = 1; - std::string s_blob_info = blob_attr_output_[i].name; - for (size_t j = 0; j < blob_attr_output_[i].n_dims; ++j) - { - s_blob_info += "\t" + std::to_string(blob_attr_output_[i].dims[j]); - blob_element_size *= blob_attr_output_[i].dims[j]; - blob_shape.push_back(blob_attr_output_[i].dims[j]); - } - LOG(INFO) << s_blob_info; - LOG(INFO) << "blob fmt: " << blob_attr_output_[i].fmt - << ", type: " << blob_attr_output_[i].type; - - map_output_blob_name2shape_[s_blob_name] = blob_shape; - blob_element_size_output_[i] = blob_element_size; - } -} - -std::shared_ptr RknnInferCore::_AllocBlobsBuffer() -{ - auto ret = std::make_shared(); - - ret->device_buffer_input.resize(blob_input_number_); - for (int i = 0; i < blob_input_number_; ++i) - { - const std::string s_blob_name = blob_attr_input_[i].name; - int64_t element_size = blob_element_size_input_[i]; - - u_char *buf = new u_char[element_size]; - ret->input_blobs_ptr.insert({s_blob_name, buf}); - ret->outer_map_blob2ptr.insert({s_blob_name, {buf, DataLocation::HOST}}); - ret->inner_map_blob2ptr.insert({s_blob_name, buf}); - - ret->map_blob_name2shape.insert({s_blob_name, map_input_blob_name2shape_[s_blob_name]}); - - // - ret->device_buffer_input[i].index = i; - ret->device_buffer_input[i].fmt = blob_attr_input_[i].fmt; - ret->device_buffer_input[i].type = blob_tensor_type_input_[i]; - ret->device_buffer_input[i].size = element_size; - } - - ret->device_buffer_output.resize(blob_output_number_); - for (int i = 0; i < blob_output_number_; ++i) - { - const std::string s_blob_name = blob_attr_output_[i].name; - int64_t element_size = blob_element_size_output_[i]; - - float *out_buf = new float[element_size]; - ret->output_blobs_ptr.insert({s_blob_name, out_buf}); - - ret->outer_map_blob2ptr.insert({s_blob_name, {out_buf, DataLocation::HOST}}); - ret->inner_map_blob2ptr.insert({s_blob_name, out_buf}); - - ret->map_blob_name2shape.insert({s_blob_name, map_output_blob_name2shape_[s_blob_name]}); - - // - ret->device_buffer_output[i].index = i; - ret->device_buffer_output[i].is_prealloc = true; - ret->device_buffer_output[i].want_float = true; - ret->device_buffer_output[i].size = element_size * sizeof(float); - } - - return ret; -} - -bool RknnInferCore::PreProcess(std::shared_ptr buffer) -{ - // - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[rknn core] PreProcess got wrong input data format!"); - - RknnBlobBuffer &buf = *p_buf; - - for (int i = 0; i < blob_input_number_; ++i) - { - const std::string s_blob_name = blob_attr_input_[i].name; - - void *outer_ptr = buf.outer_map_blob2ptr[s_blob_name].first; - buf.device_buffer_input[i].buf = outer_ptr; - } - - for (int i = 0; i < blob_output_number_; ++i) - { - const std::string s_blob_name = blob_attr_output_[i].name; - - void *ptr = buf.outer_map_blob2ptr[s_blob_name].first; - buf.device_buffer_output[i].buf = ptr; - } - - return true; -} - -#define RKNN_CHECK_STATE(state, hint) \ - { \ - if (!(state)) \ - { \ - LOG(ERROR) << (hint); \ - bq_ctx_.BlockPush(index); \ - return false; \ - } \ - } - -bool RknnInferCore::Inference(std::shared_ptr buffer) -{ - // - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[rknn core] Inference got wrong input data format!"); - - auto func_async_infer = [this, p_buf](int index) -> bool { - // - RKNN_CHECK_STATE(rknn_inputs_set(rknn_ctx_parallel_[index], blob_input_number_, - p_buf->device_buffer_input.data()) == RKNN_SUCC, - "[rknn core] Inference `rknn_inputs_set` execute failed!!!"); - RKNN_CHECK_STATE(rknn_run(rknn_ctx_parallel_[index], nullptr) == RKNN_SUCC, - "[rknn core] Inference `rknn_run` execute failed!!!"); - RKNN_CHECK_STATE(rknn_outputs_get(rknn_ctx_parallel_[index], blob_output_number_, - p_buf->device_buffer_output.data(), nullptr) == RKNN_SUCC, - "[rknn core] Inference `rknn_outputs_get` execute failed!!!"); - - RKNN_CHECK_STATE(rknn_outputs_release(rknn_ctx_parallel_[index], blob_output_number_, - p_buf->device_buffer_output.data()) == RKNN_SUCC, - "[rknn core] Inference `rknn_outputs_release failed!!!"); - - bq_ctx_.BlockPush(index); - return true; - }; - auto ctx = bq_ctx_.Take(); - if (!ctx.has_value()) - { - return false; - } - p_buf->async_infer_handle_ = std::async(func_async_infer, ctx.value()); - - return true; -} - -bool RknnInferCore::PostProcess(std::shared_ptr buffer) -{ - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[rknn core] PostProcess got wrong input data format!"); - - CHECK_STATE(p_buf->async_infer_handle_.get(), - "[rknn core] async infer handle got `false` from async process!"); - - return true; -} - -std::shared_ptr CreateRknnInferCore( - std::string model_path, - const std::unordered_map &map_blob_type, - const int mem_buf_size, - const int parallel_ctx_num) -{ - return std::make_shared(model_path, map_blob_type, mem_buf_size, parallel_ctx_num); -} - -} // namespace inference_core \ No newline at end of file diff --git a/inference_core/rknn_core/src/rknn_core_factory.cpp b/inference_core/rknn_core/src/rknn_core_factory.cpp deleted file mode 100644 index 3d80621..0000000 --- a/inference_core/rknn_core/src/rknn_core_factory.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-12-02 19:43:23 - * @LastEditTime: 2024-12-02 19:50:55 - * @FilePath: /easy_deploy/inference_core/rknn_core/src/rknn_core_factory.cpp - */ -#include "rknn_core/rknn_core.h" - -namespace inference_core { - -struct RknnInferCoreParams { - std::string model_path; - std::unordered_map map_blob_type; - int mem_buf_size; - int parallel_ctx_num; -}; - -class RknnInferCoreFactory : public BaseInferCoreFactory { -public: - RknnInferCoreFactory(const RknnInferCoreParams ¶ms) : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateRknnInferCore(params_.model_path, params_.map_blob_type, params_.mem_buf_size, - params_.parallel_ctx_num); - } - -private: - const RknnInferCoreParams params_; -}; - -std::shared_ptr CreateRknnInferCoreFactory( - std::string model_path, - const std::unordered_map &map_blob_type, - const int mem_buf_size, - const int parallel_ctx_num) -{ - RknnInferCoreParams params; - params.model_path = model_path; - params.map_blob_type = map_blob_type; - params.mem_buf_size = mem_buf_size; - params.parallel_ctx_num = parallel_ctx_num; - - return std::make_shared(params); -} - -} // namespace inference_core \ No newline at end of file diff --git a/inference_core/trt_core/CMakeLists.txt b/inference_core/trt_core/CMakeLists.txt deleted file mode 100644 index 9b08824..0000000 --- a/inference_core/trt_core/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(trt_core) - - -add_compile_options(-std=c++17) -add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) -set(CMAKE_CXX_STANDARD 17) - -set(CMAKE_THREAD_LIBS_INIT "-lpthread") -set(CMAKE_HAVE_THREADS_LIBRARY 1) -set(CMAKE_USE_WIN32_THREADS_INIT 0) -set(CMAKE_USE_PTHREADS_INIT 1) -set(THREADS_PREFER_PTHREAD_FLAG ON) - - -find_package(CUDA REQUIRED) -find_package(glog REQUIRED) - -set(source_file src/trt_core.cpp - src/trt_core_factory.cpp) - - -include_directories( - include - ${CUDA_INCLUDE_DIRS} -) - -add_library(${PROJECT_NAME} SHARED ${source_file}) - - -target_link_libraries(${PROJECT_NAME} PUBLIC - ${CUDA_LIBRARIES} - nvinfer - nvonnxparser - deploy_core -) - -install(TARGETS ${PROJECT_NAME} - LIBRARY DESTINATION lib) - -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file diff --git a/inference_core/trt_core/include/trt_core/trt_core.h b/inference_core/trt_core/include/trt_core/trt_core.h deleted file mode 100644 index 43b4e70..0000000 --- a/inference_core/trt_core/include/trt_core/trt_core.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-19 18:33:05 - * @LastEditTime: 2024-12-02 19:34:00 - * @FilePath: /easy_deploy/inference_core/trt_core/include/trt_core/trt_core.h - */ -#ifndef __EASY_DEPLOY_TRT_CORE_H -#define __EASY_DEPLOY_TRT_CORE_H - -#include -#include "deploy_core/base_infer_core.h" - -namespace inference_core { - -/** - * @brief Construct `TrtInferCore` by providing tensorrt engine file path, max(default) blobs - * shape and blob buffer pool size (defualt=5). If your model is parsed from a onnx model with - * dynamic blob shape (e.g. blob_dim=-1), a mapping of blob_name and blob_shape should be provided - * to help `TrtInferCore` alloc a apposite size blob buffer. - * - * @param engine_path Tensorrt engine file path. - * @param blobs_shape Mapping of blob_name and blob_shape. - * @param mem_buf_size Size of buffer pool. - */ -std::shared_ptr CreateTrtInferCore( - std::string model_path, - const std::unordered_map> &input_blobs_shape = {}, - const std::unordered_map> &output_blobs_shape = {}, - const int mem_buf_size = 5); - -std::shared_ptr CreateTrtInferCoreFactory( - std::string model_path, - const std::unordered_map> &input_blobs_shape = {}, - const std::unordered_map> &output_blobs_shape = {}, - const int mem_buf_size = 5); - -} // namespace inference_core - -#endif diff --git a/inference_core/trt_core/src/trt_blob_buffer.hpp b/inference_core/trt_core/src/trt_blob_buffer.hpp deleted file mode 100644 index 528fc1b..0000000 --- a/inference_core/trt_core/src/trt_blob_buffer.hpp +++ /dev/null @@ -1,260 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-11-25 14:00:38 - * @LastEditTime: 2024-11-26 09:31:01 - * @FilePath: /EasyDeploy/inference_core/trt_core/src/trt_blob_buffer.hpp - */ -#ifndef __EASY_DEPLOY_TRT_BLOB_BUFFER_H -#define __EASY_DEPLOY_TRT_BLOB_BUFFER_H - -#include - -#include "deploy_core/blob_buffer.h" - -namespace inference_core { - -template -inline Type CumVector(const std::vector &vec) -{ - Type ret = 1; - for (const auto &nn : vec) - { - ret *= nn; - } - - return ret; -} - -template -inline std::string VisualVec(const std::vector &vec) -{ - std::string ret; - for (const auto &v : vec) - { - ret += std::to_string(v) + " "; - } - return ret; -} - -class TrtBlobBuffer : public IBlobsBuffer { -public: - /** - * @brief Overrided from `IBlobsBuffer`, provide the buffer ptr which is used as - * input data of tensorrt inference engine. It depends on `SetBlobBuffer` method. - * - * @param blob_name The blob_name of model. - * @return std::pair . Will return {nullptr, UNKOWN} if `blob_name` - * does not match. - */ - std::pair GetOuterBlobBuffer(const std::string &blob_name) noexcept override - { - if (outer_map_blob2ptr_.find(blob_name) == outer_map_blob2ptr_.end()) - { - LOG(ERROR) << "[TrtBlobBuffer] `GetOuterBlobBuffer` Got invalid `blob_name`: " << blob_name; - return {nullptr, UNKOWN}; - } - return outer_map_blob2ptr_[blob_name]; - } - - /** - * @brief Overrided from `IBlobsBuffer`, users could make tensorrt inference core use customed - * data buffer to deploy inference. `data_ptr` and `location` are required to modify inner - * mapping. - * - * @param blob_name The blob_name of model. - * @param data_ptr Customed data buffer ptr. - * @param location Where the data buffer locates. - * @return true Successfully set customed data buffer. - * @return false Will return false if `blob_name` does not match, or `data_ptr` is not valid. - */ - bool SetBlobBuffer(const std::string &blob_name, - void *data_ptr, - DataLocation location) noexcept override - { - if (outer_map_blob2ptr_.find(blob_name) == outer_map_blob2ptr_.end()) - { - LOG(ERROR) << "[TrtBlobBuffer] `SetBlobBuffer` Got invalid `blob_name`: " << blob_name; - return false; - } - - if (location == DataLocation::HOST) - { - outer_map_blob2ptr_[blob_name] = {inner_map_host_blob2ptr_[blob_name], location}; - } else - { - cudaPointerAttributes attr; - cudaError_t status = cudaPointerGetAttributes(&attr, data_ptr); - if (status != cudaSuccess || attr.type != cudaMemoryType::cudaMemoryTypeDevice) - { - LOG(ERROR) << "[TrtBlobBuffer] `SetBlobBuffer` Got " - "invalid `data_ptr` " - "which should be " - << "allocated by `cudaMalloc`, but it " - "is NOT !!!"; - return false; - } - outer_map_blob2ptr_[blob_name] = {data_ptr, location}; - } - return true; - } - - /** - * @brief Overrided from `IBlobsBuffer`, set the default buffer ptr used in tensorrt - * engine inference stage. After calling `SetBlobBuffer`, `GetOuterBlobBuffer` could - * get certain buffer ptr on `location`. - * - * @param blob_name The blob_name of model. - * @param location Which buffer to use in inference stage. - * @return true Successfully set blob buffer location. - * @return false Will return false if blob_name does not match. - */ - bool SetBlobBuffer(const std::string &blob_name, DataLocation location) noexcept override - { - if (outer_map_blob2ptr_.find(blob_name) == outer_map_blob2ptr_.end()) - { - LOG(ERROR) << "[TrtBlobBuffer] `SetBlobBuffer` Got invalid `blob_name`: " << blob_name; - return false; - } - - outer_map_blob2ptr_[blob_name] = { - (location == DataLocation::HOST ? inner_map_host_blob2ptr_[blob_name] - : inner_map_device_blob2ptr_[blob_name]), - location}; - - return true; - } - - /** - * @brief Overrided from `IBlobsBuffer`, set the dynamic blob shape while tensorrt engine - * doing inference. Note that `shape` should not has more element number than origin_shape - * which is determined by model build stage. Dynamic shape suportted tensorrt inference - * core should constructed by customed max blob shape params. There should not be `0` or any - * negative values in `shape` vec. - * - * @note Please make sure your model supportes dynamic blob shape. Otherwise, it will leads - * to unknown results. - * - * @param blob_name The blob_name of model. - * @param shape Dynamic blob shape. - * @return true - * @return false Will return false if `shape` is not valid or `blob_name` does not match. - */ - bool SetBlobShape(const std::string &blob_name, - const std::vector &shape) noexcept override - { - if (map_blob_name2shape_.find(blob_name) == map_blob_name2shape_.end()) - { - LOG(ERROR) << "[TrtBlobBuffer] `SetBlobShape` Got invalid `blob_name`: " << blob_name; - return false; - } - const auto &origin_shape = map_blob_name2shape_[blob_name]; - const long long ori_element_count = CumVector(origin_shape); - const long long dyn_element_count = CumVector(shape); - if (origin_shape.size() != shape.size() || dyn_element_count > ori_element_count || - dyn_element_count < 0) - { - const std::string origin_shape_in_str = VisualVec(origin_shape); - const std::string shape_in_str = VisualVec(shape); - LOG(ERROR) << "[TrtBlobBuffer] `SetBlobShape` Got invalid `shape` input. " - << "`shape`: " << shape_in_str << "\t" - << "`origin_shape`: " << origin_shape_in_str; - return false; - } - map_blob_name2shape_[blob_name] = shape; - return true; - } - - /** - * @brief Overrided from `IBlobsBuffer`, provide default or dynamic blob shape. Default - * blob shape is defined while tensorrt inference core is built. Will return dynamic blob - * shape if `SetBlobShape` is called before `GetBlobShape`. - * - * @param blob_name The blob_name of model. - * @return const std::vector& . A const reference to blob shape recorded in buffer. - */ - const std::vector &GetBlobShape(const std::string &blob_name) const noexcept override - { - if (map_blob_name2shape_.find(blob_name) == map_blob_name2shape_.end()) - { - LOG(ERROR) << "[TrtBlobBuffer] `GetBlobShape` Got invalid `blob_name`: " << blob_name; - static std::vector empty_shape; - return empty_shape; - } - return map_blob_name2shape_.at(blob_name); - } - - /** - * @brief Overrided from `IBlobsBuffer`, provide number of blobs. - * - * @return size_t - */ - size_t Size() const noexcept override - { - return outer_map_blob2ptr_.size(); - } - - /** - * @brief Overrided from `IBlobsBuffer`, release the buffer instance. - * - */ - void Release() noexcept override - { - // release device buffer - for (void *ptr : device_blobs_buffer_) - { - if (ptr != nullptr) - cudaFree(ptr); - } - // release host buffer - for (void *ptr : host_blobs_buffer_) - { - if (ptr != nullptr) - delete[] reinterpret_cast(ptr); - } - device_blobs_buffer_.clear(); - host_blobs_buffer_.clear(); - } - - /** - * @brief Overrided from `IBlobsBuffer`, reset the buffer instance which will not - * release the buffer allocated. Mempool will call `Reset` after buffer instance is - * returned by user. - * - */ - void Reset() noexcept override - { - for (const auto &p_name_ptr : inner_map_host_blob2ptr_) - { - outer_map_blob2ptr_[p_name_ptr.first] = {p_name_ptr.second, DataLocation::HOST}; - } - } - - ~TrtBlobBuffer() - { - Release(); - } - // no copy - TrtBlobBuffer() = default; - TrtBlobBuffer(const TrtBlobBuffer &) = delete; - TrtBlobBuffer &operator=(const TrtBlobBuffer &) = delete; - - // mapping blob_name and buffer ptrs - std::unordered_map> outer_map_blob2ptr_; - std::unordered_map inner_map_device_blob2ptr_; - std::unordered_map inner_map_host_blob2ptr_; - - // buffer ptr vector, used while doing inference with tensorrt engine - std::vector buffer_input_core_; - - // maintain buffer ptrs. - std::vector device_blobs_buffer_; - std::vector host_blobs_buffer_; - - // mapping blob_name and dynamic blob shape - std::unordered_map> map_blob_name2shape_; -}; - -} // namespace inference_core - -#endif \ No newline at end of file diff --git a/inference_core/trt_core/src/trt_core.cpp b/inference_core/trt_core/src/trt_core.cpp deleted file mode 100644 index 4ad5fa8..0000000 --- a/inference_core/trt_core/src/trt_core.cpp +++ /dev/null @@ -1,475 +0,0 @@ -#include "trt_core/trt_core.h" - -// std -#include -#include -#include -#include - -// thirdparty -#include -#include -#include - -#include "trt_blob_buffer.hpp" - -namespace inference_core { - -class TensorrtLogger : public nvinfer1::ILogger { -public: - void log(Severity severity, const char *msg) noexcept override - { - if (severity == Severity::kINFO) - LOG(INFO) << "[Tensorrt] : " << msg; - else if (severity == Severity::kERROR) - LOG(ERROR) << "[Tensorrt] : " << msg; - else if (severity == Severity::kWARNING) - LOG(WARNING) << "[Tensorrt] : " << msg; - } -}; - -/** - * @brief `TrtInferCore` is derived from `BaseInferCore` and override the abstract methods - * of `BaseInferCore`. It wraps tensorrt engine loading and inference process. - * - */ -class TrtInferCore : public BaseInferCore { -public: - /** - * @brief Construct `TrtInferCore` by providing tensorrt engine file path and blob buffer - * pool size (defualt=5). This constructor does not need a map of blob_name and blob_shape, - * while it will resolve model information by it self. - * - * @warning This constructor only should be used if the blobs shape of input model is fixed. - * If you parse a model with dynamic blob shape, a exception will be thrown. - * - * @param engine_path Tensorrt engine file path. - * @param mem_buf_size Size of buffer pool. - */ - TrtInferCore(const std::string engine_path, const int mem_buf_size = 5); - - /** - * @brief Construct `TrtInferCore` by providing tensorrt engine file path, max(default) blobs - * shape and blob buffer pool size (defualt=5). If your model is parsed from a onnx model with - * dynamic blob shape (e.g. blob_dim=-1), a mapping of blob_name and blob_shape should be provided - * to help `TrtInferCore` alloc a apposite size blob buffer. - * - * @param engine_path Tensorrt engine file path. - * @param blobs_shape Mapping of blob_name and blob_shape. - * @param mem_buf_size Size of buffer pool. - */ - TrtInferCore(const std::string engine_path, - const std::unordered_map> &blobs_shape, - const int mem_buf_size = 5); - - /** - * @brief Overrided from `BaseInferCore`, construct a instance of `TrtBlobBuffer` and return - * the shared ptr of it. It is used by mem buffer pool in `BaseInferCore`, or users who wants - * to alloc a brand new buffer. - * - * @return std::shared_ptr - */ - std::shared_ptr AllocBlobsBuffer() override; - - /** - * @brief Overrided from `BaseInferCore`. The `PreProcess` stage of tensorrt inference. It - * prepares device buffers if user writes into host buffer derectly. - * - * @param buffer a common "pipeline" shared ptr. - * @return true - * @return false - */ - bool PreProcess(std::shared_ptr buffer) override; - - /** - * @brief Overrided from `BaseInferCore`. The `Inference` stage of tensorrt inference. - * - * @param buffer a common "pipeline" shared ptr. - * @return true - * @return false - */ - bool Inference(std::shared_ptr buffer) override; - - /** - * @brief Overrided from `BaseInferCore`. The `PostProcess` stage of tensorrt inference. - * It will prepare output host buffer if user needs the output of model be accessable on host. - * - * @param buffer a common "pipeline" shared ptr. - * @return true - * @return false - */ - bool PostProcess(std::shared_ptr buffer) override; - - ~TrtInferCore() override; - -private: - /** - * @brief Load the tensorrt engine file on `engine_path`. - * - * @param engine_path - */ - void LoadEngine(const std::string &engine_path); - - /** - * @brief Automatically resolve model information. - * - * @param blobs_shape - */ - void ResolveModelInformation(std::unordered_map> &blobs_shape); - -private: - // some members related to tensorrt - TensorrtLogger logger_{}; - std::unique_ptr runtime_{nullptr}; - std::unique_ptr engine_{nullptr}; - - /** - * @brief Due to tensorrt needs a unique inference context in every thread, we should maintain a - * mapping of thread_id and a ptr of tensorrt context. A context will be created when there is a - * new thread calls `Inference`. These contexts will be released when this `TrtInferCore` instance - * is released. - */ - std::unordered_map> - s_map_tid2context_; - std::mutex s_context_lck_; - - // cuda streams used in three stage. - cudaStream_t preproces_stream_, inference_stream_, postprocess_stream_; - - // some model information mapping - std::unordered_map> map_blob_name2shape_; - std::unordered_map map_input_blob_name2index_; - std::unordered_map map_output_blob_name2index_; - std::unordered_map map_blob_name2size_; -}; - -TrtInferCore::TrtInferCore(std::string engine_path, const int mem_buf_size) -{ - LoadEngine(engine_path); - ResolveModelInformation(map_blob_name2shape_); - - BaseInferCore::Init(mem_buf_size); - - cudaStreamCreate(&preproces_stream_); - cudaStreamCreate(&inference_stream_); - cudaStreamCreate(&postprocess_stream_); -} - -TrtInferCore::TrtInferCore(const std::string engine_path, - const std::unordered_map> &blobs_shape, - const int mem_buf_size) -{ - LoadEngine(engine_path); - map_blob_name2shape_ = blobs_shape; - ResolveModelInformation(map_blob_name2shape_); - - BaseInferCore::Init(mem_buf_size); - - cudaStreamCreate(&preproces_stream_); - cudaStreamCreate(&inference_stream_); - cudaStreamCreate(&postprocess_stream_); -} - -TrtInferCore::~TrtInferCore() -{ - BaseInferCore::Release(); -} - -void TrtInferCore::LoadEngine(const std::string &engine_path) -{ - std::ifstream file(engine_path, std::ios::binary); - if (!file.good()) - { - throw std::runtime_error("[TrtInferCore] Failed to read engine file!!!"); - } - - std::vector data; - - file.seekg(0, file.end); - const auto size = file.tellg(); - file.seekg(0, file.beg); - - data.resize(size); - file.read(data.data(), size); - - file.close(); - - runtime_.reset(nvinfer1::createInferRuntime(logger_)); - - engine_.reset(runtime_->deserializeCudaEngine(data.data(), data.size())); - if (engine_ == nullptr) - { - throw std::runtime_error("[TrtInferCore] Failed to create trt engine!!!"); - } - LOG(INFO) << "[TrtInferCore] created tensorrt engine and " - "context ! "; -} - -void TrtInferCore::ResolveModelInformation( - std::unordered_map> &blobs_shape) -{ - const int blob_number = engine_->getNbIOTensors(); - LOG(INFO) << "[TrtInferCore] model has " << blob_number << " blobs"; - CHECK(blob_number >= 2); - - bool resolve_blob_shape = blobs_shape.empty(); - - for (int i = 0; i < blob_number; ++i) - { - const char *blob_name = engine_->getIOTensorName(i); - nvinfer1::Dims dim = engine_->getTensorShape(blob_name); - - const std::string s_blob_name(blob_name); - if (engine_->getTensorIOMode(blob_name) == nvinfer1::TensorIOMode::kINPUT) - { - map_input_blob_name2index_.emplace(s_blob_name, i); - } else - { - map_output_blob_name2index_.emplace(s_blob_name, i); - } - - if (resolve_blob_shape) - { - blobs_shape[s_blob_name] = std::vector(); - for (int j = 0; j < dim.nbDims; ++j) - { - // 检查是否包含动态shape，自动解析暂不支持动态shape - if (dim.d[j] <= 0) - { - throw std::runtime_error("[TrtInferCore] unsupport blob dim:" + std::to_string(dim.d[j]) + - ", use explicit blob shape consturctor instead"); - } - blobs_shape[s_blob_name].push_back(dim.d[j]); - } - - std::string s_dim; - for (auto d : dim.d) - { - s_dim += std::to_string(d) + " "; - } - LOG(INFO) << "[TrtInferCore] blob name : " << blob_name << " dims : " << s_dim; - } - - size_t blob_byte_size = sizeof(float); - if (blobs_shape.find(s_blob_name) == blobs_shape.end()) - { - throw std::runtime_error("[TrtInferCore] blob name: " + s_blob_name + - " not found in provided blobs_shape map !!!"); - } - for (const int64_t d : blobs_shape[s_blob_name]) - { - blob_byte_size *= d; - } - - map_blob_name2size_[s_blob_name] = blob_byte_size; - } -} - -std::shared_ptr TrtInferCore::AllocBlobsBuffer() -{ - auto ret = std::make_shared(); - - const int blob_number = engine_->getNbIOTensors(); - CHECK(blob_number >= 2); - ret->device_blobs_buffer_.resize(blob_number); - ret->host_blobs_buffer_.resize(blob_number); - - for (int i = 0; i < blob_number; ++i) - { - const std::string s_blob_name = engine_->getIOTensorName(i); - int64_t blob_byte_size = sizeof(float); - const auto &blob_shape = map_blob_name2shape_[s_blob_name]; - for (const int64_t d : blob_shape) - { - blob_byte_size *= d; - } - - // alloc buffer memory - // on device - CHECK(cudaMalloc(&ret->device_blobs_buffer_[i], blob_byte_size) == cudaSuccess); - CHECK(cudaMemset(ret->device_blobs_buffer_[i], 0, blob_byte_size) == cudaSuccess); - CHECK(cudaDeviceSynchronize() == cudaSuccess); - // on host - ret->host_blobs_buffer_[i] = new u_char[blob_byte_size]; - - // maintain buffer ptr - ret->outer_map_blob2ptr_.emplace(s_blob_name, - std::pair{ret->host_blobs_buffer_[i], DataLocation::HOST}); - // mapping blob_name and buffer_ptr - ret->inner_map_device_blob2ptr_.emplace(s_blob_name, ret->device_blobs_buffer_[i]); - ret->inner_map_host_blob2ptr_.emplace(s_blob_name, ret->host_blobs_buffer_[i]); - - // mapping blob_name and default blob_shape - ret->map_blob_name2shape_.emplace(s_blob_name, blob_shape); - } - - // initialize the buffer ptr vector which will be used when tensorrt engine do inference. - ret->buffer_input_core_ = ret->device_blobs_buffer_; - - return ret; -} - -bool TrtInferCore::PreProcess(std::shared_ptr buffer) -{ - CHECK_STATE(buffer != nullptr, "[TrtInferCore] PreProcess got WRONG input data format!"); - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[TrtInferCore] PreProcess got WRONG p_buf data format!"); - - // Set the input buffer data - for (const auto &p_name_index : map_input_blob_name2index_) - { - const std::string &s_blob_name = p_name_index.first; - const int index = p_name_index.second; - - // Get the customed blob buffer data information, including data ptr and location. - const auto &p_ptr_loc = p_buf->GetOuterBlobBuffer(s_blob_name); - // Transport buffer data from host to device, if the customed blob data is on host. - if (p_ptr_loc.second == DataLocation::HOST) - { - p_buf->buffer_input_core_[index] = p_buf->inner_map_device_blob2ptr_[s_blob_name]; - cudaMemcpyAsync(p_buf->buffer_input_core_[index], p_ptr_loc.first, - map_blob_name2size_[s_blob_name], cudaMemcpyHostToDevice, preproces_stream_); - } else - { - p_buf->buffer_input_core_[index] = p_ptr_loc.first; - } - } - - // Set the output buffer data ptr. Allways use inner pre-allocated device buffer. - for (const auto &p_name_index : map_output_blob_name2index_) - { - const std::string &s_blob_name = p_name_index.first; - const int index = p_name_index.second; - p_buf->buffer_input_core_[index] = p_buf->inner_map_device_blob2ptr_[s_blob_name]; - } - - cudaStreamSynchronize(preproces_stream_); - - return true; -} - -bool TrtInferCore::Inference(std::shared_ptr buffer) -{ - // Create tensorrt context if this is the first time execution of this thread. - std::thread::id cur_thread_id = std::this_thread::get_id(); - if (s_map_tid2context_.find(cur_thread_id) == s_map_tid2context_.end()) - { - std::shared_ptr context{engine_->createExecutionContext()}; - { - std::unique_lock u_lck(s_context_lck_); - s_map_tid2context_.insert({cur_thread_id, context}); - } - } - auto context = s_map_tid2context_[cur_thread_id]; - - // Get buffer ptr - CHECK_STATE(buffer != nullptr, "[TrtInferCore] PreProcess got WRONG input data format!"); - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[TrtInferCore] PreProcess got WRONG p_buf data format!"); - - TrtBlobBuffer &buf = *p_buf; - - // Set dynamic blob shape - for (const auto &p_name_shape : buf.map_blob_name2shape_) - { - const auto &s_blob_name = p_name_shape.first; - const auto &v_shape = p_name_shape.second; - - if (engine_->getTensorIOMode(s_blob_name.c_str()) != nvinfer1::TensorIOMode::kINPUT) - { - continue; - } - - nvinfer1::Dims dynamic_dim; - dynamic_dim.nbDims = v_shape.size(); - for (size_t i = 0; i < v_shape.size(); ++i) - { - dynamic_dim.d[i] = v_shape[i]; - } - CHECK_STATE(context->setInputShape(s_blob_name.c_str(), dynamic_dim), - "[TrtInferCore] Inference execute `context->setInputShape` failed!!!"); - } - - // Do inference use `buf.buffer_input_core_` which is prepared by `PreProcess` stage. - CHECK_STATE(context->enqueueV2(buf.buffer_input_core_.data(), inference_stream_, nullptr), - "[TrtInferCore] Inference execute `context->enqueueV2` failed!!!"); - cudaStreamSynchronize(inference_stream_); - return true; -} - -bool TrtInferCore::PostProcess(std::shared_ptr buffer) -{ - CHECK_STATE(buffer != nullptr, "[TrtInferCore] PreProcess got WRONG input data format!"); - auto p_buf = std::dynamic_pointer_cast(buffer->GetInferBuffer()); - CHECK_STATE(p_buf != nullptr, "[TrtInferCore] PreProcess got WRONG p_buf data format!"); - - for (const auto &p_name_index : map_output_blob_name2index_) - { - const std::string &s_blob_name = p_name_index.first; - const int index = p_name_index.second; - const auto &p_ptr_loc = p_buf->GetOuterBlobBuffer(s_blob_name); - // Transport output buffer from device to host, if user needs host readable data. - if (p_ptr_loc.second == DataLocation::HOST) - { - cudaMemcpyAsync(p_ptr_loc.first, p_buf->buffer_input_core_[index], - map_blob_name2size_[s_blob_name], cudaMemcpyDeviceToHost, - postprocess_stream_); - } - // Transport output buffer from local device buffer to given device buffer. - else if (p_ptr_loc.first != p_buf->buffer_input_core_[index]) - { - cudaMemcpyAsync(p_ptr_loc.first, p_buf->buffer_input_core_[index], - map_blob_name2size_[s_blob_name], cudaMemcpyDeviceToDevice, - postprocess_stream_); - } - } - - cudaStreamSynchronize(postprocess_stream_); - return true; -} - -static bool FileSuffixCheck(const std::string &file_path, const std::string &suffix) -{ - const size_t mark = file_path.rfind('.'); - std::string suf; - return mark != file_path.npos && - (suf = file_path.substr(mark, file_path.size() - mark)) == suffix; -} - -std::shared_ptr CreateTrtInferCore(std::string model_path, const int mem_buf_size) -{ - if (!FileSuffixCheck(model_path, ".engine")) - { - throw std::invalid_argument("Trt infer core expects file end with `.engine`. But got " + - model_path + " instead"); - } - - return std::make_shared(model_path, mem_buf_size); -} - -std::shared_ptr CreateTrtInferCore( - std::string model_path, - const std::unordered_map> &input_blobs_shape, - const std::unordered_map> &output_blobs_shape, - const int mem_buf_size) -{ - if (!FileSuffixCheck(model_path, ".engine")) - { - throw std::invalid_argument("Trt infer core expects file end with `.engine`. But got " + - model_path + " instead"); - } - - std::unordered_map> blobs_shape; - for (const auto &p : input_blobs_shape) - { - blobs_shape.insert(p); - } - for (const auto &p : output_blobs_shape) - { - blobs_shape.insert(p); - } - - return std::make_shared(model_path, blobs_shape, mem_buf_size); -} - -} // namespace inference_core diff --git a/inference_core/trt_core/src/trt_core_factory.cpp b/inference_core/trt_core/src/trt_core_factory.cpp deleted file mode 100644 index b88bf52..0000000 --- a/inference_core/trt_core/src/trt_core_factory.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * @Description: - * @Author: Teddywesside 18852056629@163.com - * @Date: 2024-12-02 19:35:03 - * @LastEditTime: 2024-12-02 19:41:35 - * @FilePath: /easy_deploy/inference_core/trt_core/src/trt_core_factory.cpp - */ -#include "trt_core/trt_core.h" - -namespace inference_core { - -struct TrtInferCoreParams { - std::string model_path; - std::unordered_map> input_blobs_shape; - std::unordered_map> output_blobs_shape; - int mem_buf_size; -}; - -class TrtInferCoreFactory : public BaseInferCoreFactory { -public: - TrtInferCoreFactory(TrtInferCoreParams params) : params_(params) - {} - - std::shared_ptr Create() override - { - return CreateTrtInferCore(params_.model_path, params_.input_blobs_shape, - params_.output_blobs_shape, params_.mem_buf_size); - } - -private: - TrtInferCoreParams params_; -}; - -std::shared_ptr CreateTrtInferCoreFactory( - std::string model_path, - const std::unordered_map> &input_blobs_shape, - const std::unordered_map> &output_blobs_shape, - const int mem_buf_size) -{ - TrtInferCoreParams params; - params.model_path = model_path; - params.input_blobs_shape = input_blobs_shape; - params.output_blobs_shape = output_blobs_shape; - params.mem_buf_size = mem_buf_size; - - return std::make_shared(params); -} - -} // namespace inference_core \ No newline at end of file diff --git a/sam/sam_mobilesam/CMakeLists.txt b/sam/sam_mobilesam/CMakeLists.txt index c11ed07..66fdf30 100644 --- a/sam/sam_mobilesam/CMakeLists.txt +++ b/sam/sam_mobilesam/CMakeLists.txt @@ -17,9 +17,9 @@ include_directories( ${OpenCV_INCLUDE_DIRS} ) -add_library(${PROJECT_NAME} SHARED ${source_file}) +add_library(${PROJECT_NAME} SHARED ${source_file}) -target_link_libraries(${PROJECT_NAME} PUBLIC +target_link_libraries(${PROJECT_NAME} PUBLIC glog::glog ${OpenCV_LIBS} deploy_core @@ -28,4 +28,8 @@ target_link_libraries(${PROJECT_NAME} PUBLIC install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION lib) -target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) \ No newline at end of file +target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/include) + +if (BUILD_TESTING) + add_subdirectory(test) +endif() diff --git a/sam/sam_mobilesam/include/sam_mobilesam/mobilesam.h b/sam/sam_mobilesam/include/sam_mobilesam/mobilesam.h index a93b0a4..1030583 100644 --- a/sam/sam_mobilesam/include/sam_mobilesam/mobilesam.h +++ b/sam/sam_mobilesam/include/sam_mobilesam/mobilesam.h @@ -38,4 +38,4 @@ std::shared_ptr CreateSamMobileSamModelFactory( "has_mask_input", "masks", "scores"}); } // namespace sam -#endif \ No newline at end of file +#endif diff --git a/sam/sam_mobilesam/src/mobilesam.cpp b/sam/sam_mobilesam/src/mobilesam.cpp index e511485..efe08d5 100644 --- a/sam/sam_mobilesam/src/mobilesam.cpp +++ b/sam/sam_mobilesam/src/mobilesam.cpp @@ -3,6 +3,7 @@ #include "deploy_core/wrapper.h" #include +#include namespace sam { @@ -16,11 +17,11 @@ static void CheckBlobNameMatched(const std::string &infer_core_name, const std::shared_ptr &infer_core, const std::vector &blob_names) { - auto blob_buffer = infer_core->AllocBlobsBuffer(); - if (blob_names.size() != blob_buffer->Size()) + auto blobs_tensor = infer_core->AllocBlobsBuffer(); + if (blob_names.size() != blobs_tensor->Size()) { ThrowRuntimeError(infer_core_name + " core got different blob size with blob_names input! " + - std::to_string(blob_buffer->Size()) + " vs " + + std::to_string(blobs_tensor->Size()) + " vs " + std::to_string(blob_names.size()), __LINE__); } @@ -28,7 +29,7 @@ static void CheckBlobNameMatched(const std::string &infer_core_name, { try { - auto buffer_ptr = blob_buffer->GetOuterBlobBuffer(blob_name); + blobs_tensor->GetTensor(blob_name); } catch (std::exception e) { ThrowRuntimeError(infer_core_name + " met invalid blob_name in blob_names : " + blob_name, @@ -38,32 +39,35 @@ static void CheckBlobNameMatched(const std::string &infer_core_name, } // Bind transpose processing to big core -static void bind_to_big_core() { - cpu_set_t mask; - CPU_ZERO(&mask); +static void bind_to_big_core() +{ + cpu_set_t mask; + CPU_ZERO(&mask); - CPU_SET(4, &mask); - CPU_SET(5, &mask); - CPU_SET(6, &mask); - CPU_SET(7, &mask); + CPU_SET(4, &mask); + CPU_SET(5, &mask); + CPU_SET(6, &mask); + CPU_SET(7, &mask); - if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { - perror("sched_setaffinity failed"); - } + if (sched_setaffinity(0, sizeof(mask), &mask) == -1) + { + perror("sched_setaffinity failed"); + } } -// Unbind -static void unbind_from_big_core() { - cpu_set_t mask; - CPU_ZERO(&mask); - for (int i = 0; i < sysconf(_SC_NPROCESSORS_ONLN); ++i) { - CPU_SET(i, &mask); - } - sched_setaffinity(0, sizeof(mask), &mask); - sched_yield(); +// Unbind +static void unbind_from_big_core() +{ + cpu_set_t mask; + CPU_ZERO(&mask); + for (int i = 0; i < sysconf(_SC_NPROCESSORS_ONLN); ++i) + { + CPU_SET(i, &mask); + } + sched_setaffinity(0, sizeof(mask), &mask); + sched_yield(); } - static void rknn_nchw_2_nhwc(float *nchw, float *nhwc, int N, int C, int H, int W) { // only neccessary on `rk3588` platform. @@ -132,7 +136,6 @@ class MobileSam : public BaseSamModel { const std::string MobileSam::model_name_ = "MobileSam"; - MobileSam::MobileSam(std::shared_ptr image_encoder_core, std::shared_ptr mask_points_decoder_core, std::shared_ptr mask_boxes_decoder_core, @@ -167,18 +170,17 @@ bool MobileSam::ImagePreProcess(ParsingType package) "[MobileSam Image PreProcess] the `package` instance \ is not a instance of `SamPipelinePackage`!"); - auto encoder_blobs_buffer = p_package->image_encoder_blobs_buffer; - // make the output buffer at device side + auto encoder_blobs_tensor = p_package->image_encoder_blobs_buffer; + // make the output buffer at device side // (some inference framework will still output buffer to host side) - p_package->image_encoder_blobs_buffer->SetBlobBuffer(encoder_blob_names_[1], - DataLocation::DEVICE); + encoder_blobs_tensor->GetTensor(encoder_blob_names_[1])->SetBufferLocation(DataLocation::DEVICE); // preprocess image and write into buffer const auto scale = image_preprocess_block_->Preprocess( - p_package->input_image_data, encoder_blobs_buffer, encoder_blob_names_[0], IMAGE_INPUT_HEIGHT, - IMAGE_INPUT_WIDTH); + p_package->input_image_data, encoder_blobs_tensor->GetTensor(encoder_blob_names_[0]), + IMAGE_INPUT_HEIGHT, IMAGE_INPUT_WIDTH); // set the inference buffer - p_package->infer_buffer = p_package->image_encoder_blobs_buffer; + p_package->infer_buffer = encoder_blobs_tensor.get(); // record transform factor p_package->transform_scale = scale; @@ -193,11 +195,11 @@ bool MobileSam::PromptBoxPreProcess(ParsingType package) is not a instance of `SamPipelinePackage`!"); // 0. Get the decoder and encoder buffer - auto decoder_map_blob2ptr = p_package->mask_decoder_blobs_buffer; + auto decoder_blobs_tensor = p_package->mask_decoder_blobs_buffer; - auto encoder_map_blob2ptr = p_package->image_encoder_blobs_buffer; - const auto &encoder_output = encoder_map_blob2ptr->GetOuterBlobBuffer(encoder_blob_names_[1]); - void *image_features_ptr = encoder_output.first; + auto encoder_blobs_tensor = p_package->image_encoder_blobs_buffer; + auto encoder_output_tensor = encoder_blobs_tensor->GetTensor(encoder_blob_names_[1]); + auto image_features_ptr = encoder_output_tensor->Cast(); ////////////////// Transpose if decoder is rknn framework ////////////////// if (mask_boxes_decoder_core_->GetType() == inference_core::InferCoreType::RKNN) @@ -207,26 +209,22 @@ bool MobileSam::PromptBoxPreProcess(ParsingType package) const size_t total_image_feature_elements_num = IMAGE_FEATURE_HEIGHT * IMAGE_FEATURE_WIDTH * IMAGE_FEATURES_LEN; std::vector hwc_buffer(total_image_feature_elements_num); - rknn_nchw_2_nhwc(reinterpret_cast(encoder_output.first), hwc_buffer.data(), 1, - IMAGE_FEATURES_LEN, IMAGE_FEATURE_HEIGHT, IMAGE_FEATURE_WIDTH); - memcpy(encoder_output.first, hwc_buffer.data(), - total_image_feature_elements_num * sizeof(float)); + rknn_nchw_2_nhwc(image_features_ptr, hwc_buffer.data(), 1, IMAGE_FEATURES_LEN, + IMAGE_FEATURE_HEIGHT, IMAGE_FEATURE_WIDTH); + memcpy(image_features_ptr, hwc_buffer.data(), total_image_feature_elements_num * sizeof(float)); } //////////////////////////////////////////////////////////////////////////// // Zero-Copy Feature : let decoder use the buffer which encoder outputs - // Encoder/Decoder with different infer_core are supported. (if the hardware support) - decoder_map_blob2ptr->SetBlobBuffer( - box_dec_blob_names_[0], encoder_output.first, - encoder_output.second); + // Encoder/Decoder with different infer_core are supported. (if the hardware support) + decoder_blobs_tensor->GetTensor(box_dec_blob_names_[0])->ZeroCopy(encoder_output_tensor); // 1. Set prompt - const auto &boxes = p_package->boxes; - const auto &scale = p_package->transform_scale; - float *boxes_ptr = - static_cast(decoder_map_blob2ptr->GetOuterBlobBuffer(box_dec_blob_names_[1]).first); - const int64_t dynmaic_box_number = boxes.size(); - for (int i = 0; i < dynmaic_box_number; ++i) + const auto &boxes = p_package->boxes; + const auto &scale = p_package->transform_scale; + float *boxes_ptr = decoder_blobs_tensor->GetTensor(box_dec_blob_names_[1])->Cast(); + const uint64_t dynmaic_box_number = boxes.size(); + for (uint64_t i = 0; i < dynmaic_box_number; ++i) { const auto &box = boxes[i]; boxes_ptr[i * 4 + 0] = (box.x - box.w / 2.f) * scale; @@ -236,19 +234,17 @@ bool MobileSam::PromptBoxPreProcess(ParsingType package) } // Set dynamic shape - std::vector dynamic_shape{1, dynmaic_box_number, 4}; - decoder_map_blob2ptr->SetBlobShape(box_dec_blob_names_[1], dynamic_shape); + std::vector dynamic_shape{1, dynmaic_box_number, 4}; + decoder_blobs_tensor->GetTensor(box_dec_blob_names_[1])->SetShape(dynamic_shape); - float *mask_input = - static_cast(decoder_map_blob2ptr->GetOuterBlobBuffer(box_dec_blob_names_[2]).first); + float *mask_input = decoder_blobs_tensor->GetTensor(box_dec_blob_names_[2])->Cast(); memset(mask_input, 0, MASK_LOW_RES_WIDTH * MASK_LOW_RES_HEIGHT * sizeof(float)); - float *has_mask_input = - static_cast(decoder_map_blob2ptr->GetOuterBlobBuffer(box_dec_blob_names_[3]).first); - has_mask_input[0] = 1.f; + float *has_mask_input = decoder_blobs_tensor->GetTensor(box_dec_blob_names_[3])->Cast(); + has_mask_input[0] = 1.f; // 2. Set inference buffer - p_package->infer_buffer = decoder_map_blob2ptr; + p_package->infer_buffer = decoder_blobs_tensor.get(); return true; } @@ -261,44 +257,39 @@ bool MobileSam::PromptPointPreProcess(ParsingType package) is not a instance of `SamPipelinePackage`!"); // 0. Get the decoder and encoder buffer - auto decoder_map_blob2ptr = p_package->mask_decoder_blobs_buffer; + auto decoder_blobs_tensor = p_package->mask_decoder_blobs_buffer; + + auto encoder_blobs_tensor = p_package->image_encoder_blobs_buffer; + auto encoder_output_tensor = encoder_blobs_tensor->GetTensor(encoder_blob_names_[1]); + auto image_features_ptr = encoder_output_tensor->Cast(); - // 1. 设置image embeddings缓存指针 - auto encoder_map_blob2ptr = p_package->image_encoder_blobs_buffer; - const auto &encoder_output = encoder_map_blob2ptr->GetOuterBlobBuffer(encoder_blob_names_[1]); - void *image_features_ptr = encoder_output.first; ////////////////// Transpose if decoder is rknn framework ////////////////// - if (mask_points_decoder_core_->GetType() == inference_core::InferCoreType::RKNN) + if (mask_boxes_decoder_core_->GetType() == inference_core::InferCoreType::RKNN) { - LOG(WARNING) << "[MobileSAM] Got rknn mask point decoder! Transposing Image Features to `NHWC` " - "format!!!"; + LOG(WARNING) + << "[MobileSAM] Got rknn mask box decoder! Transposing Image Features to `NHWC` format!!!"; const size_t total_image_feature_elements_num = IMAGE_FEATURE_HEIGHT * IMAGE_FEATURE_WIDTH * IMAGE_FEATURES_LEN; std::vector hwc_buffer(total_image_feature_elements_num); - rknn_nchw_2_nhwc(reinterpret_cast(encoder_output.first), hwc_buffer.data(), 1, - IMAGE_FEATURES_LEN, IMAGE_FEATURE_HEIGHT, IMAGE_FEATURE_WIDTH); - memcpy(encoder_output.first, hwc_buffer.data(), - total_image_feature_elements_num * sizeof(float)); + rknn_nchw_2_nhwc(image_features_ptr, hwc_buffer.data(), 1, IMAGE_FEATURES_LEN, + IMAGE_FEATURE_HEIGHT, IMAGE_FEATURE_WIDTH); + memcpy(image_features_ptr, hwc_buffer.data(), total_image_feature_elements_num * sizeof(float)); } //////////////////////////////////////////////////////////////////////////// // Zero-Copy Feature : let decoder use the buffer which encoder outputs - // Encoder/Decoder with different infer_core are supported. (if the hardware support) - decoder_map_blob2ptr->SetBlobBuffer( - point_dec_blob_names_[0], encoder_output.first, - encoder_output.second); + // Encoder/Decoder with different infer_core are supported. (if the hardware support) + decoder_blobs_tensor->GetTensor(box_dec_blob_names_[0])->ZeroCopy(encoder_output_tensor); // 1. Set prompt const auto &points = p_package->points; const auto &labels = p_package->labels; const auto &scale = p_package->transform_scale; - float *points_ptr = reinterpret_cast( - decoder_map_blob2ptr->GetOuterBlobBuffer(point_dec_blob_names_[1]).first); - float *labels_ptr = reinterpret_cast( - decoder_map_blob2ptr->GetOuterBlobBuffer(point_dec_blob_names_[2]).first); + float *points_ptr = decoder_blobs_tensor->GetTensor(point_dec_blob_names_[1])->Cast(); + float *labels_ptr = decoder_blobs_tensor->GetTensor(point_dec_blob_names_[2])->Cast(); - const int64_t dynamic_point_number = points.size(); - for (int i = 0; i < dynamic_point_number; ++i) + const uint64_t dynamic_point_number = points.size(); + for (uint64_t i = 0; i < dynamic_point_number; ++i) { const auto &point = points[i]; const auto &lab = labels[i]; @@ -308,21 +299,19 @@ bool MobileSam::PromptPointPreProcess(ParsingType package) } // Set dynamic shape - std::vector coords_dynamic_shape{1, dynamic_point_number, 2}; - decoder_map_blob2ptr->SetBlobShape(point_dec_blob_names_[1], coords_dynamic_shape); - std::vector labels_dynamic_shape{1, dynamic_point_number}; - decoder_map_blob2ptr->SetBlobShape(point_dec_blob_names_[2], labels_dynamic_shape); + std::vector coords_dynamic_shape{1, dynamic_point_number, 2}; + decoder_blobs_tensor->GetTensor(point_dec_blob_names_[1])->SetShape(coords_dynamic_shape); + std::vector labels_dynamic_shape{1, dynamic_point_number}; + decoder_blobs_tensor->GetTensor(point_dec_blob_names_[2])->SetShape(labels_dynamic_shape); - float *mask_input = static_cast( - decoder_map_blob2ptr->GetOuterBlobBuffer(point_dec_blob_names_[3]).first); + float *mask_input = decoder_blobs_tensor->GetTensor(point_dec_blob_names_[3])->Cast(); memset(mask_input, 0, MASK_LOW_RES_HEIGHT * MASK_LOW_RES_WIDTH * sizeof(float)); - float *has_mask_input = static_cast( - decoder_map_blob2ptr->GetOuterBlobBuffer(point_dec_blob_names_[4]).first); - has_mask_input[0] = 1.f; + float *has_mask_input = decoder_blobs_tensor->GetTensor(point_dec_blob_names_[4])->Cast(); + has_mask_input[0] = 1.f; // 2. Set inference buffer - p_package->infer_buffer = decoder_map_blob2ptr; + p_package->infer_buffer = decoder_blobs_tensor.get(); return true; } @@ -334,11 +323,10 @@ bool MobileSam::MaskPostProcess(ParsingType package) "[MobileSam Mask PostProcess] the `package` instance \ is not a instance of `SamPipelinePackage`!"); - auto decoder_map_blob2ptr = p_package->mask_decoder_blobs_buffer; + auto decoder_blobs_tensor = p_package->mask_decoder_blobs_buffer; // 1. Get the output masks buffer - void *decoder_output_masks_ptr = - decoder_map_blob2ptr->GetOuterBlobBuffer(MASK_OUT_BLOB_NAME).first; + void *decoder_output_masks_ptr = decoder_blobs_tensor->GetTensor(MASK_OUT_BLOB_NAME)->RawPtr(); cv::Mat masks_output(MASK_LOW_RES_HEIGHT, MASK_LOW_RES_WIDTH, CV_32FC1, decoder_output_masks_ptr); // 2. resize to 1024,1024 @@ -380,5 +368,4 @@ std::shared_ptr CreateMobileSamModel( encoder_blob_names, box_dec_blob_names, point_dec_blob_names); } - } // namespace sam diff --git a/sam/sam_mobilesam/src/mobilesam_factory.cpp b/sam/sam_mobilesam/src/mobilesam_factory.cpp index 8026ff3..def3d2c 100644 --- a/sam/sam_mobilesam/src/mobilesam_factory.cpp +++ b/sam/sam_mobilesam/src/mobilesam_factory.cpp @@ -66,4 +66,4 @@ std::shared_ptr CreateSamMobileSamModelFactory( return std::make_shared(params); } -} // namespace sam \ No newline at end of file +} // namespace sam diff --git a/sam/sam_mobilesam/test/CMakeLists.txt b/sam/sam_mobilesam/test/CMakeLists.txt new file mode 100644 index 0000000..ac14acc --- /dev/null +++ b/sam/sam_mobilesam/test/CMakeLists.txt @@ -0,0 +1,55 @@ +add_compile_options(-std=c++17) +add_compile_options(-O3 -Wextra -Wdeprecated -fPIC) +set(CMAKE_CXX_STANDARD 17) + +if(ENABLE_TENSORRT) + list(APPEND platform_core_packages trt_core) +endif() + +if(ENABLE_RKNN) + list(APPEND platform_core_packages rknn_core) +endif() + +if(ENABLE_ORT) + list(APPEND platform_core_packages ort_core) +endif() + +find_package(GTest REQUIRED) +find_package(glog REQUIRED) +find_package(OpenCV REQUIRED) + +set(source_file + test_sam_mobilesam.cpp +) + +include_directories( + include + ${OpenCV_INCLUDE_DIRS} +) + +add_executable(test_sam_mobilesam ${source_file}) + +target_link_libraries(test_sam_mobilesam PUBLIC + GTest::gtest_main + glog::glog + ${OpenCV_LIBS} + deploy_core + image_processing_utils + sam_mobilesam + test_utils + ${platform_core_packages} +) + +if(ENABLE_TENSORRT) + target_compile_definitions(test_sam_mobilesam PRIVATE ENABLE_TENSORRT) +endif() + +if(ENABLE_RKNN) + target_compile_definitions(test_sam_mobilesam PRIVATE ENABLE_RKNN) +endif() + +if(ENABLE_ORT) + target_compile_definitions(test_sam_mobilesam PRIVATE ENABLE_ORT) +endif() + +gtest_discover_tests(test_sam_mobilesam) diff --git a/sam/sam_mobilesam/test/test_sam_mobilesam.cpp b/sam/sam_mobilesam/test/test_sam_mobilesam.cpp new file mode 100644 index 0000000..55c9d6b --- /dev/null +++ b/sam/sam_mobilesam/test/test_sam_mobilesam.cpp @@ -0,0 +1,264 @@ +#include + +#include "detection_2d_util/detection_2d_util.h" +#include "sam_mobilesam/mobilesam.h" +#include "test_utils/sam_test_utils.hpp" + +using namespace inference_core; +using namespace detection_2d; +using namespace sam; +using namespace test_utils; + +#define GEN_MOBILESAM_TEST_CASES(Tag, FixtureClass) \ + TEST_F(FixtureClass, test_mobilesam_##Tag##_correctness_with_points) \ + { \ + test_sam_algorithm_correctness_with_points(mobilesam_model_, points_, labels_, \ + test_image_path_, \ + test_mobilesam_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_mobilesam_##Tag##_async_correctness_with_points) \ + { \ + test_sam_algorithm_async_correctness_with_points(mobilesam_model_, points_, labels_, \ + test_image_path_, \ + test_mobilesam_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_mobilesam_##Tag##_correctness_with_boxes) \ + { \ + test_sam_algorithm_correctness_with_boxes(mobilesam_model_, boxes_, test_image_path_, \ + test_mobilesam_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_mobilesam_##Tag##_async_correctness_with_boxes) \ + { \ + test_sam_algorithm_async_correctness_with_boxes(mobilesam_model_, boxes_, test_image_path_, \ + test_mobilesam_visual_result_save_path_); \ + } + +#define GEN_NANOSAM_TEST_CASES(Tag, FixtureClass) \ + TEST_F(FixtureClass, test_nanosam_##Tag##_correctness_with_points) \ + { \ + test_sam_algorithm_correctness_with_points(nanosam_model_, points_, labels_, test_image_path_, \ + test_nanosam_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_nanosam_##Tag##_async_correctness_with_points) \ + { \ + test_sam_algorithm_async_correctness_with_points(nanosam_model_, points_, labels_, \ + test_image_path_, \ + test_nanosam_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_nanosam_##Tag##_correctness_with_boxes) \ + { \ + test_sam_algorithm_correctness_with_boxes(nanosam_model_, boxes_, test_image_path_, \ + test_nanosam_visual_result_save_path_); \ + } \ + TEST_F(FixtureClass, test_nanosam_##Tag##_async_correctness_with_boxes) \ + { \ + test_sam_algorithm_async_correctness_with_boxes(nanosam_model_, boxes_, test_image_path_, \ + test_nanosam_visual_result_save_path_); \ + } + +class BaseSamFixture : public testing::Test { +protected: + std::shared_ptr mobilesam_model_; + std::shared_ptr nanosam_model_; + + std::string test_image_path_; + std::string test_mobilesam_visual_result_save_path_; + std::string test_nanosam_visual_result_save_path_; + + std::vector> points_; + std::vector labels_; + std::vector boxes_; +}; + +#ifdef ENABLE_TENSORRT + +#include "trt_core/trt_core.h" + +class Sam_TensorRT_Fixture : public BaseSamFixture { +public: + void SetUp() override + { + auto mobilesam_image_encoder_model_path = "/workspace/models/mobile_sam_encoder.engine"; + auto nanosam_image_encoder_model_path = + "/workspace/models/nanosam_image_encoder_opset11.engine"; + auto box_decoder_model_path = "/workspace/models/modified_mobile_sam_box.engine"; + auto point_decoder_model_path = "/workspace/models/modified_mobile_sam_point.engine"; + + auto mobilesam_image_encoder = CreateTrtInferCore(mobilesam_image_encoder_model_path); + auto nanosam_image_encoder = CreateTrtInferCore(nanosam_image_encoder_model_path); + + const int SAM_MAX_BOX = 1; + const int SAM_MAX_POINTS = 8; + + auto box_decoder_factory = + CreateTrtInferCoreFactory(box_decoder_model_path, + { + {"image_embeddings", {1, 256, 64, 64}}, + {"boxes", {1, SAM_MAX_BOX, 4}}, + {"mask_input", {1, 1, 256, 256}}, + {"has_mask_input", {1}}, + }, + {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); + + auto point_decoder_factory = + CreateTrtInferCoreFactory(point_decoder_model_path, + { + {"image_embeddings", {1, 256, 64, 64}}, + {"point_coords", {1, SAM_MAX_POINTS, 2}}, + {"point_labels", {1, SAM_MAX_POINTS}}, + {"mask_input", {1, 1, 256, 256}}, + {"has_mask_input", {1}}, + }, + {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); + + auto image_preprocess_factory = CreateCudaDetPreProcessFactory(); + + mobilesam_model_ = + CreateMobileSamModel(mobilesam_image_encoder, point_decoder_factory->Create(), + box_decoder_factory->Create(), image_preprocess_factory->Create()); + + nanosam_model_ = + CreateMobileSamModel(nanosam_image_encoder, point_decoder_factory->Create(), + box_decoder_factory->Create(), image_preprocess_factory->Create()); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_mobilesam_visual_result_save_path_ = + "/workspace/test_data/mobilesam_tensorrt_test_result.jpg"; + test_nanosam_visual_result_save_path_ = "/workspace/test_data/nanosam_tensorrt_test_result.jpg"; + + points_ = {{225, 370}}; + labels_ = {1}; + + BBox2D box; + box.x = 225; + box.y = 370; + box.w = 110; + box.h = 300; + boxes_ = {box}; + } +}; + +GEN_MOBILESAM_TEST_CASES(tensorrt, Sam_TensorRT_Fixture); +GEN_NANOSAM_TEST_CASES(tensorrt, Sam_TensorRT_Fixture); + +#endif + +#ifdef ENABLE_ORT + +#include "ort_core/ort_core.h" + +class Sam_OnnxRuntime_Fixture : public BaseSamFixture { +public: + void SetUp() override + { + auto mobilesam_image_encoder_model_path = "/workspace/models/mobile_sam_encoder.onnx"; + auto nanosam_image_encoder_model_path = "/workspace/models/nanosam_image_encoder_opset11.onnx"; + auto box_decoder_model_path = "/workspace/models/modified_mobile_sam_box.onnx"; + auto point_decoder_model_path = "/workspace/models/modified_mobile_sam_point.onnx"; + + auto mobilesam_image_encoder = CreateOrtInferCore(mobilesam_image_encoder_model_path); + auto nanosam_image_encoder = CreateOrtInferCore(nanosam_image_encoder_model_path); + + const int SAM_MAX_BOX = 1; + const int SAM_MAX_POINTS = 8; + + auto box_decoder_factory = + CreateOrtInferCoreFactory(box_decoder_model_path, + { + {"image_embeddings", {1, 256, 64, 64}}, + {"boxes", {1, SAM_MAX_BOX, 4}}, + {"mask_input", {1, 1, 256, 256}}, + {"has_mask_input", {1}}, + }, + {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); + + auto point_decoder_factory = + CreateOrtInferCoreFactory(point_decoder_model_path, + { + {"image_embeddings", {1, 256, 64, 64}}, + {"point_coords", {1, SAM_MAX_POINTS, 2}}, + {"point_labels", {1, SAM_MAX_POINTS}}, + {"mask_input", {1, 1, 256, 256}}, + {"has_mask_input", {1}}, + }, + {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); + + auto image_preprocess_factory = + CreateCpuDetPreProcessFactory({0, 0, 0}, {255, 255, 255}, true, true); + + mobilesam_model_ = + CreateMobileSamModel(mobilesam_image_encoder, point_decoder_factory->Create(), + box_decoder_factory->Create(), image_preprocess_factory->Create()); + + nanosam_model_ = + CreateMobileSamModel(nanosam_image_encoder, point_decoder_factory->Create(), + box_decoder_factory->Create(), image_preprocess_factory->Create()); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_mobilesam_visual_result_save_path_ = + "/workspace/test_data/mobilesam_onnxruntime_test_result.jpg"; + test_nanosam_visual_result_save_path_ = + "/workspace/test_data/nanosam_onnxruntime_test_result.jpg"; + + points_ = {{225, 370}}; + labels_ = {1}; + + BBox2D box; + box.x = 225; + box.y = 370; + box.w = 110; + box.h = 300; + boxes_ = {box}; + } +}; + +GEN_MOBILESAM_TEST_CASES(onnxruntime, Sam_OnnxRuntime_Fixture); +GEN_NANOSAM_TEST_CASES(onnxruntime, Sam_OnnxRuntime_Fixture); + +#endif + +#ifdef ENABLE_RKNN + +#include "rknn_core/rknn_core.h" + +class Sam_Rknn_Fixture : public BaseSamFixture { +public: + void SetUp() override + { + auto nanosam_image_encoder_model_path = "/workspace/models/nanosam_image_encoder_opset11.rknn"; + auto box_decoder_model_path = "/workspace/models/modified_mobile_sam_box.rknn"; + auto point_decoder_model_path = "/workspace/models/modified_mobile_sam_point.rknn"; + + auto nanosam_image_encoder = CreateRknnInferCore( + nanosam_image_encoder_model_path, {{"images", RknnInputTensorType::RK_UINT8}}, 5, 2); + + auto box_decoder_factory = CreateRknnInferCoreFactory(box_decoder_model_path, {}, 5, 2); + + auto point_decoder_factory = CreateRknnInferCoreFactory(point_decoder_model_path, {}, 5, 2); + + auto image_preprocess_factory = + CreateCpuDetPreProcessFactory({0, 0, 0}, {255, 255, 255}, false, false); + + nanosam_model_ = + CreateMobileSamModel(nanosam_image_encoder, point_decoder_factory->Create(), + box_decoder_factory->Create(), image_preprocess_factory->Create()); + + test_image_path_ = "/workspace/test_data/persons.jpg"; + test_mobilesam_visual_result_save_path_ = "/workspace/test_data/mobilesam_rknn_test_result.jpg"; + test_nanosam_visual_result_save_path_ = "/workspace/test_data/nanosam_rknn_test_result.jpg"; + + points_ = {{225, 370}}; + labels_ = {1}; + + BBox2D box; + box.x = 225; + box.y = 370; + box.w = 110; + box.h = 300; + boxes_ = {box}; + } +}; + +GEN_NANOSAM_TEST_CASES(rknn, Sam_Rknn_Fixture); + +#endif diff --git a/simple_tests/include/tests/fps_counter.h b/simple_tests/include/tests/fps_counter.h deleted file mode 100644 index 4bd05cb..0000000 --- a/simple_tests/include/tests/fps_counter.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef __TESTS_ALL_IN_ONE_FPS_COUNTER_H -#define __TESTS_ALL_IN_ONE_FPS_COUNTER_H - -#include -#include -#include - -class FPSCounter { -public: - // 构造函数，初始化累加值和开始时间 - FPSCounter() : sum(0), is_running(false) {} - - // 开始计时 - void Start() { - start_time = std::chrono::high_resolution_clock::now(); - sum = 0; - is_running = true; - } - - // 增加帧数计数 - void Count(int i) { - if (!is_running) { - LOG(ERROR) << "Please call Start() before counting."; - return; - } - sum += i; - } - - // 获取 FPS - double GetFPS() { - if (!is_running) { - LOG(ERROR) << "Please call Start() before calculating FPS."; - return 0.0; - } - - auto current_time = std::chrono::high_resolution_clock::now(); - std::chrono::duration duration = current_time - start_time; - double duration_seconds = - std::chrono::duration_cast(duration).count(); - - if (duration_seconds == 0) { - return 0.0; // 避免除以零 - } - - return sum / duration_seconds * 1000; - } - -private: - int sum; // 累加值 - bool is_running; // 计时是否运行 - std::chrono::high_resolution_clock::time_point start_time; // 开始时间 -}; - - - - -#endif \ No newline at end of file diff --git a/simple_tests/include/tests/fs_util.h b/simple_tests/include/tests/fs_util.h deleted file mode 100644 index bd3aadd..0000000 --- a/simple_tests/include/tests/fs_util.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __TESTS_ALL_IN_ONE_FS_UTIL_H -#define __TESTS_ALL_IN_ONE_FS_UTIL_H - - -#include -#include -#include - -namespace fs = std::filesystem; - -/** - * @brief Get the absolute path of files in the directory - * - * @param directory - * @return std::vector - */ -std::vector get_files_in_directory(const fs::path& directory); - - - -#endif \ No newline at end of file diff --git a/simple_tests/include/tests/image_drawer.h b/simple_tests/include/tests/image_drawer.h deleted file mode 100644 index 7de6081..0000000 --- a/simple_tests/include/tests/image_drawer.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef __TESTS_ALL_IN_ONE_IMAGE_DRAWER_H -#define __TESTS_ALL_IN_ONE_IMAGE_DRAWER_H - -#include "deploy_core/base_detection.h" - -#include - -#define DRAW_LINE_THICKNESS 2 - -class ImageDrawHelper { -public: - ImageDrawHelper() = delete; - ImageDrawHelper(const std::shared_ptr &image) : _image(image) {}; - - void drawRect2D(const int center_x, - const int center_y, - const int width, - const int height, - const cv::Scalar color = {255, 0, 0}) - { - cv::rectangle(*_image, cv::Rect(center_x - width / 2, center_y - height / 2, width, height), - color, DRAW_LINE_THICKNESS); - } - - void drawLabel(const int x, const int y, const std::string &label, const cv::Scalar color) - { - cv::putText(*_image, label, {x, y}, 0, 2, color); - } - - void drawRect2D(const BBox2D &obj, const cv::Scalar color = {255, 0, 0}) - { - drawRect2D(obj.x, obj.y, obj.w, obj.h, color); - } - - void drawRect2DWithLabel(const BBox2D &obj, - const std::string &label, - const cv::Scalar color = {255, 0, 0}) - { - drawRect2D(obj.x, obj.y, obj.w, obj.h, color); - drawLabel(obj.x - obj.w / 2, obj.y - obj.h / 2 - 10, label, color); - } - - void drawPoint(const std::pair &point, const cv::Scalar color = {255, 0, 0}) - { - cv::circle(*_image, {point.first, point.second}, 4, color, -1); - } - - void addRedMaskToForeground(const cv::Mat &mask, float alpha = 0.5) - { - CV_Assert(mask.channels() == 1); - - // 创建一个红色掩码 - cv::Mat redMask(_image->size(), CV_8UC3, cv::Scalar(0, 0, 255)); - - // 创建输出图像 - - // 遍历像素，根据掩码加权融合 - for (int y = 0; y < mask.rows; y++) - { - for (int x = 0; x < mask.cols; x++) - { - if (mask.at(y, x) > 0) - { - // 前景区域：将红色掩码与原图像加权融合 - _image->at(y, x) = - alpha * redMask.at(y, x) + (1 - alpha) * _image->at(y, x); - } - } - } - - } - - std::shared_ptr getImage() const - { - return _image; - } - -private: - std::shared_ptr _image; -}; - -#endif \ No newline at end of file diff --git a/simple_tests/include/tests/test_func.h b/simple_tests/include/tests/test_func.h deleted file mode 100644 index 2111934..0000000 --- a/simple_tests/include/tests/test_func.h +++ /dev/null @@ -1,112 +0,0 @@ -#ifndef __TESTS_ALL_IN_ONE_TEST_FUNC_H -#define __TESTS_ALL_IN_ONE_TEST_FUNC_H - -#include "deploy_core/base_detection.h" -#include "deploy_core/base_sam.h" -#include "tests/image_drawer.h" -#include "tests/fps_counter.h" -#include "tests/fs_util.h" - -#include -#include -#include - -#include -#include - -using namespace inference_core; - -using namespace detection_2d; -using namespace async_pipeline; -using namespace sam; - -class DumbInputImageData : public IPipelineImageData { -public: - DumbInputImageData(const cv::Mat &cv_image) : inner_cv_image(cv_image) - { - image_data_info.data_pointer = cv_image.data; - image_data_info.format = ImageDataFormat::BGR; - image_data_info.image_height = cv_image.rows; - image_data_info.image_width = cv_image.cols; - image_data_info.image_channels = cv_image.channels(); - image_data_info.location = DataLocation::HOST; - } - - const ImageDataInfo &GetImageDataInfo() const - { - return image_data_info; - } - -private: - IPipelineImageData::ImageDataInfo image_data_info; - cv::Mat inner_cv_image; -}; - -float test_func_infer_core_speed(std::shared_ptr core); - -float test_func_yolov8_model_speed( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg"); - -int test_func_yolov8_model_correctness( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/test_persons_detection_results.jpg"); - -float test_func_yolov8_model_pipeline_speed( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg"); - -int test_func_yolov8_model_pipeline_correctness( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/test_persons_detection_results.jpg"); - -// int test_func_yolov8_model_pipeline_correctness_callback(std::shared_ptr -// model, -// std::string test_image_path = -// "/workspace/test_data/persons.jpg", std::string -// test_results_save_path = -// "/workspace/test_data/test_persons_detection_results.jpg"); - -void test_func_sam_point_correctness( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -void test_func_sam_box_correctness( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -void test_func_sam_point_pipeline_correctness( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -void test_func_sam_box_pipeline_correctness( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -float test_func_sam_point_speed( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -float test_func_sam_box_speed( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -float test_func_sam_point_pipeline_speed( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -float test_func_sam_box_pipeline_speed( - std::shared_ptr model, - std::string test_image_path = "/workspace/test_data/persons.jpg", - std::string test_results_save_path = "/workspace/test_data/tests_masks_output.png"); - -#endif \ No newline at end of file diff --git a/simple_tests/src/fs_util.cpp b/simple_tests/src/fs_util.cpp deleted file mode 100644 index 22c7b53..0000000 --- a/simple_tests/src/fs_util.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "tests/fs_util.h" - - - -std::vector get_files_in_directory(const fs::path& directory) { - std::vector files; - - // 检查目录是否存在 - if (!fs::exists(directory) || !fs::is_directory(directory)) { - std::cerr << "Directory does not exist or is not a directory." << std::endl; - return files; - } - - // 递归遍历目录 - for (const auto& entry : fs::recursive_directory_iterator(directory)) { - if (entry.is_regular_file()) { - files.push_back(fs::absolute(entry.path())); - } - } - - return files; -} diff --git a/simple_tests/src/gtest_main.cpp b/simple_tests/src/gtest_main.cpp deleted file mode 100644 index a794ef5..0000000 --- a/simple_tests/src/gtest_main.cpp +++ /dev/null @@ -1,26 +0,0 @@ - - -#include -#include - -#include - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - - google::InitGoogleLogging(argv[0]); - - google::SetStderrLogging(google::GLOG_WARNING); - FLAGS_logtostderr = false; // 不输出到标准错误 - FLAGS_log_dir = "./test_log/"; // 指定日志文件存放目录 - - // FLAGS_minloglevel = 0; - FLAGS_logtostderr = true; - int result = RUN_ALL_TESTS(); - - google::ShutdownGoogleLogging(); - - return result; -} - - diff --git a/simple_tests/src/test_cpu_devkit.cpp b/simple_tests/src/test_cpu_devkit.cpp deleted file mode 100644 index 3c0775f..0000000 --- a/simple_tests/src/test_cpu_devkit.cpp +++ /dev/null @@ -1,185 +0,0 @@ -#include "tests/test_func.h" -#include "detection_2d_yolov8/yolov8.h" -#include "detection_2d_rt_detr/rt_detr.h" -#include "detection_2d_util/detection_2d_util.h" -#include "ort_core/ort_core.h" -#include "tests/fs_util.h" - -#include "sam_mobilesam/mobilesam.h" - -/************************** -**** ort core test **** -***************************/ - -using namespace inference_core; -using namespace detection_2d; -using namespace sam; - -static -std::shared_ptr GetYolov8Factory() -{ - std::string model_path = "/workspace/models/yolov8n.onnx"; - const int input_height = 640; - const int input_width = 640; - const int input_channels = 3; - const int cls_number = 80; - const std::vector input_blobs_name = {"images"}; - const std::vector output_blobs_name = {"output0"}; - - auto infer_core_factory = CreateOrtInferCoreFactory(model_path); - auto preprocess_factory = CreateCpuDetPreProcessFactory(); - auto postprocess_factory = - CreateYolov8PostProcessCpuOriginFactory(input_height, input_width, cls_number); - - return CreateYolov8DetectionModelFactory( - infer_core_factory, preprocess_factory, postprocess_factory, input_height, input_width, - input_channels, cls_number, input_blobs_name, output_blobs_name); -} - -static -std::shared_ptr GetRTDetrFactory() -{ - std::string model_path = "/workspace/models/rt_detr_v2_single_input.onnx"; - const int input_height = 640; - const int input_width = 640; - const int input_channels = 3; - const int cls_number = 80; - const std::vector input_blobs_name = {"images"}; - const std::vector output_blobs_name = {"labels", "boxes", "scores"}; - - auto infer_core_factory = CreateOrtInferCoreFactory(model_path); - auto preprocess_factory = CreateCpuDetPreProcessFactory(); - - return CreateRTDetrDetectionModelFactory(infer_core_factory, preprocess_factory, input_height, - input_width, input_channels, cls_number, - input_blobs_name, output_blobs_name); -} - -const static int SAM_MAX_POINTS = 8; -const static int SAM_MAX_BOX = 1; - -static -std::shared_ptr GetMobileSamFactory() -{ - auto image_encoder_model_path = "/workspace/models/mobile_sam_encoder.onnx"; - // auto image_encoder_model_path = "/workspace/models/nanosam_image_encoder_opset11.onnx"; - auto box_decoder_model_path = "/workspace/models/modified_mobile_sam_box.onnx"; - auto point_decoder_model_path = "/workspace/models/modified_mobile_sam_point.onnx"; - - auto image_encoder_factory = CreateOrtInferCoreFactory(image_encoder_model_path); - - auto box_decoder_factory = - CreateOrtInferCoreFactory(box_decoder_model_path, - { - {"image_embeddings", {1, 256, 64, 64}}, - {"boxes", {1, SAM_MAX_BOX, 4}}, - {"mask_input", {1, 1, 256, 256}}, - {"has_mask_input", {1}}, - }, - {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); - - auto point_decoder_factory = - CreateOrtInferCoreFactory(point_decoder_model_path, - { - {"image_embeddings", {1, 256, 64, 64}}, - {"point_coords", {1, SAM_MAX_POINTS, 2}}, - {"point_labels", {1, SAM_MAX_POINTS}}, - {"mask_input", {1, 1, 256, 256}}, - {"has_mask_input", {1}}, - }, - {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); - - auto image_preprocess_factory = CreateCpuDetPreProcessFactory(); - - return CreateSamMobileSamModelFactory(image_encoder_factory, point_decoder_factory, - box_decoder_factory, image_preprocess_factory); -} - -TEST(detection_yolov8_test, ort_core_correctness) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_correctness(factory->Create()); -} - -TEST(detection_yolov8_test, ort_core_speed) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_speed(factory->Create()); -} - -TEST(detection_yolov8_test, ort_core_pipeline_correctness) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_pipeline_correctness(factory->Create()); -} - -TEST(detection_yolov8_test, ort_core_pipeline_speed) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_pipeline_speed(factory->Create()); -} - -TEST(detection_rtdetr_test, ort_core_correctness) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_correctness(factory->Create()); -} - -TEST(detection_rtdetr_test, ort_core_speed) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_speed(factory->Create()); -} - -TEST(detection_rtdetr_test, ort_core_pipeline_correctness) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_pipeline_correctness(factory->Create()); -} - -TEST(detection_rtdetr_test, ort_core_pipeline_speed) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_pipeline_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, ort_with_point_correctness) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_point_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, ort_with_point_speed) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_point_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, ort_with_box_correctness) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_box_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, ort_with_point_pipeline_correctness) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_point_pipeline_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, ort_with_point_pipeline_speed) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_point_pipeline_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, ort_with_box_pipeline_correctness) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_box_pipeline_correctness(factory->Create()); -} diff --git a/simple_tests/src/test_func.cpp b/simple_tests/src/test_func.cpp deleted file mode 100644 index 1a79998..0000000 --- a/simple_tests/src/test_func.cpp +++ /dev/null @@ -1,356 +0,0 @@ -#include "tests/test_func.h" -#include "deploy_core/wrapper.h" - -float test_func_infer_core_speed(std::shared_ptr core) -{ - auto map_blob2ptr = core->AllocBlobsBuffer(); - FPSCounter fps_counter; - fps_counter.Start(); - for (int i = 0; i < 500; ++i) - { - core->SyncInfer(map_blob2ptr); - fps_counter.Count(1); - } - LOG(WARNING) << "average fps: " << fps_counter.GetFPS(); - return fps_counter.GetFPS(); -} - -float test_func_yolov8_model_speed(std::shared_ptr model, - std::string test_image_path) -{ - cv::Mat fake_image = cv::imread(test_image_path); - FPSCounter fps_counter; - fps_counter.Start(); - for (int i = 0; i < 500; ++i) - { - std::vector results; - model->Detect(fake_image.clone(), results, 0.4); - fps_counter.Count(1); - } - LOG(WARNING) << "average fps: " << fps_counter.GetFPS(); - return fps_counter.GetFPS(); -} - -int test_func_yolov8_model_correctness(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - cv::Mat fake_image = cv::imread(test_image_path); - std::vector results; - auto start = std::chrono::high_resolution_clock::now(); - model->Detect(fake_image, results, 0.4); - auto end = std::chrono::high_resolution_clock::now(); - LOG(INFO) << "do_inference, cost : " - << std::chrono::duration_cast(end - start).count(); - - ImageDrawHelper drawer(std::make_shared(fake_image.clone())); - for (const auto &res : results) - { - drawer.drawRect2D(res); - } - auto p_image = drawer.getImage(); - cv::imwrite(test_results_save_path, *p_image); - - LOG(WARNING) << "remain objects number: " << results.size(); - return results.size(); -} - -float test_func_yolov8_model_pipeline_speed(std::shared_ptr model, - std::string test_image_path) -{ - model->InitPipeline(); - cv::Mat fake_image = cv::imread(test_image_path); - - BlockQueue>>> future_bq(100); - - auto func_push_data = [&]() { - int index = 0; - while (index++ < 2000) - { - auto p_fut = std::make_shared>>( - model->DetectAsync(fake_image.clone(), 0.4)); - future_bq.BlockPush(p_fut); - } - future_bq.SetNoMoreInput(); - }; - - FPSCounter fps_counter; - auto func_take_results = [&]() { - int index = 0; - fps_counter.Start(); - while (true) - { - auto output = future_bq.Take(); - if (!output.has_value()) - break; - output.value()->get(); - fps_counter.Count(1); - } - }; - - std::thread t_push(func_push_data); - std::thread t_take(func_take_results); - - t_push.join(); - model->StopPipeline(); - t_take.join(); - model->ClosePipeline(); - - LOG(WARNING) << "average fps: " << fps_counter.GetFPS(); - return fps_counter.GetFPS(); -} - -int test_func_yolov8_model_pipeline_correctness(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - model->InitPipeline(); - - cv::Mat fake_image = cv::imread(test_image_path); - - auto future = model->DetectAsync(fake_image, 0.4); - - std::vector results = future.get(); - - ImageDrawHelper drawer(std::make_shared(fake_image.clone())); - for (const auto &res : results) - { - drawer.drawRect2D(res); - } - auto p_image = drawer.getImage(); - cv::imwrite(test_results_save_path, *p_image); - - model->StopPipeline(); - model->ClosePipeline(); - LOG(WARNING) << "remain objects number: " << results.size(); - return results.size(); -} - -// int test_func_yolov8_model_pipeline_correctness_callback(std::shared_ptr -// model, -// std::string test_image_path, -// std::string test_results_save_path) -// { -// model->InitPipeline(); - -// bool flag = false; -// int detected_obj_num = 0; - -// cv::Mat fake_image = cv::imread(test_image_path); - -// model->DetectAsync(fake_image, -// 0.4, -// [&](const std::vector& results, int index) { -// ImageDrawHelper drawer(std::make_shared(fake_image.clone())); -// for (const auto & res : results) { -// drawer.drawRect2D(res); -// } -// auto p_image = drawer.getImage(); -// cv::imwrite(test_results_save_path, *p_image); -// LOG(WARNING) << "remain objects number: " << results.size(); -// detected_obj_num = results.size(); -// flag = true; -// }); - -// while (flag == false) {} - -// model->StopPipeline(); -// model->ClosePipeline(); -// return detected_obj_num; -// } - -void test_func_sam_point_correctness(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - cv::Mat image_test = cv::imread(test_image_path); - cv::Mat masks; - model->GenerateMask(image_test, {{225, 370}}, std::vector{1}, masks, false); - - ImageDrawHelper helper(std::make_shared(image_test.clone())); - helper.addRedMaskToForeground(masks); - - cv::imwrite(test_results_save_path, *helper.getImage()); -} - -void test_func_sam_box_correctness(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - cv::Mat image_test = cv::imread(test_image_path); - cv::Mat masks; - model->GenerateMask(image_test, {{225, 370, 110, 300}}, masks); - - ImageDrawHelper helper(std::make_shared(image_test.clone())); - helper.addRedMaskToForeground(masks); - - cv::imwrite(test_results_save_path, *helper.getImage()); -} - -void test_func_sam_point_pipeline_correctness(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - cv::Mat image_test = cv::imread(test_image_path); - - model->InitPipeline(); - - for (int i = 0; i < 5; ++i) - { - auto fut = model->GenerateMaskAsync(image_test, {{225, 370}}, std::vector{1}); - - cv::Mat masks = fut.get(); - ImageDrawHelper helper(std::make_shared(image_test.clone())); - helper.addRedMaskToForeground(masks); - - cv::imwrite("/workspace/test_data/tests_masks_output_" + std::to_string(i) + ".png", *helper.getImage()); - } -} - -void test_func_sam_box_pipeline_correctness(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - cv::Mat image_test = cv::imread(test_image_path); - - model->InitPipeline(); - - for (int i = 0; i < 5; ++i) - { - auto fut = model->GenerateMaskAsync(image_test, {{225, 370, 110, 300}}); - - cv::Mat masks = fut.get(); - ImageDrawHelper helper(std::make_shared(image_test.clone())); - helper.addRedMaskToForeground(masks); - - cv::imwrite("/workspace/test_data/tests_masks_output_" + std::to_string(i) + ".png", *helper.getImage()); - } -} - -float test_func_sam_point_speed(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - FPSCounter fps_counter; - fps_counter.Start(); - cv::Mat image_test = cv::imread(test_image_path); - for (int i = 0; i < 100; ++i) - { - cv::Mat masks; - model->GenerateMask(image_test, {{225, 370}}, {1}, masks, false); - fps_counter.Count(1); - } - LOG(WARNING) << "average fps: " << fps_counter.GetFPS(); - - return fps_counter.GetFPS(); -} - -float test_func_sam_box_speed(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - FPSCounter fps_counter; - fps_counter.Start(); - cv::Mat image_test = cv::imread(test_image_path); - for (int i = 0; i < 100; ++i) - { - cv::Mat masks; - model->GenerateMask(image_test, {{225, 370, 110, 300}}, masks, false); - fps_counter.Count(1); - } - LOG(WARNING) << "average fps: " << fps_counter.GetFPS(); - - return fps_counter.GetFPS(); -} - -float test_func_sam_point_pipeline_speed(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - cv::Mat image_test = cv::imread(test_image_path); - - model->InitPipeline(); - - BlockQueue>> future_bq(100); - - auto func_push_data = [&]() { - int index = 0; - while (index++ < 200) - { - future_bq.BlockPush(std::make_shared>( - model->GenerateMaskAsync(image_test, {{225, 370}}, std::vector{1}))); - } - future_bq.SetNoMoreInput(); - }; - - FPSCounter fps_counter; - auto func_take_results = [&]() { - int index = 0; - fps_counter.Start(); - while (true) - { - auto res = future_bq.Take(); - if (!res.has_value()) - break; - res.value()->get(); - fps_counter.Count(1); - } - }; - - std::thread t_push(func_push_data); - std::thread t_take(func_take_results); - - t_push.join(); - model->StopPipeline(); - t_take.join(); - model->ClosePipeline(); - - LOG(WARNING) << "average fps: " << fps_counter.GetFPS(); - return fps_counter.GetFPS(); -} - -float test_func_sam_box_pipeline_speed(std::shared_ptr model, - std::string test_image_path, - std::string test_results_save_path) -{ - cv::Mat image_test = cv::imread(test_image_path); - - model->InitPipeline(); - - BlockQueue>> future_bq(100); - - auto func_push_data = [&]() { - int index = 0; - while (index++ < 200) - { - future_bq.BlockPush(std::make_shared>( - model->GenerateMaskAsync(image_test, {{225, 370, 110, 300}}))); - } - future_bq.SetNoMoreInput(); - }; - - FPSCounter fps_counter; - auto func_take_results = [&]() { - int index = 0; - fps_counter.Start(); - while (true) - { - auto res = future_bq.Take(); - if (!res.has_value()) - break; - res.value()->get(); - fps_counter.Count(1); - } - }; - - std::thread t_push(func_push_data); - std::thread t_take(func_take_results); - - t_push.join(); - model->StopPipeline(); - t_take.join(); - model->ClosePipeline(); - - LOG(WARNING) << "average fps: " << fps_counter.GetFPS(); - return fps_counter.GetFPS(); -} diff --git a/simple_tests/src/test_jetson_devkit.cpp b/simple_tests/src/test_jetson_devkit.cpp deleted file mode 100644 index bf13487..0000000 --- a/simple_tests/src/test_jetson_devkit.cpp +++ /dev/null @@ -1,206 +0,0 @@ -#include "tests/test_func.h" -#include "trt_core/trt_core.h" -#include "tests/fps_counter.h" -#include "detection_2d_yolov8/yolov8.h" -#include "detection_2d_rt_detr/rt_detr.h" -#include "sam_mobilesam/mobilesam.h" -#include "detection_2d_util/detection_2d_util.h" - -/************************** -**** trt core test **** -***************************/ - -using namespace inference_core; -using namespace detection_2d; -using namespace sam; - -static -std::shared_ptr GetYolov8Factory() -{ - std::string model_path = "/workspace/models/yolov8n.engine"; - const int input_height = 640; - const int input_width = 640; - const int input_channels = 3; - const int cls_number = 80; - const std::vector input_blobs_name = {"images"}; - const std::vector output_blobs_name = {"output0"}; - - auto infer_core_factory = CreateTrtInferCoreFactory(model_path); - auto preprocess_factory = CreateCudaDetPreProcessFactory(); - auto postprocess_factory = - CreateYolov8PostProcessCpuOriginFactory(input_height, input_width, cls_number); - - return CreateYolov8DetectionModelFactory( - infer_core_factory, preprocess_factory, postprocess_factory, input_height, input_width, - input_channels, cls_number, input_blobs_name, output_blobs_name); -} - -static -std::shared_ptr GetRTDetrFactory() -{ - std::string model_path = "/workspace/models/rt_detr_v2_single_input.engine"; - const int input_height = 640; - const int input_width = 640; - const int input_channels = 3; - const int cls_number = 80; - const std::vector input_blobs_name = {"images"}; - const std::vector output_blobs_name = {"labels", "boxes", "scores"}; - - auto infer_core_factory = CreateTrtInferCoreFactory(model_path); - auto preprocess_factory = CreateCudaDetPreProcessFactory(); - - return CreateRTDetrDetectionModelFactory(infer_core_factory, preprocess_factory, input_height, - input_width, input_channels, cls_number, - input_blobs_name, output_blobs_name); -} - -const static int SAM_MAX_POINTS = 8; -const static int SAM_MAX_BOX = 1; - -static -std::shared_ptr GetMobileSamFactory() -{ - auto image_encoder_model_path = "/workspace/models/mobile_sam_encoder.engine"; - // auto image_encoder_model_path = "/workspace/models/nanosam_image_encoder_opset11.engine"; - auto box_decoder_model_path = "/workspace/models/modified_mobile_sam_box.engine"; - auto point_decoder_model_path = "/workspace/models/modified_mobile_sam_point.engine"; - - auto image_encoder_factory = CreateTrtInferCoreFactory(image_encoder_model_path); - - auto box_decoder_factory = - CreateTrtInferCoreFactory(box_decoder_model_path, - { - {"image_embeddings", {1, 256, 64, 64}}, - {"boxes", {1, SAM_MAX_BOX, 4}}, - {"mask_input", {1, 1, 256, 256}}, - {"has_mask_input", {1}}, - }, - {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); - - auto point_decoder_factory = - CreateTrtInferCoreFactory(point_decoder_model_path, - { - {"image_embeddings", {1, 256, 64, 64}}, - {"point_coords", {1, SAM_MAX_POINTS, 2}}, - {"point_labels", {1, SAM_MAX_POINTS}}, - {"mask_input", {1, 1, 256, 256}}, - {"has_mask_input", {1}}, - }, - {{"masks", {1, 1, 256, 256}}, {"scores", {1, 1}}}); - - auto image_preprocess_factory = CreateCpuDetPreProcessFactory(); - - return CreateSamMobileSamModelFactory(image_encoder_factory, point_decoder_factory, - box_decoder_factory, image_preprocess_factory); -} - -/*********************************** -**** detection with trt test **** -************************************/ - -TEST(detection_yolov8_test, trt_core_correctness) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_correctness(factory->Create()); -} - -TEST(detection_yolov8_test, trt_core_speed) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_speed(factory->Create()); -} - -TEST(detection_yolov8_test, trt_core_pipeline_correctness) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_pipeline_correctness(factory->Create()); -} - -TEST(detection_yolov8_test, trt_core_pipeline_speed) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_pipeline_speed(factory->Create()); -} - -TEST(detection_rtdetr_test, trt_core_correctness) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_correctness(factory->Create()); -} - -TEST(detection_rtdetr_test, trt_core_speed) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_speed(factory->Create()); -} - -TEST(detection_rtdetr_test, trt_core_pipeline_correctness) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_pipeline_correctness(factory->Create()); -} - -TEST(detection_rtdetr_test, trt_core_pipeline_speed) -{ - auto factory = GetRTDetrFactory(); - int res = test_func_yolov8_model_pipeline_speed(factory->Create()); -} - -/*********************************** -**** mobilesam with trt test **** -************************************/ - -TEST(sam_mobilesam_test, trt_with_point_correctness) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_point_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, trt_with_point_speed) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_point_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, trt_with_box_correctness) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_box_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, trt_with_box_speed) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_box_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, trt_with_point_pipeline_correctness) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_point_pipeline_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, trt_with_box_pipeline_correctness) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_box_pipeline_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, trt_with_point_pipeline_speed) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_point_pipeline_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, trt_with_box_pipeline_speed) -{ - auto factory = GetMobileSamFactory(); - - test_func_sam_box_pipeline_speed(factory->Create()); -} \ No newline at end of file diff --git a/simple_tests/src/test_rk_devkit.cpp b/simple_tests/src/test_rk_devkit.cpp deleted file mode 100644 index 3358694..0000000 --- a/simple_tests/src/test_rk_devkit.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include "tests/test_func.h" -#include "rknn_core/rknn_core.h" -#include "detection_2d_yolov8/yolov8.h" -#include "detection_2d_util/detection_2d_util.h" -#include "sam_mobilesam/mobilesam.h" - -/************************** -**** rknn core test **** -***************************/ - -using namespace inference_core; -using namespace detection_2d; -using namespace sam; - -static -std::shared_ptr GetYolov8Factory() -{ - std::string model_path = "/workspace/models/yolov8n_divide_opset11.rknn"; - const int input_height = 640; - const int input_width = 640; - const int input_channels = 3; - const int cls_number = 80; - const std::vector input_blobs_name = {"images"}; - const std::vector output_blobs_name = {"318", "onnx::ReduceSum_326", "331", - "338", "onnx::ReduceSum_346", "350", - "357", "onnx::ReduceSum_365", "369"}; - - auto infer_core_factory = CreateRknnInferCoreFactory(model_path, {}, 5, 3); - auto preprocess_factory = CreateCpuDetPreProcessFactory({0, 0, 0}, {1, 1, 1}, false, false); - auto postprocess_factory = - CreateYolov8PostProcessCpuDivideFactory(input_height, input_width, cls_number); - - return CreateYolov8DetectionModelFactory( - infer_core_factory, preprocess_factory, postprocess_factory, input_height, input_width, - input_channels, cls_number, input_blobs_name, output_blobs_name); -} - -static -std::shared_ptr GetMobileSamFactory() -{ - auto image_encoder_model_path = "/workspace/models/nanosam_image_encoder_opset11.rknn"; - auto box_decoder_model_path = "/workspace/models/modified_mobile_sam_box.rknn"; - auto point_decoder_model_path = "/workspace/models/modified_mobile_sam_point.rknn"; - - auto image_encoder_factory = CreateRknnInferCoreFactory( - image_encoder_model_path, {{"images", RknnInputTensorType::RK_UINT8}}, 5, 2); - - auto box_decoder_factory = - CreateRknnInferCoreFactory(box_decoder_model_path, {}, 5, 2); - - auto point_decoder_factory = - CreateRknnInferCoreFactory(point_decoder_model_path, {}, 5, 2); - - auto image_preprocess_factory = CreateCpuDetPreProcessFactory({0, 0, 0}, {1, 1, 1}, false, false); - - return CreateSamMobileSamModelFactory(image_encoder_factory, point_decoder_factory, - box_decoder_factory, image_preprocess_factory); -} - -/*********************************** -**** detection with rknn test **** -************************************/ - -TEST(detection_yolov8_test, rknn_core_correctness) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_correctness(factory->Create()); -} - -TEST(detection_yolov8_test, rknn_core_speed) -{ - auto factory = GetYolov8Factory(); - float fps = test_func_yolov8_model_speed(factory->Create()); -} - -TEST(detection_yolov8_test, rknn_core_pipeline_correctness) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_pipeline_correctness(factory->Create()); -} - -TEST(detection_yolov8_test, rknn_core_pipeline_speed) -{ - auto factory = GetYolov8Factory(); - int res = test_func_yolov8_model_pipeline_speed(factory->Create()); -} - -////////////////// 2024.10.18 UPDATED: SUPPORT all rknn mobilesam ////////////////// -TEST(sam_mobilesam_test, rknn_with_point_all_rk_correctness) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_point_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, rknn_with_box_all_rk_correctness) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_box_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, rknn_with_point_all_rk_speed) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_point_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, rknn_with_box_all_rk_speed) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_box_speed(factory->Create()); -} - -//////////////// pipeline //////////////////// - -TEST(sam_mobilesam_test, rknn_with_point_all_rk_pipeline_correctness) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_point_pipeline_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, rknn_with_box_all_rk_pipeline_correctness) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_box_pipeline_correctness(factory->Create()); -} - -TEST(sam_mobilesam_test, rknn_with_point_all_rk_pipeline_speed) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_point_pipeline_speed(factory->Create()); -} - -TEST(sam_mobilesam_test, rknn_with_box_all_rk_pipeline_speed) -{ - auto factory = GetMobileSamFactory(); - test_func_sam_box_pipeline_speed(factory->Create()); -} \ No newline at end of file diff --git a/tools/cvt_onnx2rknn_all.sh b/tools/cvt_onnx2rknn_all.sh index ea6ff8f..42e4f02 100644 --- a/tools/cvt_onnx2rknn_all.sh +++ b/tools/cvt_onnx2rknn_all.sh @@ -9,4 +9,4 @@ echo "Converting mobilesam_box_decoder ..." python3 /workspace/tools/cvt_onnx2rknn_mobilesam_box_decoder.py echo "Converting mobilesam_point_decoder ..." -python3 /workspace/tools/cvt_onnx2rknn_mobilesam_point_decoder.py \ No newline at end of file +python3 /workspace/tools/cvt_onnx2rknn_mobilesam_point_decoder.py diff --git a/tools/cvt_onnx2rknn_nanosam.py b/tools/cvt_onnx2rknn_nanosam.py index 60c6641..54677f0 100644 --- a/tools/cvt_onnx2rknn_nanosam.py +++ b/tools/cvt_onnx2rknn_nanosam.py @@ -10,8 +10,8 @@ if __name__ == '__main__': # Create RKNN object rknn = RKNN(verbose=False) - rknn.config(mean_values=[[123.675, 116.28, 103.53]], - std_values=[[58.395, 57.12, 57.375]], + rknn.config(mean_values=[[123.675, 116.28, 103.53]], + std_values=[[58.395, 57.12, 57.375]], target_platform="rk3588", optimization_level=2) diff --git a/tools/rk3588_echo_npu_usage.sh b/tools/rk3588_echo_npu_usage.sh index ca3dcfd..32d06d7 100644 --- a/tools/rk3588_echo_npu_usage.sh +++ b/tools/rk3588_echo_npu_usage.sh @@ -1,3 +1,3 @@ #!/bin/bash ################ This script should be used on host machine (not in container) ################ -sudo watch -n 1 cat /sys/kernel/debug/rknpu/load \ No newline at end of file +sudo watch -n 1 cat /sys/kernel/debug/rknpu/load diff --git a/tools/rk3588_max_performance.sh b/tools/rk3588_max_performance.sh index 5c0794d..4167e9c 100644 --- a/tools/rk3588_max_performance.sh +++ b/tools/rk3588_max_performance.sh @@ -24,7 +24,7 @@ sudo cat /sys/devices/system/cpu/cpufreq/policy6/cpuinfo_cur_freq # NPU定频 echo "NPU可用频率/NPU available frequency:" -sudo cat /sys/class/devfreq/fdab0000.npu/available_frequencies +sudo cat /sys/class/devfreq/fdab0000.npu/available_frequencies sudo echo userspace > /sys/class/devfreq/fdab0000.npu/governor sudo echo 1000000000 > /sys/class/devfreq/fdab0000.npu/userspace/set_freq echo "NPU当前频率/NPU current frequency:" diff --git a/tools/rt_detr_v2_export_onnx.py b/tools/rt_detr_v2_export_onnx.py index c51a723..f1ca93c 100644 --- a/tools/rt_detr_v2_export_onnx.py +++ b/tools/rt_detr_v2_export_onnx.py @@ -1,8 +1,8 @@ -import os -import sys +import os +import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) import torch -import torch.nn as nn +import torch.nn as nn from src.core import YAMLConfig def main(args, ): @@ -11,7 +11,7 @@ def main(args, ): cfg = YAMLConfig(args.config, resume=args.resume) if args.resume: - checkpoint = torch.load(args.resume, map_location='cpu') + checkpoint = torch.load(args.resume, map_location='cpu') if 'ema' in checkpoint: state = checkpoint['ema']['module'] else: @@ -27,7 +27,7 @@ def __init__(self, ) -> None: super().__init__() self.model = cfg.model.deploy() self.postprocessor = cfg.postprocessor.deploy() - + def forward(self, images, orig_target_sizes): outputs = self.model(images) return outputs @@ -36,15 +36,15 @@ def forward(self, images, orig_target_sizes): data = torch.rand(1, 3, 640, 640) size = torch.tensor([[640, 640]]) - + torch.onnx.export( - model, - (data, size), + model, + (data, size), args.output_file, input_names=['images', 'orig_target_sizes'], output_names=['out1', 'out2'], # dynamic_axes=dynamic_axes, - opset_version=16, + opset_version=16, verbose=False, do_constant_folding=True, ) @@ -56,9 +56,9 @@ def forward(self, images, orig_target_sizes): print('Check export onnx model done...') if args.simplify: - import onnx + import onnx import onnxsim - dynamic = True + dynamic = True input_shapes = {'images': data.shape} if dynamic else None onnx_model_simplify, check = onnxsim.simplify(args.output_file, input_shapes=input_shapes, dynamic_input_shape=dynamic) onnx.save(onnx_model_simplify, args.output_file)