From b1563ca4b29eca8df3132fbb0b0c87a850dc9d70 Mon Sep 17 00:00:00 2001 From: Mohanned Hassan Date: Mon, 29 Dec 2025 20:47:01 +0400 Subject: [PATCH 1/2] feat(devkit): Add slang support (Using Vulkan SDK) --- CMakeLists.txt | 162 +++++++- src/addons/devkit/addon.cpp | 20 +- src/utils/shader_compiler_slang.hpp | 534 ++++++++++++++++++++++++++ src/utils/shader_compiler_watcher.hpp | 82 +++- 4 files changed, 766 insertions(+), 32 deletions(-) create mode 100644 src/utils/shader_compiler_slang.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ab527775..6f87e4775 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,58 @@ find_program(SLANGC_BIN slangc.exe ${CMAKE_CURRENT_SOURCE_DIR}/bin/ REQUIRED) message("SLANGC_BIN: " ${SLANGC_BIN}) +set(SLANGC_FLAGS_LIST -O3 -g0 -entry main -Wno-30056 -Wno-15205) +list(JOIN SLANGC_FLAGS_LIST " " SLANGC_FLAGS_STR) +file(TO_CMAKE_PATH "${SLANGC_BIN}" SLANGC_BIN_CMAKE_PATH) + +if(DEFINED ENV{VULKAN_SDK}) + set(VULKAN_SDK_PATH "$ENV{VULKAN_SDK}") +endif() + +set(SPIRV_VAL_FLAGS_LIST --target-env vulkan1.3) +list(JOIN SPIRV_VAL_FLAGS_LIST " " SPIRV_VAL_FLAGS_STR) + +set(SPIRV_CROSS_FLAGS_LIST --version 450 --vulkan-semantics --no-es) +list(JOIN SPIRV_CROSS_FLAGS_LIST " " SPIRV_CROSS_FLAGS_STR) + +find_program(SPIRV_VAL_BIN spirv-val.exe + HINTS + ${VULKAN_SDK_PATH}/Bin + ${CMAKE_CURRENT_SOURCE_DIR}/bin/) +message("SPIRV_VAL_BIN: " ${SPIRV_VAL_BIN}) + +find_program(SPIRV_DIS_BIN spirv-dis.exe + HINTS + ${VULKAN_SDK_PATH}/Bin + ${CMAKE_CURRENT_SOURCE_DIR}/bin/) +message("SPIRV_DIS_BIN: " ${SPIRV_DIS_BIN}) + +find_program(SPIRV_CROSS_BIN spirv-cross.exe + HINTS + ${VULKAN_SDK_PATH}/Bin + ${CMAKE_CURRENT_SOURCE_DIR}/bin/) +message("SPIRV_CROSS_BIN: " ${SPIRV_CROSS_BIN}) + +file(TO_CMAKE_PATH "${SPIRV_CROSS_BIN}" SPIRV_CROSS_BIN_CMAKE_PATH) + +set(SPIRV_VAL_SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/spirv_val.cmake) +set(SPIRV_VAL_SCRIPT_CONTENT [==[ +if(NOT DEFINED VALIDATOR OR "${VALIDATOR}" STREQUAL "") + message(FATAL_ERROR "spirv-val validator path missing") +endif() +if(NOT DEFINED FILE OR "${FILE}" STREQUAL "") + message(FATAL_ERROR "spirv-val file missing") +endif() +set(VAL_FLAGS "${FLAGS}") +if(NOT VAL_FLAGS STREQUAL "") + separate_arguments(VAL_FLAGS NATIVE_COMMAND "${VAL_FLAGS}") +endif() +execute_process(COMMAND "${VALIDATOR}" ${VAL_FLAGS} "${FILE}" RESULT_VARIABLE RESULT OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) +if(NOT RESULT EQUAL 0) + message(STATUS "spirv-val: ${FILE}\n${OUT}\n${ERR}") +endif() +]==]) +file(WRITE ${SPIRV_VAL_SCRIPT} "${SPIRV_VAL_SCRIPT_CONTENT}") project(renodx VERSION 0.1.0 LANGUAGES CXX) @@ -150,6 +202,7 @@ endfunction() function(build_shader_target ADDON ADDON_PATH) set(EMBED_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/${ADDON}.include/embed) unset(SHADER_BINARIES) + unset(SHADER_AUX_OUTPUTS) file(GLOB_RECURSE SHADER_HLSL_SOURCES CONFIGURE_DEPENDS ${ADDON_PATH}/*.hlsl) foreach(FILE ${SHADER_HLSL_SOURCES}) @@ -291,11 +344,7 @@ function(build_shader_target ADDON ADDON_PATH) set(SHADER_COMPILER ${SLANGC_BIN}) unset(SHADER_FLAGS) - set(SHADER_FLAGS ${SHADER_FLAGS} -O3) - set(SHADER_FLAGS ${SHADER_FLAGS} -g0) - set(SHADER_FLAGS ${SHADER_FLAGS} -entry main) - set(SHADER_FLAGS ${SHADER_FLAGS} -Wno-30056) - set(SHADER_FLAGS ${SHADER_FLAGS} -Wno-15205) + set(SHADER_FLAGS ${SLANGC_FLAGS_LIST}) if(SHADER_TARGET AND (SHADER_HASH OR SHADER_NAME)) if(SHADER_TARGET STREQUAL "frag" OR SHADER_TARGET STREQUAL "vert" OR SHADER_TARGET STREQUAL "comp") @@ -316,19 +365,53 @@ function(build_shader_target ADDON ADDON_PATH) endif() if(SHADER_HASH) - add_custom_command( - OUTPUT ${EMBED_FOLDER}/${SHADER_HASH}.spv - COMMAND ${SHADER_COMPILER} ${FILE} -stage ${SHADER_STAGE} ${SHADER_FLAGS} -target spirv -o ${EMBED_FOLDER}/${SHADER_HASH}.spv - DEPENDS ${HLSL_DEPENDENCIES} - ) + if(SPIRV_VAL_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_HASH}.spv + COMMAND ${SHADER_COMPILER} ${FILE} -stage ${SHADER_STAGE} ${SHADER_FLAGS} -target spirv -o ${EMBED_FOLDER}/${SHADER_HASH}.spv + COMMAND ${CMAKE_COMMAND} -DVALIDATOR="${SPIRV_VAL_BIN}" -DFILE="${EMBED_FOLDER}/${SHADER_HASH}.spv" -DFLAGS="${SPIRV_VAL_FLAGS_STR}" -P ${SPIRV_VAL_SCRIPT} + DEPENDS ${HLSL_DEPENDENCIES} + ) + else() + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_HASH}.spv + COMMAND ${SHADER_COMPILER} ${FILE} -stage ${SHADER_STAGE} ${SHADER_FLAGS} -target spirv -o ${EMBED_FOLDER}/${SHADER_HASH}.spv + DEPENDS ${HLSL_DEPENDENCIES} + ) + endif() list(APPEND SHADER_BINARIES ${EMBED_FOLDER}/${SHADER_HASH}.spv) + if(SPIRV_DIS_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_HASH}.spvasm + COMMAND "${SPIRV_DIS_BIN}" "${EMBED_FOLDER}/${SHADER_HASH}.spv" -o "${EMBED_FOLDER}/${SHADER_HASH}.spvasm" + DEPENDS ${EMBED_FOLDER}/${SHADER_HASH}.spv + ) + list(APPEND SHADER_AUX_OUTPUTS ${EMBED_FOLDER}/${SHADER_HASH}.spvasm) + endif() else() - add_custom_command( - OUTPUT ${EMBED_FOLDER}/${SHADER_NAME}.spv - COMMAND ${SHADER_COMPILER} ${FILE} -stage ${SHADER_STAGE} ${SHADER_FLAGS} -target spirv -o ${EMBED_FOLDER}/${SHADER_NAME}.spv - DEPENDS ${HLSL_DEPENDENCIES} - ) + if(SPIRV_VAL_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_NAME}.spv + COMMAND ${SHADER_COMPILER} ${FILE} -stage ${SHADER_STAGE} ${SHADER_FLAGS} -target spirv -o ${EMBED_FOLDER}/${SHADER_NAME}.spv + COMMAND ${CMAKE_COMMAND} -DVALIDATOR="${SPIRV_VAL_BIN}" -DFILE="${EMBED_FOLDER}/${SHADER_NAME}.spv" -DFLAGS="${SPIRV_VAL_FLAGS_STR}" -P ${SPIRV_VAL_SCRIPT} + DEPENDS ${HLSL_DEPENDENCIES} + ) + else() + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_NAME}.spv + COMMAND ${SHADER_COMPILER} ${FILE} -stage ${SHADER_STAGE} ${SHADER_FLAGS} -target spirv -o ${EMBED_FOLDER}/${SHADER_NAME}.spv + DEPENDS ${HLSL_DEPENDENCIES} + ) + endif() list(APPEND SHADER_BINARIES ${EMBED_FOLDER}/${SHADER_NAME}.spv) + if(SPIRV_DIS_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_NAME}.spvasm + COMMAND "${SPIRV_DIS_BIN}" "${EMBED_FOLDER}/${SHADER_NAME}.spv" -o "${EMBED_FOLDER}/${SHADER_NAME}.spvasm" + DEPENDS ${EMBED_FOLDER}/${SHADER_NAME}.spv + ) + list(APPEND SHADER_AUX_OUTPUTS ${EMBED_FOLDER}/${SHADER_NAME}.spvasm) + endif() endif() else() string(SUBSTRING ${SHADER_TARGET} 3 1 SHADER_TARGET_MAJOR) @@ -441,11 +524,53 @@ function(build_shader_target ADDON ADDON_PATH) endif() if(SHADER_HASH) - configure_file(${FILE} ${EMBED_FOLDER}/${SHADER_HASH}.spv COPYONLY) + if(SPIRV_VAL_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_HASH}.spv + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FILE} ${EMBED_FOLDER}/${SHADER_HASH}.spv + COMMAND ${CMAKE_COMMAND} -DVALIDATOR="${SPIRV_VAL_BIN}" -DFILE="${EMBED_FOLDER}/${SHADER_HASH}.spv" -DFLAGS="${SPIRV_VAL_FLAGS_STR}" -P ${SPIRV_VAL_SCRIPT} + DEPENDS ${FILE} + ) + else() + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_HASH}.spv + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FILE} ${EMBED_FOLDER}/${SHADER_HASH}.spv + DEPENDS ${FILE} + ) + endif() list(APPEND SHADER_BINARIES ${EMBED_FOLDER}/${SHADER_HASH}.spv) + if(SPIRV_DIS_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_HASH}.spvasm + COMMAND "${SPIRV_DIS_BIN}" "${EMBED_FOLDER}/${SHADER_HASH}.spv" -o "${EMBED_FOLDER}/${SHADER_HASH}.spvasm" + DEPENDS ${FILE} + ) + list(APPEND SHADER_AUX_OUTPUTS ${EMBED_FOLDER}/${SHADER_HASH}.spvasm) + endif() elseif(SHADER_NAME) - configure_file(${FILE} ${EMBED_FOLDER}/${SHADER_NAME}.spv COPYONLY) + if(SPIRV_VAL_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_NAME}.spv + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FILE} ${EMBED_FOLDER}/${SHADER_NAME}.spv + COMMAND ${CMAKE_COMMAND} -DVALIDATOR="${SPIRV_VAL_BIN}" -DFILE="${EMBED_FOLDER}/${SHADER_NAME}.spv" -DFLAGS="${SPIRV_VAL_FLAGS_STR}" -P ${SPIRV_VAL_SCRIPT} + DEPENDS ${FILE} + ) + else() + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_NAME}.spv + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FILE} ${EMBED_FOLDER}/${SHADER_NAME}.spv + DEPENDS ${FILE} + ) + endif() list(APPEND SHADER_BINARIES ${EMBED_FOLDER}/${SHADER_NAME}.spv) + if(SPIRV_DIS_BIN) + add_custom_command( + OUTPUT ${EMBED_FOLDER}/${SHADER_NAME}.spvasm + COMMAND "${SPIRV_DIS_BIN}" "${EMBED_FOLDER}/${SHADER_NAME}.spv" -o "${EMBED_FOLDER}/${SHADER_NAME}.spvasm" + DEPENDS ${FILE} + ) + list(APPEND SHADER_AUX_OUTPUTS ${EMBED_FOLDER}/${SHADER_NAME}.spvasm) + endif() endif() endforeach() @@ -530,7 +655,7 @@ function(build_shader_target ADDON ADDON_PATH) endif() list(JOIN SHADER_HEADER_FILES " " SHADER_HEADER_FILE_ITEMS) - add_custom_target(${ADDON}-shaders DEPENDS ${SHADER_HEADER_FILES}) + add_custom_target(${ADDON}-shaders DEPENDS ${SHADER_HEADER_FILES} ${SHADER_AUX_OUTPUTS}) if(SHADERS_H_OUTPUT) file(GENERATE OUTPUT ${EMBED_FOLDER}/shaders.h CONTENT "${SHADERS_H_OUTPUT}") @@ -593,6 +718,7 @@ foreach(FILE ${ADDON_FILES}) generate_resource_rc(${ADDON}) add_dependencies(${ADDON} ${ADDON}-shaders) add_dependencies(${ADDON} detours) + target_compile_definitions(${ADDON} PRIVATE RENODX_SLANGC_BIN_PATH="${SLANGC_BIN_CMAKE_PATH}" RENODX_SLANGC_FLAGS="${SLANGC_FLAGS_STR}" RENODX_SPIRV_CROSS_BIN_PATH="${SPIRV_CROSS_BIN_CMAKE_PATH}" RENODX_SPIRV_CROSS_FLAGS="${SPIRV_CROSS_FLAGS_STR}") target_link_libraries(${ADDON} PRIVATE detours) target_include_directories(${ADDON} SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/external/Detours/include) target_include_directories(${ADDON} SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/external/frozen/include) diff --git a/src/addons/devkit/addon.cpp b/src/addons/devkit/addon.cpp index cde83d991..3f5398e4f 100644 --- a/src/addons/devkit/addon.cpp +++ b/src/addons/devkit/addon.cpp @@ -41,6 +41,7 @@ #include "../../utils/pipeline_layout.hpp" #include "../../utils/shader.hpp" #include "../../utils/shader_compiler_directx.hpp" +#include "../../utils/shader_compiler_slang.hpp" #include "../../utils/shader_compiler_watcher.hpp" #include "../../utils/shader_decompiler_dxc.hpp" #include "../../utils/shader_dump.hpp" @@ -322,6 +323,8 @@ bool ComputeDisassemblyForShaderDetails(reshade::api::device* device, DeviceData shader_details->disassembly = std::string( shader_details->shader_data.data(), shader_details->shader_data.data() + shader_details->shader_data.size()); + } else if (device->get_api() == reshade::api::device_api::vulkan) { + shader_details->disassembly = renodx::utils::shader::compiler::slang::DisassembleSpirv(shader_details->shader_data, shader_details->shader_hash); } else { throw std::exception("Unsupported device API."); } @@ -2535,11 +2538,13 @@ void RenderCapturePane(reshade::api::device* device, DeviceData* data) { } // Creates a selectable with the given label that jumps to the specified snapshot index -inline void CreateDrawIndexLink(const std::string& label, int draw_index) { +inline bool CreateDrawIndexLink(const std::string& label, int draw_index) { if (ImGui::TextLink(label.c_str())) { setting_nav_item = 0; // Snapshot is the first nav item pending_draw_index_focus = draw_index; + return true; } + return false; } enum ShaderPaneColumns : uint8_t { @@ -2714,8 +2719,9 @@ void RenderShadersPane(reshade::api::device* device, DeviceData* data) { if (ImGui::TableSetColumnIndex(SHADER_PANE_COLUMN_SNAPSHOT)) { // Snapshot ImGui::PushID(cell_index_id++); if (snapshot_index != -1) { - MakeSelectionCurrent(selection); - CreateDrawIndexLink(std::format("{:03}", snapshot_index), snapshot_index); + if (CreateDrawIndexLink(std::format("{:03}", snapshot_index), snapshot_index)) { + MakeSelectionCurrent(selection); + } } ImGui::PopID(); } @@ -3074,7 +3080,9 @@ void RenderShaderViewLive(reshade::api::device* device, DeviceData* data, Shader if (shader_details->disk_shader.has_value()) { if (!shader_details->disk_shader->IsCompilationOK()) { live_string = shader_details->disk_shader->GetCompilationException().what(); - } else if (shader_details->disk_shader->is_hlsl || shader_details->disk_shader->is_glsl) { + } else if (shader_details->disk_shader->is_hlsl + || shader_details->disk_shader->is_glsl + || shader_details->disk_shader->is_slang) { try { live_string = renodx::utils::path::ReadTextFile(shader_details->disk_shader->file_path); } catch (std::exception& e) { @@ -3133,6 +3141,10 @@ void RenderShaderViewDecompilation(reshade::api::device* device, DeviceData* dat { .flatten = true, }); + } else if (device->get_api() == reshade::api::device_api::vulkan) { + shader_details->decompilation = renodx::utils::shader::compiler::slang::DecompileSpirvToGlsl( + shader_details->shader_data, + shader_details->shader_hash); } else if (device->get_api() == reshade::api::device_api::opengl) { shader_details->disassembly = std::string( shader_details->shader_data.data(), diff --git a/src/utils/shader_compiler_slang.hpp b/src/utils/shader_compiler_slang.hpp new file mode 100644 index 000000000..4feff119a --- /dev/null +++ b/src/utils/shader_compiler_slang.hpp @@ -0,0 +1,534 @@ +/* + * Copyright (C) 2024 Carlos Lopez + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "./path.hpp" + +namespace renodx::utils::shader::compiler::slang { + +namespace internal { + +static std::shared_mutex mutex_slangc_path; +static std::filesystem::path cached_slangc_path; +static std::atomic_uint32_t compile_counter = 0; +static std::optional cached_spirv_dis; +static std::optional cached_spirv_cross; + +inline std::wstring ToWide(const char* value, UINT code_page = CP_UTF8) { + if (value == nullptr || value[0] == '\0') return {}; + const int wide_char_length = MultiByteToWideChar(code_page, 0, value, -1, nullptr, 0); + if (wide_char_length == 0) return {}; + std::wstring wide_string(wide_char_length, L'\0'); + MultiByteToWideChar(code_page, 0, value, -1, wide_string.data(), wide_char_length); + wide_string.resize(wide_char_length - 1); + return wide_string; +} + +inline std::wstring QuoteArg(const std::wstring& arg) { + if (arg.empty()) return L"\"\""; + if (arg.find_first_of(L" \t\"") == std::wstring::npos) return arg; + std::wstring escaped = L"\""; + for (const auto ch : arg) { + if (ch == L'"') { + escaped += L'\\'; + } + escaped += ch; + } + escaped += L"\""; + return escaped; +} + +inline std::wstring JoinArgs(const std::vector& args) { + std::wstring output; + for (size_t i = 0; i < args.size(); ++i) { + if (i != 0) output += L" "; + output += QuoteArg(args[i]); + } + return output; +} + +struct ProcessResult { + DWORD exit_code = 0; + std::string output; +}; + +inline ProcessResult RunProcess(const std::wstring& command_line, const std::filesystem::path& working_dir) { + SECURITY_ATTRIBUTES security_attributes{}; + security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); + security_attributes.bInheritHandle = TRUE; + + HANDLE read_pipe = nullptr; + HANDLE write_pipe = nullptr; + if (!CreatePipe(&read_pipe, &write_pipe, &security_attributes, 0)) { + throw std::exception("Failed to create slangc pipe."); + } + if (!SetHandleInformation(read_pipe, HANDLE_FLAG_INHERIT, 0)) { + CloseHandle(read_pipe); + CloseHandle(write_pipe); + throw std::exception("Failed to configure slangc pipe."); + } + + STARTUPINFOW startup_info{}; + startup_info.cb = sizeof(STARTUPINFOW); + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdOutput = write_pipe; + startup_info.hStdError = write_pipe; + + PROCESS_INFORMATION process_info{}; + std::wstring command_line_mutable = command_line; + + BOOL created = CreateProcessW( + nullptr, + command_line_mutable.data(), + nullptr, + nullptr, + TRUE, + CREATE_NO_WINDOW, + nullptr, + working_dir.empty() ? nullptr : working_dir.wstring().c_str(), + &startup_info, + &process_info); + + CloseHandle(write_pipe); + + if (!created) { + CloseHandle(read_pipe); + throw std::exception("Failed to launch slangc."); + } + + std::string output; + std::array buffer{}; + DWORD bytes_read = 0; + while (ReadFile(read_pipe, buffer.data(), static_cast(buffer.size()), &bytes_read, nullptr) && bytes_read > 0) { + output.append(buffer.data(), bytes_read); + } + + WaitForSingleObject(process_info.hProcess, INFINITE); + DWORD exit_code = 0; + GetExitCodeProcess(process_info.hProcess, &exit_code); + + CloseHandle(read_pipe); + CloseHandle(process_info.hProcess); + CloseHandle(process_info.hThread); + + return ProcessResult{.exit_code = exit_code, .output = output}; +} + +inline std::filesystem::path FindSlangcPath() { + { + std::shared_lock lock(mutex_slangc_path); + if (!cached_slangc_path.empty()) return cached_slangc_path; + } + + std::unique_lock lock(mutex_slangc_path); + if (!cached_slangc_path.empty()) return cached_slangc_path; + +#ifdef RENODX_SLANGC_BIN_PATH + if (!std::string(RENODX_SLANGC_BIN_PATH).empty()) { + auto cmake_path = std::filesystem::path(RENODX_SLANGC_BIN_PATH); + if (std::filesystem::exists(cmake_path)) { + cached_slangc_path = cmake_path; + return cached_slangc_path; + } + } +#endif + + auto try_candidate = [&](const std::filesystem::path& candidate) { + if (!candidate.empty() && std::filesystem::exists(candidate)) { + cached_slangc_path = candidate; + return true; + } + return false; + }; + + auto try_env = [&](const wchar_t* name) { + wchar_t* env_path = nullptr; + size_t env_len = 0; + if (_wdupenv_s(&env_path, &env_len, name) != 0 || env_path == nullptr) { + return false; + } + std::filesystem::path candidate = env_path; + free(env_path); + return try_candidate(candidate); + }; + + if (try_env(L"SLANGC_BIN")) return cached_slangc_path; + if (try_env(L"SLANGC")) return cached_slangc_path; + + try { + if (try_candidate(std::filesystem::current_path() / "slangc.exe")) return cached_slangc_path; + } catch (...) { + } + + std::array module_path{}; + GetModuleFileNameW(nullptr, module_path.data(), static_cast(module_path.size())); + std::filesystem::path search_root = std::filesystem::path(module_path.data()).parent_path(); + for (int i = 0; i < 4; ++i) { + if (try_candidate(search_root / "slangc.exe")) return cached_slangc_path; + if (try_candidate(search_root / "bin" / "slangc.exe")) return cached_slangc_path; + if (!search_root.has_parent_path()) break; + search_root = search_root.parent_path(); + } + + throw std::exception("Could not locate slangc.exe. Set SLANGC_BIN or place it next to the addon or in a bin folder."); +} + +inline std::filesystem::path GetTempOutputPath(const std::wstring& extension) { + auto output_dir = renodx::utils::path::GetOutputPath(); + if (!std::filesystem::exists(output_dir)) { + std::filesystem::create_directory(output_dir); + } + output_dir /= "slang"; + if (!std::filesystem::exists(output_dir)) { + std::filesystem::create_directory(output_dir); + } + + const uint32_t counter = compile_counter.fetch_add(1); + std::array filename{}; + swprintf_s(filename.data(), filename.size(), L"slang_%08X", counter); + output_dir /= filename.data(); + output_dir += extension; + return output_dir; +} + +inline std::string ToLower(std::string input) { + std::transform(input.begin(), input.end(), input.begin(), [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + return input; +} + +inline void AppendFlagsFromString(const std::string& flags, std::vector& args) { + std::istringstream stream(flags); + std::string token; + while (stream >> token) { + args.emplace_back(ToWide(token.c_str())); + } +} + +inline void AppendDefaultFlags(std::vector& args) { +#ifdef RENODX_SLANGC_FLAGS + AppendFlagsFromString(RENODX_SLANGC_FLAGS, args); +#else + args.emplace_back(L"-O3"); + args.emplace_back(L"-g0"); + args.emplace_back(L"-entry"); + args.emplace_back(L"main"); + args.emplace_back(L"-Wno-30056"); + args.emplace_back(L"-Wno-15205"); +#endif +} + +inline std::optional FindSpirvDisPath() { + if (cached_spirv_dis.has_value()) return cached_spirv_dis; + + auto try_candidate = [&](const std::filesystem::path& candidate) { + if (!candidate.empty() && std::filesystem::exists(candidate)) { + cached_spirv_dis = candidate; + return true; + } + return false; + }; + + auto try_env = [&](const wchar_t* name) { + wchar_t* env_path = nullptr; + size_t env_len = 0; + if (_wdupenv_s(&env_path, &env_len, name) != 0 || env_path == nullptr) { + return false; + } + std::filesystem::path candidate = env_path; + free(env_path); + return try_candidate(candidate); + }; + + if (try_env(L"SPIRV_DIS")) return cached_spirv_dis; + + wchar_t* vulkan_sdk = nullptr; + size_t vulkan_sdk_len = 0; + if (_wdupenv_s(&vulkan_sdk, &vulkan_sdk_len, L"VULKAN_SDK") == 0 && vulkan_sdk != nullptr) { + std::filesystem::path sdk_path = vulkan_sdk; + free(vulkan_sdk); + if (try_candidate(sdk_path / "Bin" / "spirv-dis.exe")) return cached_spirv_dis; + } + + try { + if (try_candidate(std::filesystem::current_path() / "spirv-dis.exe")) return cached_spirv_dis; + } catch (...) { + } + + std::array module_path{}; + GetModuleFileNameW(nullptr, module_path.data(), static_cast(module_path.size())); + std::filesystem::path search_root = std::filesystem::path(module_path.data()).parent_path(); + for (int i = 0; i < 4; ++i) { + if (try_candidate(search_root / "spirv-dis.exe")) return cached_spirv_dis; + if (try_candidate(search_root / "bin" / "spirv-dis.exe")) return cached_spirv_dis; + if (!search_root.has_parent_path()) break; + search_root = search_root.parent_path(); + } + + return std::nullopt; +} + +inline std::optional FindSpirvCrossPath() { + if (cached_spirv_cross.has_value()) return cached_spirv_cross; + + auto try_candidate = [&](const std::filesystem::path& candidate) { + if (!candidate.empty() && std::filesystem::exists(candidate)) { + cached_spirv_cross = candidate; + return true; + } + return false; + }; + +#ifdef RENODX_SPIRV_CROSS_BIN_PATH + if (!std::string(RENODX_SPIRV_CROSS_BIN_PATH).empty()) { + auto cmake_path = std::filesystem::path(RENODX_SPIRV_CROSS_BIN_PATH); + if (std::filesystem::exists(cmake_path)) { + cached_spirv_cross = cmake_path; + return cached_spirv_cross; + } + } +#endif + + wchar_t* vulkan_sdk = nullptr; + size_t vulkan_sdk_len = 0; + if (_wdupenv_s(&vulkan_sdk, &vulkan_sdk_len, L"VULKAN_SDK") == 0 && vulkan_sdk != nullptr) { + std::filesystem::path sdk_path = vulkan_sdk; + free(vulkan_sdk); + if (try_candidate(sdk_path / "Bin" / "spirv-cross.exe")) return cached_spirv_cross; + } + + try { + if (try_candidate(std::filesystem::current_path() / "spirv-cross.exe")) return cached_spirv_cross; + } catch (...) { + } + + std::array module_path{}; + GetModuleFileNameW(nullptr, module_path.data(), static_cast(module_path.size())); + std::filesystem::path search_root = std::filesystem::path(module_path.data()).parent_path(); + for (int i = 0; i < 4; ++i) { + if (try_candidate(search_root / "spirv-cross.exe")) return cached_spirv_cross; + if (try_candidate(search_root / "bin" / "spirv-cross.exe")) return cached_spirv_cross; + if (!search_root.has_parent_path()) break; + search_root = search_root.parent_path(); + } + + return std::nullopt; +} + +} // namespace internal + +inline void AppendSpirvCrossFlags(std::vector& args) { +#ifdef RENODX_SPIRV_CROSS_FLAGS + internal::AppendFlagsFromString(RENODX_SPIRV_CROSS_FLAGS, args); +#else + args.emplace_back(L"--version"); + args.emplace_back(L"450"); + args.emplace_back(L"--vulkan-semantics"); + args.emplace_back(L"--no-es"); +#endif +} + +inline std::string DisassembleSpirv(std::span data, uint32_t shader_hash) { + auto spirv_dis = internal::FindSpirvDisPath(); + if (!spirv_dis.has_value()) { + throw std::exception("spirv-dis.exe not found."); + } + + auto output_dir = renodx::utils::path::GetOutputPath(); + output_dir /= "spirv"; + std::filesystem::create_directories(output_dir); + + std::array hash_string{}; + swprintf_s(hash_string.data(), hash_string.size(), L"0x%08X", shader_hash); + + auto spv_path = output_dir / hash_string.data(); + spv_path += L".spv"; + auto spvasm_path = output_dir / hash_string.data(); + spvasm_path += L".spvasm"; + + std::vector data_copy(data.begin(), data.end()); + renodx::utils::path::WriteBinaryFile(spv_path, data_copy); + + std::wstring command_line = L"\""; + command_line += spirv_dis->wstring(); + command_line += L"\" \""; + command_line += spv_path.wstring(); + command_line += L"\" -o \""; + command_line += spvasm_path.wstring(); + command_line += L"\""; + + auto result = internal::RunProcess(command_line, output_dir); + if (result.exit_code != 0) { + std::filesystem::remove(spv_path); + std::filesystem::remove(spvasm_path); + if (result.output.empty()) { + throw std::exception("spirv-dis failed."); + } + throw std::exception(result.output.c_str()); + } + + std::string disassembly = renodx::utils::path::ReadTextFile(spvasm_path); + std::filesystem::remove(spv_path); + std::filesystem::remove(spvasm_path); + if (disassembly.empty()) { + throw std::exception("spirv-dis output is empty."); + } + return disassembly; +} + +inline std::string DecompileSpirvToGlsl(std::span data, uint32_t shader_hash) { + auto spirv_cross = internal::FindSpirvCrossPath(); + if (!spirv_cross.has_value()) { + throw std::exception("spirv-cross.exe not found."); + } + + auto output_dir = renodx::utils::path::GetOutputPath(); + output_dir /= "spirv_cross"; + std::filesystem::create_directories(output_dir); + + std::array hash_string{}; + swprintf_s(hash_string.data(), hash_string.size(), L"0x%08X", shader_hash); + + auto spv_path = output_dir / hash_string.data(); + spv_path += L".spv"; + + std::vector data_copy(data.begin(), data.end()); + renodx::utils::path::WriteBinaryFile(spv_path, data_copy); + + std::vector args; + args.emplace_back(spv_path.wstring()); + AppendSpirvCrossFlags(args); + + const std::wstring command_line = internal::QuoteArg(spirv_cross->wstring()) + L" " + internal::JoinArgs(args); + auto result = internal::RunProcess(command_line, output_dir); + std::filesystem::remove(spv_path); + + if (result.exit_code != 0) { + if (result.output.empty()) { + throw std::exception("spirv-cross failed."); + } + throw std::exception(result.output.c_str()); + } + + if (result.output.empty()) { + throw std::exception("spirv-cross output is empty."); + } + + return result.output; +} + +inline std::vector CompileShaderFromFile( + LPCWSTR file_path, + LPCSTR shader_target, + const std::vector>& defines = {}) { + if (file_path == nullptr || shader_target == nullptr) { + throw std::exception("Missing slang compile arguments."); + } + + const std::filesystem::path source_path = file_path; + if (!std::filesystem::exists(source_path)) { + throw std::exception("Slang source file not found."); + } + + const std::string target = internal::ToLower(shader_target); + bool is_stage_target = false; + std::wstring stage; + + if (target == "frag") { + is_stage_target = true; + stage = L"fragment"; + } else if (target == "vert") { + is_stage_target = true; + stage = L"vertex"; + } else if (target == "comp") { + is_stage_target = true; + stage = L"compute"; + } + + std::wstring output_extension = is_stage_target ? L".spv" : L".cso"; + auto output_path = internal::GetTempOutputPath(output_extension); + + std::vector args; + args.emplace_back(source_path.wstring()); + internal::AppendDefaultFlags(args); + args.emplace_back(L"-I"); + args.emplace_back(source_path.parent_path().wstring()); + + if (is_stage_target) { + args.emplace_back(L"-stage"); + args.emplace_back(stage); + args.emplace_back(L"-target"); + args.emplace_back(L"spirv"); + } else { + if (strlen(shader_target) < 5 || shader_target[2] != '_' || shader_target[4] != '_') { + throw std::exception("Invalid slang target profile."); + } + args.emplace_back(L"-profile"); + args.emplace_back(std::wstring(shader_target, shader_target + strlen(shader_target))); + + char major = shader_target[3]; + if (major >= '6') { + args.emplace_back(L"-target"); + args.emplace_back(L"dxil"); + } else { + args.emplace_back(L"-target"); + args.emplace_back(L"dxbc"); + } + } + + for (const auto& [key, value] : defines) { + if (key.empty()) continue; + std::wstring definition = L"-D"; + definition.append(std::wstring(key.begin(), key.end())); + if (!value.empty()) { + definition.push_back(L'='); + definition.append(std::wstring(value.begin(), value.end())); + } + args.emplace_back(definition); + } + + args.emplace_back(L"-o"); + args.emplace_back(output_path.wstring()); + + const auto slangc_path = internal::FindSlangcPath(); + const std::wstring command_line = internal::QuoteArg(slangc_path.wstring()) + L" " + internal::JoinArgs(args); + const auto result = internal::RunProcess(command_line, source_path.parent_path()); + + if (result.exit_code != 0) { + if (result.output.empty()) { + throw std::exception("slangc failed."); + } + throw std::exception(result.output.c_str()); + } + + auto output_data = renodx::utils::path::ReadBinaryFile(output_path); + if (output_data.empty()) { + throw std::exception("slangc did not produce output."); + } + std::filesystem::remove(output_path); + return output_data; +} + +} // namespace renodx::utils::shader::compiler::slang diff --git a/src/utils/shader_compiler_watcher.hpp b/src/utils/shader_compiler_watcher.hpp index be5faf7a8..a92505123 100644 --- a/src/utils/shader_compiler_watcher.hpp +++ b/src/utils/shader_compiler_watcher.hpp @@ -6,7 +6,9 @@ #include #include +#ifndef NOMINMAX #define NOMINMAX +#endif #include #include @@ -21,6 +23,7 @@ #include "./format.hpp" #include "./path.hpp" #include "./shader_compiler_directx.hpp" +#include "./shader_compiler_slang.hpp" namespace renodx::utils::shader::compiler::watcher { @@ -30,6 +33,7 @@ struct CustomShader { std::variant> compilation; bool is_hlsl = false; bool is_glsl = false; + bool is_slang = false; std::filesystem::path file_path; uint32_t shader_hash; bool removed = false; @@ -47,14 +51,28 @@ struct CustomShader { } [[nodiscard]] std::string GetFileAlias() const { - if (!is_hlsl) return ""; - static const auto CHARACTERS_TO_REMOVE_FROM_END = std::string("0x12345678.xx_x_x.hlsl").length(); auto filename = file_path.filename().string(); - filename.erase(filename.length() - (std::min)(CHARACTERS_TO_REMOVE_FROM_END, filename.length())); - if (filename.ends_with("_")) { - filename.erase(filename.length() - 1); + if (is_hlsl) { + static const auto CHARACTERS_TO_REMOVE_FROM_END = std::string("0x12345678.xx_x_x.hlsl").length(); + filename.erase(filename.length() - (std::min)(CHARACTERS_TO_REMOVE_FROM_END, filename.length())); + if (filename.ends_with("_")) { + filename.erase(filename.length() - 1); + } + return filename; + } + if (is_slang) { + if (auto suffix_pos = filename.rfind(".slang"); suffix_pos != std::string::npos) { + filename.erase(suffix_pos); + } + if (auto target_pos = filename.rfind('.'); target_pos != std::string::npos) { + filename.erase(target_pos); + } + if (auto hash_pos = filename.rfind("_0x"); hash_pos != std::string::npos) { + filename.erase(hash_pos); + } + return filename; } - return filename; + return ""; } }; @@ -112,7 +130,8 @@ static bool CompileCustomShaders() { const bool is_cso = entry_path.extension().compare(".cso") == 0; const bool is_glsl = entry_path.extension().compare(".glsl") == 0; const bool is_spv = entry_path.extension().compare(".spv") == 0; - if (!is_hlsl && !is_cso && !is_spv && !is_glsl) continue; + const bool is_slang = entry_path.extension().compare(".slang") == 0; + if (!is_hlsl && !is_cso && !is_spv && !is_glsl && !is_slang) continue; auto basename = entry_path.stem().string(); std::string hash_string; @@ -127,6 +146,17 @@ static bool CompileCustomShaders() { if (shader_target[4] != '_') continue; // uint32_t versionMajor = shader_target[3] - '0'; hash_string = basename.substr(length - strlen("12345678.xx_x_x"), 8); + } else if (is_slang) { + const auto& filename = entry_path.filename().string(); + const auto slang_suffix = filename.rfind(".slang"); + if (slang_suffix == std::string::npos || slang_suffix == 0) continue; + const auto before_suffix = filename.substr(0, slang_suffix); + const auto target_dot = before_suffix.rfind('.'); + if (target_dot == std::string::npos || target_dot == 0) continue; + shader_target = before_suffix.substr(target_dot + 1); + const auto shader_name = before_suffix.substr(0, target_dot); + if (shader_name.size() < 10 || shader_name.rfind("0x") != shader_name.size() - 10) continue; + hash_string = shader_name.substr(shader_name.size() - 8, 8); } else if (is_cso || is_spv || is_glsl) { // Binaries files must start with 0x12345678. The rest of the basename is ignored. if (basename.size() < 10) { @@ -168,6 +198,7 @@ static bool CompileCustomShaders() { CustomShader custom_shader = { .is_hlsl = is_hlsl, .is_glsl = is_glsl, + .is_slang = is_slang, .file_path = entry_path, }; @@ -200,6 +231,35 @@ static bool CompileCustomShaders() { reshade::log::message(reshade::log::level::warning, s.str().c_str()); } + } else if (is_slang) { + { + std::stringstream s; + s << "loadCustomShaders(Compiling slang file: "; + s << entry_path.string(); + s << ", hash: " << PRINT_CRC32(shader_hash); + s << ", target: " << shader_target; + s << ")"; + reshade::log::message(reshade::log::level::debug, s.str().c_str()); + } + + try { + custom_shader.compilation = renodx::utils::shader::compiler::slang::CompileShaderFromFile( + entry_path.c_str(), + shader_target.c_str(), + shared_shader_defines); + shader_hashes_processed.emplace(shader_hash); + shader_hashes_failed.erase(shader_hash); + } catch (std::exception& e) { + shader_hashes_failed.emplace(shader_hash); + custom_shader.compilation = e; + std::stringstream s; + s << "loadCustomShaders(Compilation failed: "; + s << entry_path.string(); + s << ", " << custom_shader.GetCompilationException().what(); + s << ")"; + reshade::log::message(reshade::log::level::warning, s.str().c_str()); + } + } else if (is_cso || is_spv || is_glsl) { try { custom_shader.compilation = utils::path::ReadBinaryFile(entry_path); @@ -221,8 +281,10 @@ static bool CompileCustomShaders() { && previous_custom_shader.IsCompilationOK() && custom_shader.GetCompilationData() == previous_custom_shader.GetCompilationData()) { // Same data, update source - previous_custom_shader.is_hlsl = is_hlsl, - previous_custom_shader.file_path = entry_path, + previous_custom_shader.is_hlsl = is_hlsl; + previous_custom_shader.is_glsl = is_glsl; + previous_custom_shader.is_slang = is_slang; + previous_custom_shader.file_path = entry_path; insert_or_replace = false; } } @@ -425,4 +487,4 @@ static void RequestCompile() { internal::shared_compile_pending = true; } -} // namespace renodx::utils::shader::compiler::watcher \ No newline at end of file +} // namespace renodx::utils::shader::compiler::watcher From 6f0f7ca4c69c24816fd852c0de42bf4b9b4fb80c Mon Sep 17 00:00:00 2001 From: Mohanned Hassan Date: Tue, 30 Dec 2025 21:42:29 +0400 Subject: [PATCH 2/2] feat(devkit): Parallelize shader compilation (dxc, fxc and slang), and handle sm5_x files --- src/addons/devkit/addon.cpp | 1 + src/utils/shader_compiler_directx.hpp | 35 ++- src/utils/shader_compiler_watcher.hpp | 382 ++++++++++++++++++++------ 3 files changed, 323 insertions(+), 95 deletions(-) diff --git a/src/addons/devkit/addon.cpp b/src/addons/devkit/addon.cpp index 3f5398e4f..3e2ffb43e 100644 --- a/src/addons/devkit/addon.cpp +++ b/src/addons/devkit/addon.cpp @@ -1430,6 +1430,7 @@ void ActivateShader(reshade::api::device* device, uint32_t shader_hash, std::spa } void LoadDiskShaders(reshade::api::device* device, DeviceData* data, bool activate = true) { + renodx::utils::shader::compiler::watcher::SetDeviceApi(device->get_api()); if (setting_live_reload) { if (!renodx::utils::shader::compiler::watcher::HasChanged()) return; } else { diff --git a/src/utils/shader_compiler_directx.hpp b/src/utils/shader_compiler_directx.hpp index 04651e8ed..b84e19e94 100644 --- a/src/utils/shader_compiler_directx.hpp +++ b/src/utils/shader_compiler_directx.hpp @@ -40,6 +40,9 @@ static HMODULE fxc_compiler_library = nullptr; static HMODULE dxc_compiler_library = nullptr; static std::shared_mutex mutex_fxc_compiler; static std::shared_mutex mutex_dxc_compiler; +static std::once_flag dxc_compiler_library_once; +constexpr bool USE_FXC_MUTEX = true; +constexpr bool USE_DXC_MUTEX = false; class FxcD3DInclude : public ID3DInclude { public: @@ -119,9 +122,9 @@ class FxcD3DInclude : public ID3DInclude { }; inline HMODULE LoadDXCompiler(const std::wstring& path = L"dxcompiler.dll") { - if (dxc_compiler_library == nullptr) { + std::call_once(dxc_compiler_library_once, [&]() { dxc_compiler_library = LoadLibraryW(path.c_str()); - } + }); return dxc_compiler_library; } @@ -132,9 +135,7 @@ inline HMODULE LoadDXCompiler(const std::string& path) { inline HRESULT CreateLibrary(IDxcLibrary** dxc_library) { // HMODULE dxil_loader = LoadLibraryW(L"dxil.dll"); - if (dxc_compiler_library == nullptr) { - dxc_compiler_library = LoadDXCompiler(); - } + dxc_compiler_library = LoadDXCompiler(); if (dxc_compiler_library == nullptr) { return -1; } @@ -146,9 +147,7 @@ inline HRESULT CreateLibrary(IDxcLibrary** dxc_library) { inline HRESULT CreateCompiler(IDxcCompiler** dxc_compiler) { // HMODULE dxil_loader = LoadLibraryW(L"dxil.dll"); - if (dxc_compiler_library == nullptr) { - dxc_compiler_library = LoadLibraryW(L"dxcompiler.dll"); - } + dxc_compiler_library = LoadDXCompiler(); if (dxc_compiler_library == nullptr) { return -1; } @@ -378,7 +377,10 @@ inline std::vector CompileShaderFromFileFXC( // Create a new Custom D3DInclude that supports relative imports auto custom_include = FxcD3DInclude(file_path); - std::unique_lock lock(mutex_fxc_compiler); + std::unique_lock lock(mutex_fxc_compiler, std::defer_lock); + if constexpr (USE_FXC_MUTEX) { + lock.lock(); + } ID3DBlobPtr error_blob; if (FAILED(d3d_compilefromfile( file_path, @@ -411,7 +413,10 @@ inline std::vector CompileShaderFromFileDXC( ID3DBlobPtr out_blob; ID3DBlobPtr error_blob; - std::unique_lock lock(mutex_dxc_compiler); + std::unique_lock lock(mutex_dxc_compiler, std::defer_lock); + if constexpr (USE_DXC_MUTEX) { + lock.lock(); + } if (FAILED(internal::BridgeD3DCompileFromFile( file_path, defines, @@ -444,7 +449,10 @@ inline std::string DisassembleShaderFXC(std::span blob) { if (d3d_disassemble == nullptr) throw std::exception("Could not to load D3DDisassemble in D3DCompiler_47.dll"); // Function may not be thread-safe - std::unique_lock lock(mutex_fxc_compiler); + std::unique_lock lock(mutex_fxc_compiler, std::defer_lock); + if constexpr (USE_FXC_MUTEX) { + lock.lock(); + } ID3DBlobPtr out_blob; if (FAILED(d3d_disassemble( blob.data(), @@ -467,7 +475,10 @@ inline std::string DisassembleShaderDXC(std::span blob) { IDxcBlobEncodingPtr disassembly_text; ID3DBlobPtr disassembly; - std::unique_lock lock(mutex_dxc_compiler); + std::unique_lock lock(mutex_dxc_compiler, std::defer_lock); + if constexpr (USE_DXC_MUTEX) { + lock.lock(); + } if (FAILED(internal::CreateLibrary(&library))) throw std::exception("Could not create library."); if (FAILED(library->CreateBlobWithEncodingFromPinned(blob.data(), blob.size(), CP_ACP, &source))) throw std::exception("Could not prepare blob."); if (FAILED(internal::CreateCompiler(&compiler))) throw std::exception("Could not create compiler object."); diff --git a/src/utils/shader_compiler_watcher.hpp b/src/utils/shader_compiler_watcher.hpp index a92505123..681b22eb4 100644 --- a/src/utils/shader_compiler_watcher.hpp +++ b/src/utils/shader_compiler_watcher.hpp @@ -46,6 +46,10 @@ struct CustomShader { return std::get>(compilation); } + [[nodiscard]] const std::vector& GetCompilationData() const { + return std::get>(compilation); + } + [[nodiscard]] std::exception& GetCompilationException() { return std::get(compilation); } @@ -86,8 +90,12 @@ static std::optional worker_thread; constexpr uint32_t MAX_SHADER_DEFINES = 4; const bool PRECOMPILE_CUSTOM_SHADERS = true; +constexpr bool ENABLE_AB_TOGGLE = false; // For debugging static std::shared_mutex mutex; +static std::mutex compile_mutex; +static std::atomic_int shared_device_api = static_cast(reshade::api::device_api::d3d11); +static std::atomic_int shared_ab_toggle = 0; static std::unordered_map custom_shaders_cache; static std::vector> shared_shader_defines; static std::string live_path; @@ -97,10 +105,45 @@ static HANDLE m_target_dir_handle = INVALID_HANDLE_VALUE; static std::aligned_storage_t<1U << 18, std::max(alignof(FILE_NOTIFY_EXTENDED_INFORMATION), alignof(FILE_NOTIFY_INFORMATION))> watch_buffer; static bool CompileCustomShaders() { - const std::unique_lock lock(mutex); + const std::unique_lock compile_lock(compile_mutex); + struct WorkItem { + std::filesystem::path entry_path; + bool is_hlsl = false; + bool is_glsl = false; + bool is_slang = false; + bool is_cso = false; + bool is_spv = false; + std::string shader_target; + uint32_t shader_hash = 0; + }; + + struct CompileResult { + uint32_t shader_hash = 0; + CustomShader custom_shader = {}; + bool processed = false; + bool failed = false; + }; + + auto ResolveShaderTargetForApi = [&](const std::string& target) { + if (target.size() < 4 || !target.ends_with("_x")) return target; + if (target.find("_5_") == std::string::npos) return target; + const auto device_api = static_cast(shared_device_api.load()); + std::string resolved = target; + resolved[resolved.size() - 1] = + (device_api == reshade::api::device_api::d3d12) ? '1' : '0'; + return resolved; + }; + + std::vector> local_shader_defines; + std::string local_live_path; + { + const std::shared_lock lock(mutex); + local_shader_defines = shared_shader_defines; + local_live_path = live_path; + } std::filesystem::path directory; - if (live_path.empty()) { + if (local_live_path.empty()) { directory = renodx::utils::path::GetOutputPath(); if (!std::filesystem::exists(directory)) { std::filesystem::create_directory(directory); @@ -108,11 +151,12 @@ static bool CompileCustomShaders() { } directory /= "live"; } else { - directory = live_path; + directory = local_live_path; } std::unordered_set shader_hashes_processed = {}; std::unordered_set shader_hashes_failed = {}; std::unordered_set shader_hashes_updated = {}; + std::vector work_items; try { if (!std::filesystem::exists(directory)) { std::filesystem::create_directory(directory); @@ -144,6 +188,7 @@ static bool CompileCustomShaders() { shader_target = basename.substr(length - strlen("xx_x_x"), strlen("xx_x_x")); if (shader_target[2] != '_') continue; if (shader_target[4] != '_') continue; + shader_target = ResolveShaderTargetForApi(shader_target); // uint32_t versionMajor = shader_target[3] - '0'; hash_string = basename.substr(length - strlen("12345678.xx_x_x"), 8); } else if (is_slang) { @@ -183,93 +228,240 @@ static bool CompileCustomShaders() { continue; } - if (shader_hashes_processed.contains(shader_hash)) { - std::stringstream s; - s << "CompileCustomShaders(Ignoring duplicate shader: "; - s << PRINT_CRC32(shader_hash); - s << ", at " << entry_path; - s << ")"; - reshade::log::message(reshade::log::level::warning, s.str().c_str()); - continue; - } - - // Prepare new custom shader entry but hold off unless it actually compiles () - - CustomShader custom_shader = { + work_items.push_back({ + .entry_path = entry_path, .is_hlsl = is_hlsl, .is_glsl = is_glsl, .is_slang = is_slang, - .file_path = entry_path, - }; + .is_cso = is_cso, + .is_spv = is_spv, + .shader_target = shader_target, + .shader_hash = shader_hash, + }); + } + } catch (std::exception& ex) { + reshade::log::message(reshade::log::level::error, ex.what()); + return false; + } - if (is_hlsl) { - { - std::stringstream s; - s << "loadCustomShaders(Compiling file: "; - s << entry_path.string(); - s << ", hash: " << PRINT_CRC32(shader_hash); - s << ", target: " << shader_target; - s << ")"; - reshade::log::message(reshade::log::level::debug, s.str().c_str()); + std::vector results(work_items.size()); + long long compile_ms = -1; + if (!work_items.empty()) { + const auto compile_start = std::chrono::steady_clock::now(); + const size_t max_threads = (std::max)(static_cast(1), + static_cast((std::max)(1u, std::thread::hardware_concurrency() / 2))); + const size_t thread_count = (std::min)(max_threads, work_items.size()); + const bool use_parallel = ENABLE_AB_TOGGLE + ? (shared_ab_toggle.fetch_add(1) % 2) == 1 + : true; + { + std::stringstream s; + s << "CompileCustomShaders(start: items=" << work_items.size() + << ", threads=" << thread_count + << ", mode=" << (use_parallel ? "parallel" : "single") + << ", fxc=" << (renodx::utils::shader::compiler::directx::internal::USE_FXC_MUTEX ? "single" : "parallel") + << ", dxc=" << (renodx::utils::shader::compiler::directx::internal::USE_DXC_MUTEX ? "single" : "parallel") + << ")"; + reshade::log::message(reshade::log::level::debug, s.str().c_str()); + } + + constexpr int kCompileRetryCount = 3; + constexpr int kCompileRetryDelayMs = 25; + + // In case of file locks + auto RetryCompile = [&](auto&& compile_fn, const std::filesystem::path& path, std::variant>& out) -> bool { + for (int attempt = 0; attempt < kCompileRetryCount; ++attempt) { + try { + out = compile_fn(); + if (attempt > 0) { + std::stringstream s; + s << "CompileCustomShaders(RetryCompile: " << path.string() + << ", attempts=" << (attempt + 1) << ")"; + reshade::log::message(reshade::log::level::debug, s.str().c_str()); + } + return true; + } catch (std::exception& e) { + out = e; + if (attempt + 1 < kCompileRetryCount) { + { + std::stringstream s; + s << "CompileCustomShaders(RetryCompile: " << path.string() + << ", attempt=" << (attempt + 1) << "/" << kCompileRetryCount + << ", error=" << e.what() << ")"; + reshade::log::message(reshade::log::level::debug, s.str().c_str()); + } + std::this_thread::sleep_for(std::chrono::milliseconds(kCompileRetryDelayMs)); + continue; + } + { + std::stringstream s; + s << "CompileCustomShaders(RetryCompile: " << path.string() + << ", attempt=" << (attempt + 1) << "/" << kCompileRetryCount + << ", error=" << e.what() << ")"; + reshade::log::message(reshade::log::level::debug, s.str().c_str()); + } + return false; } + } + return false; + }; + + auto process_index = [&](size_t index) { + const auto& item = work_items[index]; + CompileResult result = { + .shader_hash = item.shader_hash, + .custom_shader = { + .is_hlsl = item.is_hlsl, + .is_glsl = item.is_glsl, + .is_slang = item.is_slang, + .file_path = item.entry_path, + }, + }; + if (item.is_hlsl) { try { - custom_shader.compilation = renodx::utils::shader::compiler::directx::CompileShaderFromFile( - entry_path.c_str(), - shader_target.c_str(), - shared_shader_defines); - shader_hashes_processed.emplace(shader_hash); - shader_hashes_failed.erase(shader_hash); + result.processed = RetryCompile( + [&]() { + return renodx::utils::shader::compiler::directx::CompileShaderFromFile( + item.entry_path.c_str(), + item.shader_target.c_str(), + local_shader_defines); + }, + item.entry_path, + result.custom_shader.compilation); + if (!result.processed) { + result.failed = true; + std::stringstream s; + s << "loadCustomShaders(Compilation failed: "; + s << item.entry_path.string(); + s << ", " << result.custom_shader.GetCompilationException().what(); + s << ")"; + reshade::log::message(reshade::log::level::warning, s.str().c_str()); + } } catch (std::exception& e) { - shader_hashes_failed.emplace(shader_hash); - custom_shader.compilation = e; + result.failed = true; + result.custom_shader.compilation = e; std::stringstream s; s << "loadCustomShaders(Compilation failed: "; - s << entry_path.string(); - s << ", " << custom_shader.GetCompilationException().what(); + s << item.entry_path.string(); + s << ", " << result.custom_shader.GetCompilationException().what(); s << ")"; reshade::log::message(reshade::log::level::warning, s.str().c_str()); } - } else if (is_slang) { - { - std::stringstream s; - s << "loadCustomShaders(Compiling slang file: "; - s << entry_path.string(); - s << ", hash: " << PRINT_CRC32(shader_hash); - s << ", target: " << shader_target; - s << ")"; - reshade::log::message(reshade::log::level::debug, s.str().c_str()); - } - + } else if (item.is_slang) { try { - custom_shader.compilation = renodx::utils::shader::compiler::slang::CompileShaderFromFile( - entry_path.c_str(), - shader_target.c_str(), - shared_shader_defines); - shader_hashes_processed.emplace(shader_hash); - shader_hashes_failed.erase(shader_hash); + result.processed = RetryCompile( + [&]() { + return renodx::utils::shader::compiler::slang::CompileShaderFromFile( + item.entry_path.c_str(), + item.shader_target.c_str(), + local_shader_defines); + }, + item.entry_path, + result.custom_shader.compilation); + if (!result.processed) { + result.failed = true; + std::stringstream s; + s << "loadCustomShaders(Compilation failed: "; + s << item.entry_path.string(); + s << ", " << result.custom_shader.GetCompilationException().what(); + s << ")"; + reshade::log::message(reshade::log::level::warning, s.str().c_str()); + } } catch (std::exception& e) { - shader_hashes_failed.emplace(shader_hash); - custom_shader.compilation = e; + result.failed = true; + result.custom_shader.compilation = e; std::stringstream s; s << "loadCustomShaders(Compilation failed: "; - s << entry_path.string(); - s << ", " << custom_shader.GetCompilationException().what(); + s << item.entry_path.string(); + s << ", " << result.custom_shader.GetCompilationException().what(); s << ")"; reshade::log::message(reshade::log::level::warning, s.str().c_str()); } - } else if (is_cso || is_spv || is_glsl) { + } else if (item.is_cso || item.is_spv || item.is_glsl) { try { - custom_shader.compilation = utils::path::ReadBinaryFile(entry_path); - shader_hashes_processed.emplace(shader_hash); - shader_hashes_failed.erase(shader_hash); + result.processed = RetryCompile( + [&]() { + return utils::path::ReadBinaryFile(item.entry_path); + }, + item.entry_path, + result.custom_shader.compilation); + if (!result.processed) { + result.failed = true; + } } catch (std::exception& e) { - custom_shader.compilation = e; - shader_hashes_failed.emplace(shader_hash); - continue; + result.custom_shader.compilation = e; + result.failed = true; + } + } + + results[index] = std::move(result); + }; + + if (use_parallel && thread_count > 1) { + std::atomic_size_t next_index = 0; + auto worker = [&]() { + while (true) { + const size_t index = next_index.fetch_add(1); + if (index >= work_items.size()) { + return; + } + process_index(index); } + }; + + std::vector threads; + threads.reserve(thread_count); + for (size_t i = 0; i < thread_count; ++i) { + threads.emplace_back(worker); + } + for (auto& thread : threads) { + thread.join(); + } + } else { + for (size_t i = 0; i < work_items.size(); ++i) { + process_index(i); + } + } + const auto compile_end = std::chrono::steady_clock::now(); + compile_ms = std::chrono::duration_cast(compile_end - compile_start).count(); + } else { + compile_ms = 0; + } + + size_t processed_count = 0; + size_t failed_count = 0; + for (const auto& result : results) { + if (result.processed) ++processed_count; + if (result.failed) ++failed_count; + } + + { + const std::unique_lock lock(mutex); + for (size_t index = 0; index < results.size(); ++index) { + const auto& item = work_items[index]; + const auto& result = results[index]; + const auto& custom_shader = result.custom_shader; + const uint32_t shader_hash = result.shader_hash; + + if (shader_hashes_processed.contains(shader_hash)) { + std::stringstream s; + s << "CompileCustomShaders(Ignoring duplicate shader: "; + s << PRINT_CRC32(shader_hash); + s << ", at " << item.entry_path; + s << ")"; + reshade::log::message(reshade::log::level::warning, s.str().c_str()); + continue; + } + + if (result.processed) { + shader_hashes_processed.emplace(shader_hash); + shader_hashes_failed.erase(shader_hash); + } + if (result.failed) { + shader_hashes_failed.emplace(shader_hash); } // Find previous entry @@ -281,10 +473,10 @@ static bool CompileCustomShaders() { && previous_custom_shader.IsCompilationOK() && custom_shader.GetCompilationData() == previous_custom_shader.GetCompilationData()) { // Same data, update source - previous_custom_shader.is_hlsl = is_hlsl; - previous_custom_shader.is_glsl = is_glsl; - previous_custom_shader.is_slang = is_slang; - previous_custom_shader.file_path = entry_path; + previous_custom_shader.is_hlsl = custom_shader.is_hlsl; + previous_custom_shader.is_glsl = custom_shader.is_glsl; + previous_custom_shader.is_slang = custom_shader.is_slang; + previous_custom_shader.file_path = custom_shader.file_path; insert_or_replace = false; } } @@ -294,24 +486,44 @@ static bool CompileCustomShaders() { shader_hashes_updated.emplace(shader_hash); } } - } catch (std::exception& ex) { - reshade::log::message(reshade::log::level::error, ex.what()); - return false; - } - for (auto& [shader_hash, custom_shader] : custom_shaders_cache) { - if (!shader_hashes_processed.contains(shader_hash)) { - if (!custom_shader.removed) { - custom_shader.removed = true; - shader_hashes_updated.insert(shader_hash); + for (auto& [shader_hash, custom_shader] : custom_shaders_cache) { + if (!shader_hashes_processed.contains(shader_hash)) { + if (!custom_shader.removed) { + custom_shader.removed = true; + shader_hashes_updated.insert(shader_hash); + } } } - } - if (shader_hashes_updated.size() != 0) { - custom_shaders_count = custom_shaders_cache.size(); - shared_shaders_changed = true; - return true; + if (shader_hashes_updated.size() != 0) { + custom_shaders_count = custom_shaders_cache.size(); + shared_shaders_changed = true; + { + std::stringstream s; + s << "CompileCustomShaders(done: processed=" << processed_count + << ", failed=" << failed_count + << ", updated=" << shader_hashes_updated.size() + << ", cache=" << custom_shaders_cache.size(); + if (compile_ms >= 0) { + s << ", time=" << compile_ms << "ms"; + } + s << ")"; + reshade::log::message(reshade::log::level::debug, s.str().c_str()); + } + return true; + } + } + { + std::stringstream s; + s << "CompileCustomShaders(done: processed=" << processed_count + << ", failed=" << failed_count + << ", updated=0"; + if (compile_ms >= 0) { + s << ", time=" << compile_ms << "ms"; + } + s << ")"; + reshade::log::message(reshade::log::level::debug, s.str().c_str()); } return false; } @@ -478,6 +690,10 @@ static std::string GetLivePath() { return internal::live_path; } +static void SetDeviceApi(reshade::api::device_api device_api) { + internal::shared_device_api.store(static_cast(device_api)); +} + static void SetLivePath(const std::string& live_path) { const std::unique_lock lock(internal::mutex); internal::live_path.assign(live_path);