diff --git a/Plugins/CMakeLists.txt b/Plugins/CMakeLists.txt index 310b96d2..589829f7 100644 --- a/Plugins/CMakeLists.txt +++ b/Plugins/CMakeLists.txt @@ -41,6 +41,7 @@ endif() add_subdirectory(nosReflect) add_subdirectory(nosStrings) add_subdirectory(nosAnimation) +add_subdirectory(nosGraphics) nos_get_targets(PLUGINS_COMMON_EXTERNAL_TARGETS "./External") nos_group_targets("${PLUGINS_COMMON_EXTERNAL_TARGETS}" "External") diff --git a/Plugins/nosGraphics/CMakeLists.txt b/Plugins/nosGraphics/CMakeLists.txt new file mode 100644 index 00000000..470294fc --- /dev/null +++ b/Plugins/nosGraphics/CMakeLists.txt @@ -0,0 +1,26 @@ +set(NOS_VULKAN_NAME "nos.sys.vulkan") +set(NOS_VULKAN_VERSION "6.0") + +set(NOS_TRACK_NAME "nos.track") +set(NOS_TRACK_VERSION "1.9") + +set(MODULE_DEPENDENCIES "${NOS_VULKAN_NAME}-${NOS_VULKAN_VERSION}" "${NOS_TRACK_NAME}-${NOS_TRACK_VERSION}") +set(dep_idx 0) +foreach(module_name_version ${MODULE_DEPENDENCIES}) + # module_name_version: - + string(REPLACE "-" ";" module_name_version ${module_name_version}) + list(GET module_name_version 0 module_name) + list(GET module_name_version 1 module_version) + nos_get_module("${module_name}" "${module_version}" DEP_${dep_idx}) + list(APPEND MODULE_DEPENDENCIES_TARGETS ${DEP_${dep_idx}}) +endforeach() +list(APPEND MODULE_DEPENDENCIES_TARGETS ${NOS_PLUGIN_SDK_TARGET}) + +nos_find_module_path(${NOS_VULKAN_NAME} ${NOS_VULKAN_VERSION} NOS_VULKAN_PATH) + +set(GENERATED_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/Generated") +nos_generate_flatbuffers("${CMAKE_CURRENT_SOURCE_DIR}/Config" "${GENERATED_OUTPUT_DIR}" "cpp" "${NOS_SDK_DIR}/Types;${NOS_VULKAN_PATH}" nosGraphics_generated) + +list(APPEND MODULE_DEPENDENCIES_TARGETS nosGraphics_generated) +set(INCLUDE_FOLDERS "${GENERATED_OUTPUT_DIR}") +nos_add_plugin("nosGraphics" "${MODULE_DEPENDENCIES_TARGETS}" "${INCLUDE_FOLDERS}") \ No newline at end of file diff --git a/Plugins/nosGraphics/Config/BillboardMask.nosdef b/Plugins/nosGraphics/Config/BillboardMask.nosdef new file mode 100644 index 00000000..46d64907 --- /dev/null +++ b/Plugins/nosGraphics/Config/BillboardMask.nosdef @@ -0,0 +1,63 @@ +{ + "nodes": [ + { + "class_name": "BillboardMask", + "menu_info": { + "category": "Rendering", + "display_name": "Billboard Mask" + }, + "node": { + "contents_type": "Job", + "pins": [ + { + "name": "Position", + "type_name": "nos.fb.vec3", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Size", + "type_name": "nos.fb.vec2", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 100.0, + "y": 100.0 + } + }, + { + "name": "RenderView", + "type_name": "nos.graphics.RenderView", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Resolution", + "type_name": "nos.fb.vec2u", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 1920, + "y": 1080 + } + }, + { + "name": "OutRenderTarget", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + }, + { + "name": "OutDepth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "data": { + "format": "D32_SFLOAT" + } + } + ] + } + } + ] +} \ No newline at end of file diff --git a/Plugins/nosGraphics/Config/Graphics.fbs b/Plugins/nosGraphics/Config/Graphics.fbs new file mode 100644 index 00000000..b0a1b663 --- /dev/null +++ b/Plugins/nosGraphics/Config/Graphics.fbs @@ -0,0 +1,46 @@ +include "Common.fbs"; + +namespace nos.graphics; + +table PerspectiveProjection +{ + aspect_ratio: float; + fov_x: float; +} + +table OrthographicProjection +{ + width: float; + height: float; +} + +enum ProjectionType : ubyte +{ + Perspective, + Orthographic +} + +table Projection +{ + clip_planes: nos.fb.vec2 (native_inline); + center_shift: nos.fb.vec2 (native_inline); + projection_type: ProjectionType; + perspective: PerspectiveProjection; + orthographic: OrthographicProjection; +} + +table RenderView +{ + view: nos.fb.mat4(native_inline); + left_handed_projection_matrix: nos.fb.mat4(native_inline); + projection: Projection; +} + +struct Transform +{ + position: nos.fb.vec3d; + rotation: nos.fb.vec3d; + scale: nos.fb.vec3d; +} + +// TODO: Move nos.fb.Transform here too \ No newline at end of file diff --git a/Plugins/nosGraphics/Config/TrackToView.nosdef b/Plugins/nosGraphics/Config/TrackToView.nosdef new file mode 100644 index 00000000..cea910d4 --- /dev/null +++ b/Plugins/nosGraphics/Config/TrackToView.nosdef @@ -0,0 +1,35 @@ +{ + "nodes": [ + { + "class_name": "TrackToView", + "menu_info": { + "category": "Rendering", + "display_name": "Track To View" + }, + "node": { + "contents_type": "Job", + "pins": [ + { + "name": "Track", + "type_name": "nos.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Clip", + "type_name": "nos.fb.vec2", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": {"x": 0.1, "y": 10000.0} + }, + { + "name": "View", + "type_name": "nos.graphics.RenderView", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/Plugins/nosGraphics/Graphics.noscfg b/Plugins/nosGraphics/Graphics.noscfg new file mode 100644 index 00000000..b2ab41e2 --- /dev/null +++ b/Plugins/nosGraphics/Graphics.noscfg @@ -0,0 +1,28 @@ +{ + "info": { + "id": { + "name": "nos.graphics", + "version": "0.1.0" + }, + "description": "Types & Nodes for graphics operations.", + "display_name": "Graphics", + "category": "Graphics", + "dependencies": [ + { + "name": "nos.sys.vulkan", + "version": "6.0" + }, + { + "name": "nos.track", + "version": "1.9" + } + ] + }, + "custom_types": [ "Config/Graphics.fbs" ], + "node_definitions": [ + "Config/BillboardMask.nosdef", + "Config/TrackToView.nosdef" + ], + "binary_path": "Binaries/nosGraphics", + "associated_nodes": [] +} \ No newline at end of file diff --git a/Plugins/nosGraphics/Shaders/BillboardMask.frag b/Plugins/nosGraphics/Shaders/BillboardMask.frag new file mode 100644 index 00000000..fe70a464 --- /dev/null +++ b/Plugins/nosGraphics/Shaders/BillboardMask.frag @@ -0,0 +1,9 @@ +#version 450 +#extension GL_EXT_scalar_block_layout : enable + +layout(location = 0) out vec4 rt; + +void main() +{ + rt = vec4(1.0); +} \ No newline at end of file diff --git a/Plugins/nosGraphics/Shaders/BillboardMask.vert b/Plugins/nosGraphics/Shaders/BillboardMask.vert new file mode 100644 index 00000000..3661846f --- /dev/null +++ b/Plugins/nosGraphics/Shaders/BillboardMask.vert @@ -0,0 +1,23 @@ +#version 450 +#extension GL_EXT_scalar_block_layout : enable + +layout (binding = 0, std430) uniform UBO +{ + mat4 MVP; +} ubo; + +const vec2 pos[6] = + vec2[6]( + vec2(+0.5, +1.0), + vec2(-0.5, +1.0), + vec2(+0.5, 0.0), + vec2(+0.5, 0.0), + vec2(-0.5, +1.0), + vec2(-0.5, 0.0)); + +void main() +{ + vec4 vertPos = vec4(pos[gl_VertexIndex].x, 0.0, pos[gl_VertexIndex].y, 1.0); + vec4 clipPos = ubo.MVP * vertPos; + gl_Position = clipPos; +} \ No newline at end of file diff --git a/Plugins/nosGraphics/Source/BillboardMask.cpp b/Plugins/nosGraphics/Source/BillboardMask.cpp new file mode 100644 index 00000000..ba1952de --- /dev/null +++ b/Plugins/nosGraphics/Source/BillboardMask.cpp @@ -0,0 +1,222 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include +#include +#include + +namespace nos::graphics +{ +NOS_REGISTER_NAME(BillboardMask) + +NOS_REGISTER_NAME(Position) +NOS_REGISTER_NAME(Size) +NOS_REGISTER_NAME(RenderView) +NOS_REGISTER_NAME(Resolution) +NOS_REGISTER_NAME(OutRenderTarget) +NOS_REGISTER_NAME(OutDepth) + +NOS_REGISTER_NAME(BillboardMask_Pass) + +struct BillboardMask : NodeContext +{ + using NodeContext::NodeContext; + + enum class StatusMessageType + { + FailedToCreateRenderTarget = 1, + FailedToCreateDepthBuffer = 2, + }; + + std::unordered_map ActiveStatusMessages; + + void SerializeAndSendStatusMessages() + { + if (ActiveStatusMessages.size() == 0) + { + ClearNodeStatusMessages(); + return; + } + std::vector messages; + messages.reserve(ActiveStatusMessages.size()); + for (const auto& [type, statusMessage] : ActiveStatusMessages) + { + messages.push_back(statusMessage); + } + SetNodeStatusMessages(messages); + } + + void SetOrAddStatusMessage(StatusMessageType msgType, nos::fb::TNodeStatusMessage message) + { + if (auto it = ActiveStatusMessages.find(msgType); it != ActiveStatusMessages.end()) + { + if (it->second.text == message.text && it->second.type == message.type) + return; + } + ActiveStatusMessages[msgType] = std::move(message); + SerializeAndSendStatusMessages(); + } + + void RemoveStatusMessage(StatusMessageType msgType) + { + if (ActiveStatusMessages.erase(msgType) > 0) + SerializeAndSendStatusMessages(); + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + NodeExecuteParams pins(params); + TRenderView view = pins.GetPinData(NSN_RenderView); + glm::vec2 size = *pins.GetPinData(NSN_Size); + glm::vec3 position = *pins.GetPinData(NSN_Position); + + nosVec2u resolution = *pins.GetPinData(NSN_Resolution); + + nosResourceShareInfo rt = vkss::DeserializeTextureInfo(pins[NSN_OutRenderTarget].Data->Data); + + if (rt.Info.Texture.Width != resolution.x || rt.Info.Texture.Height != resolution.y) + { + auto newRt = vkss::Resource::Create( + nosTextureInfo{.Width = resolution.x, + .Height = resolution.y, + .Format = NOS_FORMAT_R8_UNORM, + .Filter = NOS_TEXTURE_FILTER_LINEAR, + .Usage = nosImageUsage(NOS_IMAGE_USAGE_SAMPLED | NOS_IMAGE_USAGE_RENDER_TARGET)}, + "Billboard Mask Output"); + if (!newRt) + { + SetOrAddStatusMessage(StatusMessageType::FailedToCreateRenderTarget, + nos::fb::TNodeStatusMessage{ + .text = "Failed to create render target for Billboard Mask node.", + .type = + nos::fb::NodeStatusMessageType::FAILURE, + }); + rt = {}; + } + else + { + SetPinValue(NSN_OutRenderTarget, newRt->ToPinData()); + rt = nosResourceShareInfo(*newRt); + RemoveStatusMessage(StatusMessageType::FailedToCreateRenderTarget); + } + } + + nosResourceShareInfo depth = vkss::DeserializeTextureInfo(pins[NSN_OutDepth].Data->Data); + if (depth.Info.Texture.Width != resolution.x || depth.Info.Texture.Height != resolution.y || + depth.Info.Texture.Format != NOS_FORMAT_D32_SFLOAT) + { + auto newDepth = vkss::Resource::Create( + nosTextureInfo{.Width = resolution.x, + .Height = resolution.y, + .Format = NOS_FORMAT_D32_SFLOAT, + .Filter = NOS_TEXTURE_FILTER_LINEAR, + .Usage = nosImageUsage(NOS_IMAGE_USAGE_SAMPLED | NOS_IMAGE_USAGE_DEPTH_STENCIL)}, + "Billboard Mask Depth"); + if (!newDepth) + { + SetOrAddStatusMessage(StatusMessageType::FailedToCreateDepthBuffer, + nos::fb::TNodeStatusMessage{ + .text = "Failed to create depth buffer for Billboard Mask node.", + .type = nos::fb::NodeStatusMessageType::FAILURE, + }); + depth = {}; + } + else + { + SetPinValue(NSN_OutDepth, newDepth->ToPinData()); + depth = nosResourceShareInfo(*newDepth); + RemoveStatusMessage(StatusMessageType::FailedToCreateDepthBuffer); + } + } + + if (rt.Memory.Handle == 0 || depth.Memory.Handle == 0) + { + return NOS_RESULT_FAILED; + } + + glm::mat4 viewMat = reinterpret_cast(view.view); + glm::mat4 projMat = reinterpret_cast(view.left_handed_projection_matrix); + + glm::mat4 modelMat = glm::mat4(1.0f); + // Face the camera + glm::vec3 cameraPos = glm::vec3(glm::inverse(viewMat)[3]); + cameraPos.z = position.z; + glm::vec3 toCamera = glm::normalize(cameraPos - position); + glm::vec3 up = glm::vec3(0.0f, 0.0f, 1.0f); + glm::vec3 right = glm::normalize(glm::cross(toCamera, up)); + modelMat = glm::translate(modelMat, position); + modelMat[0] = glm::vec4(right, 0.0f); + modelMat[1] = glm::vec4(toCamera, 0.0f); + modelMat[2] = glm::vec4(up, 0.0f); + // Translation writes to the 4th column, so no need to set it again + modelMat = glm::scale(modelMat, glm::vec3(size.x, 1.0f, size.y)); + + glm::mat4 mvp = projMat * viewMat * modelMat; + + auto bindings = std::array{vkss::ShaderBinding(NOS_NAME("MVP"), mvp)}; + + nosDrawCall drawCall{.Bindings = bindings.data(), + .BindingCount = bindings.size(), + .Vertices = nosVertexData{ + .IndexCount = 6, + .DepthFunc = NOS_DEPTH_FUNCTION_LESS, + .DepthWrite = true, + .DepthTest = true, + }}; + + nosRunPass2Params passParams{ + .Key = NSN_BillboardMask_Pass, + .Output = rt, + .DrawCalls = &drawCall, + .DrawCallCount = 1, + .Wireframe = NOS_FALSE, + .Benchmark = 0, + .DoNotClear = false, + .ClearCol = {0.f, 0.f, 0.f, 0.f}, + .DepthAttachment = nosDepthAttachment{.DepthBuffer = depth, .DoNotClear = false, .ClearValue = 1.0f}}; + + auto cmd = vkss::BeginCmd(NOS_NAME("Billboard Mask"), NodeId); + nosVulkan->RunPass2(cmd, &passParams); + nosVulkan->End(cmd, nullptr); + return NOS_RESULT_SUCCESS; + } +}; + +nosResult RegisterBillboardMask(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NSN_BillboardMask, BillboardMask, fn); + + fs::path root = nosEngine.Module->RootFolderPath; + auto vertPath = (root / "Shaders" / "BillboardMask.vert").generic_string(); + auto fragPath = (root / "Shaders" / "BillboardMask.frag").generic_string(); + + // Register shaders + auto billboardMask_Frag = NOS_NAME("BillboardMask_Frag"); + auto billboardMask_Vert = NOS_NAME("BillboardMask_Vert"); + std::array shaders = { + nosShaderInfo{.ShaderName = billboardMask_Frag, + .Source = {.Stage = NOS_SHADER_STAGE_FRAG, .GLSLPath = fragPath.c_str()}, + .AssociatedNodeClassName = NSN_BillboardMask}, + nosShaderInfo{.ShaderName = billboardMask_Vert, + .Source = {.Stage = NOS_SHADER_STAGE_VERT, .GLSLPath = vertPath.c_str()}, + .AssociatedNodeClassName = NSN_BillboardMask}}; + auto ret = nosVulkan->RegisterShaders(shaders.size(), shaders.data()); + if (NOS_RESULT_SUCCESS != ret) + return ret; + + nosPassInfo pass = { + .Key = NSN_BillboardMask_Pass, + .Shader = billboardMask_Frag, + .VertexShader = billboardMask_Vert, + .MultiSample = 1, + .Blend = NOS_BLEND_MODE_ALPHA_BLENDING, + }; + ret = nosVulkan->RegisterPasses(1, &pass); + if (NOS_RESULT_SUCCESS != ret) + return ret; + + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::graphics diff --git a/Plugins/nosGraphics/Source/GraphicsMain.cpp b/Plugins/nosGraphics/Source/GraphicsMain.cpp new file mode 100644 index 00000000..967a24ef --- /dev/null +++ b/Plugins/nosGraphics/Source/GraphicsMain.cpp @@ -0,0 +1,54 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +// Includes +#include +#include + +NOS_INIT() +NOS_VULKAN_INIT() + +NOS_BEGIN_IMPORT_DEPS() +NOS_VULKAN_IMPORT() +NOS_END_IMPORT_DEPS() + +namespace nos::graphics +{ +enum Nodes : int +{ // CPU nodes + TrackToView, + BillboardMask, + Count +}; + +nosResult RegisterTrackToView(nosNodeFunctions*); +nosResult RegisterBillboardMask(nosNodeFunctions*); + +struct PluginFunctions : nos::PluginFunctions +{ + nosResult NOSAPI_CALL ExportNodeFunctions(size_t& outSize, nosNodeFunctions** outFunctions) + { + outSize = Nodes::Count; + if (!outFunctions) + return NOS_RESULT_SUCCESS; +#define GEN_CASE_NODE(name) \ + case Nodes::name: { \ + auto ret = Register##name(node); \ + if (NOS_RESULT_SUCCESS != ret) \ + return ret; \ + break; \ + } + for (int i = 0; i < Nodes::Count; ++i) + { + auto node = outFunctions[i]; + switch ((Nodes)i) + { + GEN_CASE_NODE(TrackToView) + GEN_CASE_NODE(BillboardMask) + default: break; + } + } + return NOS_RESULT_SUCCESS; + } +}; +NOS_EXPORT_PLUGIN_FUNCTIONS(PluginFunctions) +} // namespace nos::graphics \ No newline at end of file diff --git a/Plugins/nosGraphics/Source/TrackToView.cpp b/Plugins/nosGraphics/Source/TrackToView.cpp new file mode 100644 index 00000000..2c6653b3 --- /dev/null +++ b/Plugins/nosGraphics/Source/TrackToView.cpp @@ -0,0 +1,94 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include + +#include +#include + +namespace nos::graphics +{ +glm::mat4 MakeView(glm::vec3 pos, glm::vec3 rot) +{ + rot = glm::radians(rot); + auto mat = (glm::mat3)glm::eulerAngleZYX(rot.z, -rot.y, -rot.x); + return glm::lookAtLH(pos, pos + mat[0], mat[2]); +} + +glm::mat4 Perspective(float fovx, float aspectRatio, glm::vec2 projectionShift, glm::vec2 clipPlanes) +{ + const f32 X = 1.f / tanf(glm::radians(fovx * 0.5f)); + const f32 Y = -X * aspectRatio; + const f32 Z = clipPlanes.y / (clipPlanes.y - clipPlanes.x); + return glm::mat4(glm::vec4(X, 0, 0, 0), + glm::vec4(0, Y, 0, 0), + glm::vec4(projectionShift.x, projectionShift.y, Z, 1.0f), + glm::vec4(0, 0, -clipPlanes.x * Z, 0)); +} + +glm::vec2 CalculateProjectionShift(glm::vec2 sensorSize, glm::vec2 centerShift) +{ + if (centerShift == glm::vec2(0)) + return glm::vec2(0); + if (sensorSize == glm::vec2(0)) + sensorSize = glm::vec2(1); + auto projectionShift = glm::vec2(-centerShift / sensorSize); + projectionShift.y = -projectionShift.y; + return projectionShift; +} + +NOS_REGISTER_NAME(Track) +NOS_REGISTER_NAME(Clip) +NOS_REGISTER_NAME(View) +struct TrackToView : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + NodeExecuteParams pins(params); + nos::track::TTrack const& track = pins.GetPinData(NSN_Track); + nos::fb::vec2 clip = *pins.GetPinData(NSN_Clip); + + glm::vec2 sensorSize = reinterpret_cast(track.sensor_size); + glm::vec2 centerShift = reinterpret_cast(track.lens_distortion.center_shift()); + glm::vec2 projShift = CalculateProjectionShift(sensorSize, centerShift); + if (glm::vec2(0) == sensorSize) + { + sensorSize = glm::vec2(1); + } + float aspectRatio = sensorSize.x / sensorSize.y * track.pixel_aspect_ratio; + TPerspectiveProjection perspectiveProjection{}; + perspectiveProjection.aspect_ratio = aspectRatio; + perspectiveProjection.fov_x = track.fov; + TProjection projection{}; + projection.clip_planes = clip; + projection.center_shift = reinterpret_cast(centerShift); + projection.projection_type = ProjectionType::Perspective; + projection.perspective = std::make_unique(perspectiveProjection); + glm::mat4 viewMatrix = MakeView( + reinterpret_cast(track.location), + reinterpret_cast(track.rotation)); + + glm::mat4 projectionMatrix = Perspective( + track.fov, aspectRatio, centerShift, reinterpret_cast(clip)); + + TRenderView view{}; + view.projection = std::make_unique(projection); + view.view = reinterpret_cast(viewMatrix); + view.left_handed_projection_matrix = reinterpret_cast(projectionMatrix); + SetPinValue(NSN_View, Buffer::From(view)); + return NOS_RESULT_SUCCESS; + } + + static nosResult OnRegister() { return NOS_RESULT_SUCCESS; } +}; + +nosResult RegisterTrackToView(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("TrackToView"), TrackToView, fn); + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::graphics