From fd07539737165d894b41e5009b2dc29a849a41d7 Mon Sep 17 00:00:00 2001 From: Bin Xie Date: Sat, 28 Apr 2018 13:22:18 +0800 Subject: [PATCH 1/2] 1. use the new SendBundleAsyncSequenced so the databuffer won't be out of sequence 2. enable audio capture and streaming --- .../LowLatencyMRC/UWP/Viewer/Capture.xaml.cs | 2 +- .../UWP/Viewer/Package.appxmanifest | 1 + .../Source/Shared/Media/CaptureEngine.cpp | 2 +- .../Shared/Media/NetworkMediaSinkStream.cpp | 2 +- .../Source/Shared/Media/PlaybackEngine.cpp | 3 +- .../Shared/MixedRemoteViewCompositor.idl | 3 + .../Source/Shared/Network/Connection.cpp | 115 ++++++++++++++++++ .../Source/Shared/Network/Connection.h | 3 + 8 files changed, 127 insertions(+), 4 deletions(-) diff --git a/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml.cs b/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml.cs index 19ea8992..f1843f3c 100644 --- a/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml.cs +++ b/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml.cs @@ -48,7 +48,7 @@ private async void bnListen_Click(object sender, RoutedEventArgs e) private async void StartCapture() { - captureEngine = await CaptureEngine.CreateAsync(false); + captureEngine = await CaptureEngine.CreateAsync(true); if (this.captureEngine != null) { await this.captureEngine.StartAsync(false, this.connection); diff --git a/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Package.appxmanifest b/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Package.appxmanifest index 7e492e20..a9218524 100644 --- a/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Package.appxmanifest +++ b/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Package.appxmanifest @@ -27,5 +27,6 @@ + \ No newline at end of file diff --git a/MixedRemoteViewCompositor/Source/Shared/Media/CaptureEngine.cpp b/MixedRemoteViewCompositor/Source/Shared/Media/CaptureEngine.cpp index cfb8f5bf..3975f66c 100644 --- a/MixedRemoteViewCompositor/Source/Shared/Media/CaptureEngine.cpp +++ b/MixedRemoteViewCompositor/Source/Shared/Media/CaptureEngine.cpp @@ -458,7 +458,7 @@ HRESULT CaptureEngineImpl::StartAsync( ComPtr mrcAudioEffectDefinition; MakeAndInitialize(&mrcAudioEffectDefinition); - IFR(mrcAudioEffectDefinition->put_MixerMode(AudioMixerMode_Mic)); + IFR(mrcAudioEffectDefinition->put_MixerMode(AudioMixerMode_MicAndLoopback)); ComPtr audioEffectDefinition; IFR(mrcAudioEffectDefinition.As(&audioEffectDefinition)); diff --git a/MixedRemoteViewCompositor/Source/Shared/Media/NetworkMediaSinkStream.cpp b/MixedRemoteViewCompositor/Source/Shared/Media/NetworkMediaSinkStream.cpp index 127b4d2c..52a2e31f 100644 --- a/MixedRemoteViewCompositor/Source/Shared/Media/NetworkMediaSinkStream.cpp +++ b/MixedRemoteViewCompositor/Source/Shared/Media/NetworkMediaSinkStream.cpp @@ -864,7 +864,7 @@ HRESULT NetworkMediaSinkStreamImpl::ProcessSamplesFromQueue( if (nullptr != spDataBundle.Get()) { ComPtr spSendAction; - if (FAILED(_spConnection->SendBundleAsync(spDataBundle.Get(), &spSendAction))) + if (FAILED(_spConnection->SendBundleAsyncSequenced(spDataBundle.Get(), &spSendAction))) { fProcessingSample = false; } diff --git a/MixedRemoteViewCompositor/Source/Shared/Media/PlaybackEngine.cpp b/MixedRemoteViewCompositor/Source/Shared/Media/PlaybackEngine.cpp index 25e51dcd..ae2c3021 100644 --- a/MixedRemoteViewCompositor/Source/Shared/Media/PlaybackEngine.cpp +++ b/MixedRemoteViewCompositor/Source/Shared/Media/PlaybackEngine.cpp @@ -411,7 +411,8 @@ HRESULT PlaybackEngineImpl::StartPlayback() NULL_CHK_HR(_sourceReader, E_NOT_SET); - IFR(RequestNextSampleAsync(MF_SOURCE_READER_FIRST_VIDEO_STREAM)); + IFR(RequestNextSampleAsync(MF_SOURCE_READER_ANY_STREAM)); + //IFR(RequestNextSampleAsync(MF_SOURCE_READER_FIRST_VIDEO_STREAM)); //IFR(RequestNextSampleAsync(MF_SOURCE_READER_FIRST_AUDIO_STREAM)); diff --git a/MixedRemoteViewCompositor/Source/Shared/MixedRemoteViewCompositor.idl b/MixedRemoteViewCompositor/Source/Shared/MixedRemoteViewCompositor.idl index 1205e089..408dd9a1 100644 --- a/MixedRemoteViewCompositor/Source/Shared/MixedRemoteViewCompositor.idl +++ b/MixedRemoteViewCompositor/Source/Shared/MixedRemoteViewCompositor.idl @@ -617,6 +617,9 @@ namespace MixedRemoteViewCompositor{ namespace Network { HRESULT SendBundleAsync( [in] MixedRemoteViewCompositor.Network.DataBundle* dataBundle, [out, retval] Windows.Foundation.IAsyncAction** asyncAction); + HRESULT SendBundleAsyncSequenced( + [in] MixedRemoteViewCompositor.Network.DataBundle* dataBundle, + [out, retval] Windows.Foundation.IAsyncAction** asyncAction); }; [exclusiveto(MixedRemoteViewCompositor.Network.Connection)] diff --git a/MixedRemoteViewCompositor/Source/Shared/Network/Connection.cpp b/MixedRemoteViewCompositor/Source/Shared/Network/Connection.cpp index 7f10342d..d57e0dcb 100644 --- a/MixedRemoteViewCompositor/Source/Shared/Network/Connection.cpp +++ b/MixedRemoteViewCompositor/Source/Shared/Network/Connection.cpp @@ -434,6 +434,121 @@ HRESULT ConnectionImpl::SendBundleAsync( return spWriteAction.CopyTo(sendAction); } +_Use_decl_annotations_ +HRESULT ConnectionImpl::SendBundleAsyncSequenced( + IDataBundle *dataBundle, + IAsyncAction **sendAction) +{ + Log(Log_Level_Info, L"ConnectionImpl::SendBundleAsyncSequenced\n"); + + NULL_CHK(dataBundle); + NULL_CHK(sendAction); + + { + auto lock = _lock.Lock(); + + IFR(CheckClosed()); + } + + ComPtr spWriteAction; + IFR(MakeAndInitialize(&spWriteAction, 1)); + + // define workitem + ComPtr spDataBundle(dataBundle); + ComPtr spThis(this); + auto workItem = + Microsoft::WRL::Callback( + [this, spThis, spDataBundle, spWriteAction](IAsyncAction* asyncAction) -> HRESULT + { + ComPtr spOutputStream; + DataBundleImpl::Container buffers; + DataBundleImpl::Iterator iter; + + ComPtr writeOperation; + ComPtr rawBuffer; + ComPtr singleBuffer; + + HRESULT hr = S_OK; + + { + auto lock = _lock.Lock(); + + IFC(CheckClosed()); + + // get the output stream for socket + if (nullptr != _streamSocket) + { + IFC(_streamSocket->get_OutputStream(&spOutputStream)); + } + else + { + IFC(E_UNEXPECTED); + } + } + + DataBundleImpl* bundleImpl = static_cast(spDataBundle.Get()); + if (nullptr == bundleImpl) + { + IFC(E_INVALIDARG); + } + + //make a single big buffer + ULONG totalSize; + IFC(bundleImpl->get_TotalSize(&totalSize)); + + IFC(MakeAndInitialize(&singleBuffer, totalSize)); + + DataBufferImpl * bufferImpl = static_cast(singleBuffer.Get()); + + if (nullptr == bufferImpl) + { + IFC(E_INVALIDARG); + } + + BYTE* rawSingleBuffer; + IFC(bufferImpl->get_Buffer(&rawSingleBuffer)); + + DWORD copied; + IFC(bundleImpl->CopyTo(0, totalSize, &rawSingleBuffer[0], &copied)); + + IFC(bufferImpl->put_CurrentLength(copied)); + + IFC(singleBuffer.As(&rawBuffer)); + + IFC(spOutputStream->WriteAsync(rawBuffer.Get(), &writeOperation)); + + hr = SyncWait(writeOperation.Get(), 1); + + spWriteAction->SignalCompleted(hr); + + done: + if (FAILED(hr)) + { + spWriteAction->SignalCompleted(hr); + } + + return S_OK; + }); + + ComPtr workerAsync; + IFR(_threadPoolStatics->RunAsync(workItem.Get(), &workerAsync)); + + IFR(StartAsyncThen( + workerAsync.Get(), + [this, spThis, spWriteAction](_In_ HRESULT hr, _In_ IAsyncAction *asyncResult, _In_ AsyncStatus asyncStatus) -> HRESULT + { + if (FAILED(hr)) + { + spWriteAction->SignalCompleted(hr); + } + + return S_OK; + })); + + // hand off async op + return spWriteAction.CopyTo(sendAction); +} + // IConnectionInternal _Use_decl_annotations_ HRESULT ConnectionImpl::WaitForHeader() diff --git a/MixedRemoteViewCompositor/Source/Shared/Network/Connection.h b/MixedRemoteViewCompositor/Source/Shared/Network/Connection.h index 96e1e543..dc89d326 100644 --- a/MixedRemoteViewCompositor/Source/Shared/Network/Connection.h +++ b/MixedRemoteViewCompositor/Source/Shared/Network/Connection.h @@ -153,6 +153,9 @@ namespace MixedRemoteViewCompositor IFACEMETHOD(SendBundleAsync)( _In_ ABI::MixedRemoteViewCompositor::Network::IDataBundle *dataBundle, _Out_ ABI::Windows::Foundation::IAsyncAction **sendAction); + IFACEMETHOD(SendBundleAsyncSequenced)( + _In_ ABI::MixedRemoteViewCompositor::Network::IDataBundle *dataBundle, + _Out_ ABI::Windows::Foundation::IAsyncAction **sendAction); protected: // IConnectionInternal From 2dbb2abbfecec9a92e4d5164c89edf2cc4878c93 Mon Sep 17 00:00:00 2001 From: Bin Xie Date: Tue, 10 Jul 2018 13:24:54 +0800 Subject: [PATCH 2/2] 1. change function signature to pass in the bool value for audio in StartCapture 2. refactor the single buffer into SendBundleAsync --- .../LowLatencyMRC/UWP/Viewer/Capture.xaml | 1 + .../LowLatencyMRC/UWP/Viewer/Capture.xaml.cs | 6 +- .../Shared/Media/NetworkMediaSinkStream.cpp | 2 +- .../Shared/MixedRemoteViewCompositor.idl | 3 - .../Source/Shared/Network/Connection.cpp | 109 ------------------ .../Source/Shared/Network/Connection.h | 3 - 6 files changed, 5 insertions(+), 119 deletions(-) diff --git a/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml b/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml index f1b74f57..6890ed0e 100644 --- a/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml +++ b/MixedRemoteViewCompositor/Samples/LowLatencyMRC/UWP/Viewer/Capture.xaml @@ -17,6 +17,7 @@