diff --git a/Samples/WindowsStudio/DocAssets/DemoAppScreenshot.png b/Samples/WindowsStudio/DocAssets/DemoAppScreenshot.png new file mode 100644 index 0000000..f442cc6 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/DemoAppScreenshot.png differ diff --git a/Samples/WindowsStudio/DocAssets/DistortionCalibrationExample.png b/Samples/WindowsStudio/DocAssets/DistortionCalibrationExample.png new file mode 100644 index 0000000..8b4a9f4 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/DistortionCalibrationExample.png differ diff --git a/Samples/WindowsStudio/DocAssets/FaceLandmarks.png b/Samples/WindowsStudio/DocAssets/FaceLandmarks.png new file mode 100644 index 0000000..b75d979 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/FaceLandmarks.png differ diff --git a/Samples/WindowsStudio/DocAssets/FacePose.png b/Samples/WindowsStudio/DocAssets/FacePose.png new file mode 100644 index 0000000..4e59e70 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/FacePose.png differ diff --git a/Samples/WindowsStudio/DocAssets/FaceTrackingBoundingBox.png b/Samples/WindowsStudio/DocAssets/FaceTrackingBoundingBox.png new file mode 100644 index 0000000..5542d5a Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/FaceTrackingBoundingBox.png differ diff --git a/Samples/WindowsStudio/DocAssets/HighRes1.png b/Samples/WindowsStudio/DocAssets/HighRes1.png new file mode 100644 index 0000000..5d0b882 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/HighRes1.png differ diff --git a/Samples/WindowsStudio/DocAssets/HighRes2.png b/Samples/WindowsStudio/DocAssets/HighRes2.png new file mode 100644 index 0000000..7323284 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/HighRes2.png differ diff --git a/Samples/WindowsStudio/DocAssets/LensMaxDist.png b/Samples/WindowsStudio/DocAssets/LensMaxDist.png new file mode 100644 index 0000000..dba46fd Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/LensMaxDist.png differ diff --git a/Samples/WindowsStudio/DocAssets/RayDistortion.png b/Samples/WindowsStudio/DocAssets/RayDistortion.png new file mode 100644 index 0000000..56f86b1 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/RayDistortion.png differ diff --git a/Samples/WindowsStudio/DocAssets/SensorCrop1.png b/Samples/WindowsStudio/DocAssets/SensorCrop1.png new file mode 100644 index 0000000..8419dfc Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/SensorCrop1.png differ diff --git a/Samples/WindowsStudio/DocAssets/SensorCrop2.png b/Samples/WindowsStudio/DocAssets/SensorCrop2.png new file mode 100644 index 0000000..577035b Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/SensorCrop2.png differ diff --git a/Samples/WindowsStudio/DocAssets/SensorCrop3.png b/Samples/WindowsStudio/DocAssets/SensorCrop3.png new file mode 100644 index 0000000..a0dae43 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/SensorCrop3.png differ diff --git a/Samples/WindowsStudio/DocAssets/ViewportUpdated1.png b/Samples/WindowsStudio/DocAssets/ViewportUpdated1.png new file mode 100644 index 0000000..3f14d01 Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/ViewportUpdated1.png differ diff --git a/Samples/WindowsStudio/DocAssets/ViewportUpdated2.png b/Samples/WindowsStudio/DocAssets/ViewportUpdated2.png new file mode 100644 index 0000000..c1d19fe Binary files /dev/null and b/Samples/WindowsStudio/DocAssets/ViewportUpdated2.png differ diff --git a/Samples/WindowsStudio/README.md b/Samples/WindowsStudio/README.md index 5b40ba0..b88a525 100644 --- a/Samples/WindowsStudio/README.md +++ b/Samples/WindowsStudio/README.md @@ -1,27 +1,64 @@ # Windows Studio Effects camera sample application - C# .Net WinUI & WinRT -additional documentation regarding ***[Windows Studio Effect (WSE) and its Driver-Defined Interfaces (DDIs)](<./Windows Studio Effects DDIs.md>)*** + +> See additional documentation regarding ***[Windows Studio Effect (WSE) and its Driver-Defined Interfaces (DDIs) and custom device properties](<./Windows Studio Effects DDIs.md>)*** + +--- + +![Demo App screenshot](./DocAssets/DemoAppScreenshot.png) + >This sample will only run fully on a system equipped with a [Windows Studio Effects (*WSE*)](https://learn.microsoft.com/en-us/windows/ai/studio-effects/) camera, which in itself requires: >1. a compatible NPU >2. the related Windows Studio Effects driver package installed or pulled-in via Windows Update ->3. a camera opted into *WSE* by the device manufacturer in its driver. Currently these camera must be front-facing +>3. a camera opted into *WSE* by the device manufacturer in its driver. Currently these camera must be front-facing and embedded in the device chassis (i.e. such as on laptops and tablets). -This folder contains a C# sample named **WindowsStudioSample_WinUI** which checks if a Windows Studio Effects camera is available on the system. It then proceeds using WinRT APIs to query support and toggle DDIs. +This folder contains a C# sample named **WindowsStudioSample_WinUI** which checks if a Windows Studio Effects camera is available on the system. It then proceeds using WinRT APIs to query support and toggle DDIs for *WSE* effects. -## WSE v1 effects +## Windows Studio Effects feature set and versions +Over time there were multiple released versions of *WSE* targetting a variety of SoCs with incrisingly capable NPUs. The *WSE* feature set has expanded with each version. + +### *WSE* v1 effects The sample leverages [extended camera controls](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/kspropertysetid-extendedcameracontrol) standardized in the OS and defined in Windows SDK such as the following 3 implemented as Windows Studio Effects in version 1: -- Standard Blur, Portrait Blur and Segmentation Mask Metadata : KSPROPERTY_CAMERACONTROL_EXTENDED_BACKGROUNDSEGMENTATION (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-backgroundsegmentation)*) -- Eye Contact Standard and Teleprompter: KSPROPERTY_CAMERACONTROL_EXTENDED_EYEGAZECORRECTION (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-eyegazecorrection)*) -- Automatic Framing: KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-digitalwindow)*) and KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW_CONFIGCAPS (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-digitalwindow-configcaps)*) +- **Standard Blur, Portrait Blur and Segmentation Mask Metadata** : KSPROPERTY_CAMERACONTROL_EXTENDED_BACKGROUNDSEGMENTATION (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-backgroundsegmentation)*) +- **Eye Contact Standard and Teleprompter**: KSPROPERTY_CAMERACONTROL_EXTENDED_EYEGAZECORRECTION (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-eyegazecorrection)*) +- **Automatic Framing**: KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-digitalwindow)*) and KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW_CONFIGCAPS (*[DDI documentation](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-digitalwindow-configcaps)*) -## WSE v2 effects +### *WSE* v2 effects It also taps into newer effects available in version 2 that are exposed using a set of DDIs custom to Windows Studio Effects. Since these are exclusive to Windows Studio Effects and shipped outside the OS, their definition is not part of the Windows SDK and has to be copied into your code base ([see DDI documentation](<./Windows Studio Effects DDIs.md>)): -- Portrait Light (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_stagelight-control>)*) -- Creative Filters (Animated, Watercolor and Illustrated) (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_creativefilter-control>)*) +- **Portrait Light** (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_stagelight-control>)*) +- **Creative Filters** (Animated, Watercolor and Illustrated) (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_creativefilter-control>)*) + +Version 2 also includes a DDI for disclosing to an app if *WSE* may resort to a mean to mitigate performance degradation, at which point the app can then poll the driver to understand if this performance mitigation is applied or not: +- **Performance mitigation** (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_performancemitigation-control>)*) + +### *WSE* v3 effects +The sample also now covers the latest *WSE* feature set for devices capable of running the version 3. This version of *WSE* contains a new Automatic Framing mode that can be toggled using a *DDI* by applications or manualy actionable from a user standpoint via the Windows Settings page of the camera: +- **Cinematic Automatic Framing** (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_automaticframingkind-control>)*) + +Version 3 also exposes a new DDI to allow an app to better locate a predominant face, track facial landmarks and a face pose: +- **Face Metadata** (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_facemetadata-control>)*) + +### *WSE* driver-exclusive DDIs and device properties +WSE includes multiple optimizations and features made available for OEMs to rely on under the hood, be it from a driver component standpoint (i.e. a DMFT) or to shape how WSE behaves in certain situation from an .inf. These functionalties are not meants to be interacted with from an app standpoint (and therefore not covered in the code sample) such as: + +#### *WSE* driver-exclusive DDIs +- **Set notification** (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_setnotification-control>)*) +- **Sensor crop mode query** (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_sensorcentercrop-control>)*) +- **Viewport updates** (*[DDI documentation](<./Windows Studio Effects DDIs.md#ksproperty_cameracontrol_windowsstudio_updated_viewport-control>)*) + + +#### *WSE* Device properties +- **High resolution modes** (*[DDI documentation](<./Windows Studio Effects DDIs.md#high-resolution-modes>)*) +- **High resolution mode conditional on power state** (*[DDI documentation](<./Windows Studio Effects DDIs.md#high-resolution-mode-conditional-on-power-state>)*) +- **Face detection DDI support and frequency of face-based ROIs for 3A driven by *WSE*** (*[DDI documentation](<./Windows Studio Effects DDIs.md#face-detection-ddi-support-and-frequency-of-face-based-rois-for-3a-driven-by-wse>)*) +- **Lens Distortion Correction** (*[DDI documentation](<./Windows Studio Effects DDIs.md#lens-distortion-correction>)*) +- **Camera pinhole intrinsics** (*[DDI documentation](<./Windows Studio Effects DDIs.md#pinhole-intrinsics-device-property-key>)*) +- **Camera lens distortion model** (*[DDI documentation](<./Windows Studio Effects DDIs.md#lens-distortion-model-device-property-keys>)*) + -## WSE limits frame formats and profiles +## *WSE* limits frame formats and profiles The code sample allows to see all the MediaTypes (frame formats) exposed by the camera and change the [camera profile](https://learn.microsoft.com/en-us/windows/uwp/audio-video-camera/camera-profiles) it is provisioned with. When initialized with profile for processing effects, *WSE* limits: - The number of streams available from a source with effects applied to 1 - Processing of only color stream (i.e. no infrared stream, etc.) @@ -31,9 +68,9 @@ The code sample allows to see all the MediaTypes (frame formats) exposed by the - Subtype in NV12 format only (*WSE* may take as input other subtypes, but will currently only expose out NV12) That said as alluded to above, these limits are only imposed when the camera profile used is one of the below [known video profile](https://learn.microsoft.com/en-us/uwp/api/windows.media.capture.knownvideoprofile?view=winrt-26100): -- the default (or not specified), also known as *Legacy* profile -- *VideoRecording* -- *VideoConferencing* +- the default (or not specified), also known as *Legacy* profile (*KSCAMERAPROFILE_Legacy*) +- *VideoRecording* (*KSCAMERAPROFILE_VideoRecording*) +- *VideoConferencing* (*KSCAMERAPROFILE_VideoConferencing*) **With other profiles however, while these limits are not imposed and instead rely on the capabilities defined by the original camera driver, none of the Windows Studio Effects DDIs are supported**. This is why for example, the Windows Camera Application might not support any effects when entering photo capture mode but offer a different set of MediaTypes to exercise such as with higher resolution and other subtypes. @@ -80,23 +117,32 @@ The app demonstrates the following: }); ``` -3. Check if the newer set of Windows Studio Effects in version 2 are supported. These new DDIs are defined in a new property set [see DDI documentation](<./Windows Studio Effects DDIs.md>). +3. Check if the a Windows Studio Effects is supported. These new DDIs are defined in a new property set [see DDI documentation](<./Windows Studio Effects DDIs.md>). ```csharp // New Windows Studio Effects custom KsProperties live under this property set public static readonly Guid KSPROPERTYSETID_WindowsStudioEffects = Guid.Parse("1666d655-21A6-4982-9728-52c39E869F90"); - // Custom KsProperties exposed in version 2 + // Custom KsProperties exposed in version 2 and 3 public enum KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS : uint { + // v2 KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED = 0, KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT = 1, KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER = 2, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION = 3, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4, + + // v3 + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA = 5, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP = 6, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND = 7, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT = 8 }; // ... // in InitializeCameraAndUI() - // query support for new effects in v2 + // query support for effects; for example check if StageLight (a.k.a PortraitLight) and CreativeFilter are supported byte[] byteResultPayload = GetExtendedControlPayload( m_mediaCapture.VideoDeviceController, KSPROPERTYSETID_WindowsStudioEffects, diff --git a/Samples/WindowsStudio/Windows Studio Effects DDIs.md b/Samples/WindowsStudio/Windows Studio Effects DDIs.md index 800af89..543a2cb 100644 --- a/Samples/WindowsStudio/Windows Studio Effects DDIs.md +++ b/Samples/WindowsStudio/Windows Studio Effects DDIs.md @@ -1,7 +1,16 @@ -# Windows Studio Effects *(WSE)* driver – Windows DDIs and Exclusive DDIs +# Windows Studio Effects *(WSE)* driver – Windows DDIs and Exclusive DDIs, *WSE* device properties -The *WSE* driver can be interacted with using APIs which transacts according to a specified Driver-Defined Interface *(DDI)* using a [KSPROPERTY](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-structure) that dictates either a GET or a SET operation, followed by a data payload (*PropertyData*). +- [Windows Standardized DDIs implemented by *WSE* (blur, eye contact, automatic framing)](#windows-standardized-ddis-implemented-by-wse) +- [Custom DDIs implemented by *WSE* for apps (creative filters, portrait light, face metadata)](#wse-custom-ddi-specification) +- [Custom DDIs implemented by *WSE* for drivers (set notification, sensor crop, viewport updates)](#wse-custom-ddis-and-driver-properties-intended-for-driver-implementers-oems-and-ihvs) +- [Custom *WSE* device properties for drivers (high resolution modes, lens distortion correction, pinhole intrinsics, lens distortion model, face detection for 3A)](#wse-device-properties) +# Driver-Defined Interfaces (DDIs) +The *WSE* driver can be interacted with using APIs which transact according to a specified Driver-Defined Interface *(DDI)* using a [KSPROPERTY](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-structure) call that +1. dictates either a GET or a SET operation as defined in a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) +2. followed by a related data payload (*PropertyData*) which can be either a common (and often unused) [KSCAMERA_EXTENDEDPROP_VALUE](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_value) or a custom payload format. + +The below specification details the allowed operations for each DDIs and the expected payload format. ## Windows standardized DDIs implemented by *WSE* @@ -53,17 +62,21 @@ an instance of the *[IMFExtendedCameraControl](https://learn.microsoft.com/en-us ## Custom DDIs implemented by *WSE* driver -A newer set of effects supported in *WSE* version 2 are exposed as custom DDIs under a different and exclusive property set (***KSPROPERTYSETID_WindowsStudioCameraControl*** defined below) not standardized with a particular Windows SDK: +A newer set of effects supported in *WSE* version 2 and 3 are exposed as custom DDIs under a different and exclusive property set (***KSPROPERTYSETID_WindowsStudioCameraControl*** defined below) not standardized with a particular Windows SDK: - - **Portrait Light: [KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT-Control)** - - **Creative Filters: [KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER-Control)** + - **Portrait Light: [KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT-Control)** (version 2+) + - **Creative Filters: [KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER-Control)** (version 2+) + - **Automatic Framing kind: [KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND-Control)** (version 3+) - Furthermore, the Windows Studio driver component exposes additional DDIs that can be useful to applications to advertise some capabilities: - - Report which *DDIs* are supported and implemented by *WSE* specifically: **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED-Control)** - - Disclose if any defined mean to mitigate performance decrease is possible and used: **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION-Control)** + Furthermore, the Windows Studio driver component exposes additional DDIs that can be useful to applications to: + - Report which *DDIs* are supported and implemented by *WSE* specifically (***version 1+***): **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED-Control)** + - Disclose if any predefined mean to mitigate performance decrease can be exercised and is currently used when polled (***version 2+***): **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION-Control)** + - Exercise and retrieve useful face tracking metadata (***version 3+***): **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA-Control)**, [MF_WINDOWSSTUDIO_METADATA_FACETRACKING](#MF_WINDOWSSTUDIO_METADATA_FACETRACKING), [MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS](#MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS), [MF_WINDOWSSTUDIO_METADATA_FACEPOSE](#MF_WINDOWSSTUDIO_METADATA_FACEPOSE) - Finally, the Windows Studio driver component exposes a DDI useful to other driver components so that they can register to be notified when certain effects are toggled in WSE: - - **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION-Control)** + Finally, the Windows Studio driver component exposes DDIs meant to interop only with other driver components such as DMFTs and AVStream driver (and not and application) so that: + - they can register to be notified when certain effects are toggled in *WSE*: **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION-Control)** (version 2+) + - they can register to be notified when *WSE* leverages only a part of its input frame (such as when applying automatic framing or cropping) for processing its output frame: **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT-Control)** (version 3+) + - they can report when the region of the sensor leveraged is cropped in a certain custom way: **[KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP](#KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP-Control)** (version 3+) #### C++ definition of *WSE* custom DDIs: ~~~cpp @@ -77,7 +90,11 @@ typedef enum { KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT = 1, KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER = 2, KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION = 3, - KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4 + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA = 5, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP = 6, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND = 7, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT = 8 } KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PROPERTY; ~~~ In order to use these DDIs for *WSE* version 2 in your code, you would copy those definitions into your header file as they are not included in any of the Windows SDK. @@ -187,7 +204,7 @@ From an application standpoint, these *WSE* custom DDIs can be programmatically } ~~~ -# WSE custom profile +# *WSE* custom profile *WSE* exposes a custom *"passthrough"* profile that exposes all MediaTypes but prevents usage of any effect DDIs. This profile can be leveraged by application to bypass [stream, resolution and framerate limits](<./README.md#WSE-limits-frame-formats-and-profiles>) imposed by *WSE* for scenarios such as recording at higher resolution when possible. This custom profile definition needs to be copied in your code as it is not a Windows standard profile defined in WDK. #### C++ definition of custom "passthrough" profile exposed by *WSE* @@ -200,28 +217,28 @@ From an application standpoint, these *WSE* custom DDIs can be programmatically static GUID KSCAMERAPROFILE_WindowsStudioNoEffectsColorPassthrough = { 0xe4ed96d9, 0xcd40, 0x412f, 0xb2, 0xa, 0xb7, 0x40, 0x2a, 0x43, 0xdc, 0xd2 }; ~~~ -# WSE custom DDI specification -The GET and SET data payload format for each of these custom WSE DDIs is covered below. +# *WSE* custom DDI specification +The GET and SET data payload format for each of these custom *WSE* DDIs is covered below. + ----------- ## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED Control This control allows to retrieve a list of supported KSProperties intercepted and implemented specifically by the Windows Studio Effects (*WSE*) camera driver component. Think of it as exclusively a *getter* for which controls corresponding to KSPROPERTY_CAMERACONTROL_EXTENDED_\*, KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_\* and any other custom property set that are implemented directly within *WSE* driver component. It can be preemptively queried before individually proceeding to getting capabilities for each of them if interaction with exclusively the *WSE* driver is desired or simply used to correlate with an existing set of DDI capabilities already fetched to understand if their implementation is provided by *WSE* or another component that is part of the camera driver stack. ### Usage Summary | Scope | Control | Type | GET | SET | | ----- | ----- | ----- | ----- | ----- | -| Version 1 | Filter | Synchronous| ✔️ | ❌ | +| Version 1 | Filter | Synchronous| ✔️ | ❌ | The SET call of this control will fail. The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of *sizeof(KSPROPERTY) \** ***n*** -where ***“n”*** is the count of [KSPROPERTY](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-structure) reported. +where ***"n"*** is the count of [KSPROPERTY](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-structure) reported. #### KSCAMERA_EXTENDEDPROP_HEADER | **Member** | **Description** | | ----- | ----- | -| **Version** | Must be 1 | -| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). -| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + n * sizeof(KSPROPERTY) where “n” is the count of KSPROPERTY reported | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + n * sizeof(KSPROPERTY) where "n" is the count of KSPROPERTY reported | | **Result** | Unused, must be 0 | **Capability** | Unused, must be 0 | **Flags** | Unused, must be 0 @@ -239,6 +256,7 @@ for example, a GET call could return the following payload to be reinterpreted: |⬇️| [KSPROPERTY](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-structure) | *Set* = KSPROPERTYSETID_ExtendedCameraControl
*Id* = KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER
*Flags* = KSPROPERTY_TYPE_GET | |⬇️| [KSPROPERTY](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-structure) | *Set* = KSPROPERTYSETID_ExtendedCameraControl
*Id* = KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION
*Flags* = KSPROPERTY_TYPE_GET | |🔚| [KSPROPERTY](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-structure) | *Set* = KSPROPERTYSETID_ExtendedCameraControl
*Id* = KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION
*Flags* = KSPROPERTY_TYPE_GET | +---------- ---------- ## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT Control @@ -247,7 +265,7 @@ The Stage Light DDI allows to turn ON or OFF the application of the Stage Light ### Usage Summary | Scope | Control | Type | GET | SET | | ----- | ----- | ----- | ----- | ----- | -| Version 1 | Filter | Synchronous| ✔️ | ✔️ | +| Version 1 | Filter | Synchronous| ✔️ | ✔️ | The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of a [KSCAMERA_EXTENDEDPROP_VALUE](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_value). @@ -260,12 +278,13 @@ The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERAC #### KSCAMERA_EXTENDEDPROP_HEADER | **Member** | **Description** | | ----- | ----- | -| **Version** | Must be 1 | -| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | -| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | | **Result** | Unused, must be 0 | | **Capability** | Must at least contain a different valid potential flag value than KSCAMERA_WINDOWSSTUDIO_STAGELIGHT_OFF | | **Flags** | KSCAMERA_WINDOWSSTUDIO_STAGELIGHT_OFF or KSCAMERA_WINDOWSSTUDIO_STAGELIGHT_ON | +---------- ---------- ## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER Control @@ -274,7 +293,7 @@ The Creative Filter DDI allows to toggle a pre-defined effect that alters the ap ### Usage Summary | Scope | Control | Type | GET | SET | | ----- | ----- | ----- | ----- | ----- | -| Version 1 | Filter | Synchronous| ✔️ | ✔️ | +| Version 1 | Filter | Synchronous| ✔️ | ✔️ | The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of a [KSCAMERA_EXTENDEDPROP_VALUE](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_value). @@ -289,28 +308,230 @@ The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERAC #### KSCAMERA_EXTENDEDPROP_HEADER | **Member** | **Description** | | ----- | ----- | -| **Version** | Must be 1 | -| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | -| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | | **Result** | Unused, must be 0 | | **Capability** | Must at least contain a different valid potential flag value than KSCAMERA_WINDOWSSTUDIO_CREATIVEFILTER_OFF | | **Flags** |KSCAMERA_WINDOWSSTUDIO_ CREATIVEFILTER_OFF or KSCAMERA_WINDOWSSTUDIO_CREATIVEFILTER_ILLUSTRATED or KSCAMERA_WINDOWSSTUDIO_CREATIVEFILTER_ANIMATED or KSCAMERA_WINDOWSSTUDIO_CREATIVEFILTER_WATERCOLOR | +---------- + +---------- +## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION Control +This control act as a polling mechanism to identify if the current camera opted into *WSE* can be subject to means of performance mitigation when experiencing for example a dip in framerate due to compute resource contention causing frame processing to take longer than the frame timing deadline. In effect this can take the shape of degrading slightly the quality of an effect in an attempt to lighten the computational load. Having this control allows an app to preemptively advertise this cue to the user as well as query the current state of such mitigation. + +### Usage Summary +| Scope | Control | Type | GET | SET | +| ----- | ----- | ----- | ----- | ----- | +| Version 1 | Filter | Synchronous| ✔️ | ❌ | + +The SET call of this control will fail. + +The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of *sizeof(KSPROPERTY) \** ***n*** +where ***"n"*** is the count of KSPROPERTY reported. + +~~~cpp +// PerformanceMitigation possible flags values +#define KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_NONE 0x0000000000000000 +#define KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_EYEGAZECORRECTION_STARE 0x0000000000000001 +~~~ + +#### KSCAMERA_EXTENDEDPROP_HEADER +| **Member** | **Description** | +| ----- | ----- | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + n * sizeof(KSPROPERTY) where "n" is the count of KSPROPERTY reported | +| **Result** | Unused, must be 0 +| **Capability** | ***GET call:*** Bitmask of supported flag values (KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_*). Must at least contain a different valid potential flag value than KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_NONE. +| **Flags** | ***GET call:*** Bitmask of current flag value in operation **(KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_\*)**. For example, if the flags value is KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_NONE, then it means no performance mitigation are currently applied. +---------- + +---------- +## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA Control +This control enables or disables the production of metadata specific to Windows Studio Effects camera driver component attached to each frame detailing face-related information in camera space such as: +- face tracking +- head pose +- 2D facial landmarks + +This set of frame metadata will be available via a new set of metadata attached to each processed sample under the MFSampleExtension_CaptureMetadata IMFAttribute store. + +### Usage Summary +| Scope | Control | Type | GET | SET | +| ----- | ----- | ----- | ----- | ----- | +| Version 1 | Filter | Synchronous| ✔️ | ✔️ | + +The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of a [KSCAMERA_EXTENDEDPROP_VALUE](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_value). + +~~~cpp +// Face metadata control possible flags values +#define KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_OFF 0x0000000000000000 +#define KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACETRACKING 0x0000000000000001 +#define KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACELANDMARKS 0x0000000000000002 +#define KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACEPOSE 0x0000000000000004 + +// Metadata header used for each metadata blob in MF_WINDOWSSTUDIO_METADATA_FACE* and MF_WINDOWSSTUDIO_METADATA_GAZE* +struct WSEFaceMetadataHeader +{ + uint32_t Count; + uint32_t Size; +}; + +__if_not_exists(MF_FLOAT2) +{ + typedef struct _MF_FLOAT2 + { + FLOAT x; + FLOAT y; + } MF_FLOAT2; +} +// MF_WINDOWSSTUDIO_METADATA_FACETRACKING +// Type: IUnknown (IMFAttributes) +// This is the IMFAttributes store for all the metadata custom to the Windows Studio Effects. +// {F008811B-70C6-4A4F-9424-D576AC159905} +static const GUID MF_WINDOWSSTUDIO_METADATA_FACETRACKING = +{ 0xf008811b, 0x70c6, 0x4a4f, { 0x94, 0x24, 0xd5, 0x76, 0xac, 0x15, 0x99, 0x5 } }; + +// Metadata structure associate with MF_WINDOWSSTUDIO_METADATA_FACETRACKING +struct WSEFaceTrackingMetadata +{ + MF_FLOAT2 TopLeft; // Top left corner of the bounding box of face in image relative coordinates [0, 1] + MF_FLOAT2 BoxSize; // Width and Height of the bounding box of face in image relative coordinates [0, 1] + float Confidence; // Confidence of this region being an actual face (0..1) + uint32_t TrackId; // Corresponding track id +}; + +// MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS +// Type: IUnknown (IMFAttributes) +// This is the IMFAttributes store for all the metadata custom to the Windows Studio Effects. +// {1239D90E-6920-49A9-B72E-5F6D77944675} +static const GUID MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS = +{ 0x1239d90e, 0x6920, 0x49a9, { 0xb7, 0x2e, 0x5f, 0x6d, 0x77, 0x94, 0x46, 0x75 } }; + +// Metadata structure associate with MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS +struct WSEFaceLandmarksMetadata +{ + static constexpr uint32_t MAX_NUM_LANDMARKS = 70; // maximum number of landmarks + MF_FLOAT2 Landmarks2D[MAX_NUM_LANDMARKS]; // landmark location of faces in image relative coordinates [0, 1]. + FLOAT Confidence[MAX_NUM_LANDMARKS]; // individual confidence for each landmark (0..1) + uint32_t TrackId; // corresponding track id +}; + +// MF_WINDOWSSTUDIO_METADATA_FACEPOSE +// Type: IUnknown (IMFAttributes) +// This is the IMFAttributes store for all the metadata custom to the Windows Studio Effects. +// {397757AA-B630-43CE-AD47-AB3AC5FE4728} +static const GUID MF_WINDOWSSTUDIO_METADATA_FACEPOSE = +{ 0x397757aa, 0xb630, 0x43ce, { 0xad, 0x47, 0xab, 0x3a, 0xc5, 0xfe, 0x47, 0x28 } }; + +// Metadata structure associate with MF_WINDOWSSTUDIO_METADATA_FACEPOSE +struct WSEFacePoseMetadata +{ + static constexpr uint32_t EULER_ANGLE_COUNT = 3; // number of angles in pose + FLOAT Pose[EULER_ANGLE_COUNT]; // yaw, pitch, roll + FLOAT Confidence; // overall confidence of pose detection results (0..1) + uint32_t TrackId; // corresponding track id +}; +~~~ + +#### KSCAMERA_EXTENDEDPROP_HEADER +| **Member** | **Description** | +| ----- | ----- | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | +| **Result** | Unused, must be 0 | +| **Capability** | GET call: Bitmask of supported flag values (KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_\*). Must at least contain a different valid potential flag value than KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_OFF.
SET call: Unused, must be 0 +| **Flags** | GET call: Bitmask of current flag values in operation (KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_\*).
SET call: Bitmask of flag values supported (KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_\*). + +When the Flags value is anything but KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_OFF, each sample will get the following related metadata attached to it: + +## MF_WINDOWSSTUDIO_METADATA_FACETRACKING frame metadata +Produced when the control KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA is set with a bit mask containing KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACETRACKING. This metadata contains the bounding box of each face tracked in the frame. + +![face tracking bounding box](./DocAssets/FaceTrackingBoundingBox.png) + +The metadata payload is in the following layout: **WSEFaceMetadataHeader** + 'n' * **WSEFaceTrackingMetadata** +| | | +| -- | -- | +| **WSEFaceMetadataHeader** | **Count** (UINT32): Count of faces tracked with this metadata
**Size** (UINT32): Size (bytes) of this entire metadata payload, sizeof(WSEFaceMetadataHeader) + Count * sizeof(WSEFaceTrackingMetadata) | +| **WSEFaceTrackingMetadata** | **TopLeft** (float[2]): Top left corner (x,y) of the bounding box of face in image relative coordinates [0, 1]
**BoxSize** (float[2]): Width and Height of the bounding box of face in image relative coordinates [0, 1]
**Confidence** (float): Confidence of this region being an actual face (0..1)
**TrackId** (UINT32): Corresponding track id to correlate with other metadata +| Any additional **WSEFaceTrackingMetadata** according to WSEFaceMetadataHeader.Count | | + +## MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS frame metadata +Produced when the control KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA is set with a bit mask containing KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACELANDMARKS. This metadata contains the 2D facial landmarks of each face tracked in the frame. The landmarks provided follow the Multi-PIE 68 points mark-up scheme and adds 2 extra points, one for each center of the iris, for a total of 70 tracked per face. + +![Facial landmarks - 70 points](./DocAssets/FaceLandmarks.png) + +The metadata payload is in the following layout: **WSEFaceMetadataHeader** + 'n' * **WSEFaceLandmarksMetadata** +| | | +| -- | -- | +| **WSEFaceMetadataHeader** | **Count** (UINT32): Count of faces tracked with this metadata
**Size** (UINT32): Size (bytes) of this entire metadata payload, sizeof(WSEFaceMetadataHeader) + Count * sizeof(WSEFaceLandmarksMetadata) | +| **WSEFaceLandmarksMetadata** | **Landmarks2D** (float[70][2]): 2D Landmark (x,y) location of faces in image relative coordinates [0, 1]
**Confidence** (float): Confidence of this region being an actual face (0..1)
**TrackId** (UINT32): Corresponding track id to correlate with other metadata +| Any additional **WSEFaceLandmarksMetadata** according to WSEFaceMetadataHeader.Count | | + +## MF_WINDOWSSTUDIO_METADATA_FACEPOSE frame metadata +Produced when the control KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA is set with a bit mask containing KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACEPOSE. This metadata contains the 3D orientation of each face tracked in the frame in Euler angles (pitch, yaw, roll) along each axis (x, y, z) in degrees in a right-handedness coordinate system, where the Z axis points towards the user. + +![Face pose](./DocAssets/FacePose.png) + +The metadata payload is in the following layout: **WSEFaceMetadataHeader** + 'n' * **WSEFacePoseMetadata** +| | | +| -- | -- | +| **WSEFaceMetadataHeader** | **Count** (UINT32): Count of faces tracked with this metadata
**Size** (UINT32): Size (bytes) of this entire metadata payload, sizeof(WSEFaceMetadataHeader) + Count * sizeof(WSEFacePoseMetadata) | +| **WSEFacePoseMetadata** | **Pose** (float[3]): Estimated pose of the face (yaw, pitch, roll)
**Confidence** (float): Confidence of this region being an actual face (0..1)
**TrackId** (UINT32): Corresponding track id to correlate with other metadata +| Any additional **WSEFacePoseMetadata** according to WSEFaceMetadataHeader.Count | | +---------- + +---------- +## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND Control +The Automatic Framing Kind DDI allows to query and specify which Automatic Framing solution implemented in *WSE* to leverage when the Digital Window DDI is set with flags to operate automatic face framing (KSCAMERA_EXTENDEDPROP_DIGITALWINDOW_AUTOFACEFRAMING). + +### Usage Summary +| Scope | Control | Type | GET | SET | +| ----- | ----- | ----- | ----- | ----- | +| Version 1 | Filter | Synchronous| ✔️ | ✔️ | + + +The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of a [KSCAMERA_EXTENDEDPROP_VALUE](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_value). + +~~~cpp +// Automatic framing kind possible flags values +#define KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_WINDOW 0x0000000000000001 +#define KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_CINEMATIC 0x0000000000000002 +~~~ + +#### KSCAMERA_EXTENDEDPROP_HEADER +| **Member** | **Description** | +| ----- | ----- | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | +| **Result** | Unused, must be 0 | +| **Capability** | Must at least contain a valid flags value or a bit combination of them:
- KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_WINDOW
- KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_CINEMATIC | +| **Flags** | Can be KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_WINDOW or KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_CINEMATIC | + +There are no expectations of default Flags value. In a GET call, *WSE* driver should return the current settings in the Flags field. + +---------- + + +# *WSE* Custom DDIs and driver properties intended for driver implementers, OEMs and IHVs ---------- ## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION Control -This control is not meant to be used by application and camera session clients. +>**This DDI is intended for other camera driver component to implement, not application.** It serves as a way for *WSE* driver component to inform other driver components sitting in the driver stack such as a DMFT that an effect DDI state was modified. This may trigger internal changes in these other driver components to accommodate the state change such as disabling or altering pixel processing accordingly. Therefore a driver component that wants to be notified of a Windows Studio Effects DDI state change may expose support for this KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION DDI. An example of this could be when *WSE* is requested to turn ON Automatic Framing via the KSProperty KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW; a DeviceMFT may change the way it compensates for lens distortion as to mitigate image quality artifacts when zooming into the fringes of the frame. This notification is sent from *WSE* and relies on each subsequent driver component in the chain to relay it further to the next component. -**If a driver component intercepts this KSProperty DDI, it shall as well relay it to the subsequent component for both a SET and a GET in which case it needs to aggregate capability from the next component into its own reported capability** - +>**If a driver component intercepts this KSProperty DDI, it shall as well relay it to the subsequent component for both a SET and a GET in which case it needs to aggregate capability from the next component into its own reported capability** ### Usage Summary | Scope | Control | Type | GET | SET | | ----- | ----- | ----- | ----- | ----- | -| Version 1 | Filter | Synchronous| ✔️ | ✔️ | +| Version 1 | Filter | Synchronous| ✔️ | ✔️ | ***For GET call***: the KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a [KSCAMERA_EXTENDEDPROP_VALUE](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_value). The capability field defines which supported SET notifications to receive. @@ -325,46 +546,1007 @@ The capability field defines which supported SET notifications to receive. #define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_BACKGROUNDSEGMENTATION 0x0000000000000004 #define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_STAGELIGHT 0x0000000000000008 #define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_CREATIVEFILTER 0x0000000000000010 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_FACEDETECTION 0x0000000000000020 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_LDC 0x0000000000000040 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_FACEMETADATA 0x000000000000080 ~~~ #### KSCAMERA_EXTENDEDPROP_HEADER | **Member** | **Description** | | ----- | ----- | -| **Version** | Must be 1 | -| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | -| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | | **Result** | Unused, must be 0 | | **Capability** | ***GET call***: Bitmask of notification supported **(KSCAMERA_WINDOWSSTUDIO_ SETNOTIFICATION_\*) for this component and all the subsequent ones up to this point**. Must at least contain a different valid potential flag value than KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_NONE.
***SET call***: Unused, must be 0 | -| **Flags** | ***GET call***: Unused, must be 0.
***SET call***: One of the KSCAMERA_WINDOWSSTUDIO_ SETNOTIFICATION_* value other than KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_NONE. - | +| **Flags** | ***GET call***: Unused, must be 0.
***SET call***: One of the KSCAMERA_WINDOWSSTUDIO_ SETNOTIFICATION_* value other than KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_NONE. | - ---------- -## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION Control -This control act as a polling mechanism to identify if the current camera opted into *WSE* can be subject to means of performance mitigation when experiencing for example a dip in framerate due to compute resource contention causing frame processing to take longer than the frame timing deadline. In effect this can take the shape of degrading slightly the quality of an effect in an attempt to lighten the computational load. Having this control allows an app to preemptively advertise this cue to the user as well as query the current state of such mitigation. +This is an example of how a device MFT may report support for and relay a KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION payload. +~~~cpp + +// .h +// redefine locally GUID defined in OS SDK ksmedia.h for convenience +static GUID KSPROPERTYSETID_WindowsStudioCameraControl = { 0x1666d655, 0x21a6, 0x4982, 0x97, 0x28, 0x52, 0xc3, 0x9e, 0x86, 0x9f, 0x90 }; + +// redefine locally +typedef enum { + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED = 0, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT = 1, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER = 2, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION = 3, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA = 5, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP = 6, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND = 7, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT = 8 +} KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PROPERTY; + +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_NONE 0x0000000000000000 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_DIGITALWINDOW 0x0000000000000001 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_EYECORRECTION 0x0000000000000002 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_BACKGROUNDSEGMENTATION 0x0000000000000004 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_STAGELIGHT 0x0000000000000008 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_CREATIVEFILTER 0x0000000000000010 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_FACEDETECTION 0x0000000000000020 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_LDC 0x0000000000000040 +#define KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_FACEMETADATA 0x000000000000080 + +class SampleDMFT + : public IMFDeviceTransform, +// … +{ +public: + HRESULT InitializeTransform(_In_ IMFAttributes* pAttributes) override; + HRESULT KSProperty( + _In_reads_bytes_(ulPropertyLength) PKSPROPERTY pProperty, + _In_ ULONG ulPropertyLength, + _Inout_updates_to_(ulDataLength, *pulBytesReturned) LPVOID pPropertyData, + _In_ ULONG ulDataLength, + _Out_ ULONG* pulBytesReturned) override; + wil::com_ptr_nothrow m_spKsControl; +// … +} + +// .cpp + +/// +/// This function is the entry point of the transform. The +/// following things may be initialized here: +/// 1) Query for MF_DEVICEMFT_CONNECTED_FILTER_KSCONTROL on the +/// attributes supplied. +HRESULT SampleDMFT::InitializeTransform(_In_ IMFAttributes *pAttributes) +{ + HRESULT hr = S_OK; + wil::com_ptr_nothrow spFilterUnk; +// … + hr = pAttributes->GetUnknown(MF_DEVICEMFT_CONNECTED_FILTER_KSCONTROL, IID_PPV_ARGS(&spFilterUnk)); + if(hr != S_OK) + { + return hr; + } + + hr = spFilterUnk.query_to(IID_PPV_ARGS(&m_spKsControl)); + if(hr != S_OK) + { + return hr; + } +// … +} + +/// +/// This function is the intercept point for SET/GET KSProperty. +/// This sample shows how to intercept the KSPROPERTY_CAMERACONTROL_WINDOWSTUDIO_SETNOTIFICATION DDI from a DMFT standpoint that wants to be alerted of changes to both DigitalWindow and CreativeFilter DDIs in the Windows Studio Effect driver component. +HRESULT SampleDMFT::KsProperty( + _In_reads_bytes_(ulPropertyLength) PKSPROPERTY pProperty, + _In_ ULONG ulPropertyLength, + _Inout_updates_to_(ulDataLength, *pulBytesReturned) LPVOID pPropertyData, + _In_ ULONG ulDataLength, + _Out_ ULONG* pulBytesReturned +) +{ + RETURN_HR_IF_NULL(E_POINTER, pProperty); + if (ulPropertyLength < sizeof(KSPROPERTY)) + { + RETURN_IF_FAILED(E_INVALIDARG); + } + + wil::com_ptr_nothrow m_spKsControl; + + if (IsEqualCLSID(pProperty->Set, KSPROPERTYSETID_WindowsCameraEffect)) + { + if (pProperty->Id == KSPROPERTY_CAMERACONTROL_WINDOWSTUDIO_SETNOTIFICATION) + { + // --GET-- + if (pProperty->Flags & KSPROPERTY_TYPE_GET) + { + if (ulDataLength < sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE)) + { + *pulBytesReturned = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE); + return HRESULT_FROM_WIN32(ERROR_MORE_DATA); + } + else if (pPropertyData) + { + KSCAMERA_EXTENDEDPROP_HEADER* pExtendedHeader = (KSCAMERA_EXTENDEDPROP_HEADER*)pPropertyData; + + // query the next driver component and if it supports this DDI, aggregate this DMFT's capability + HRESULT hr = m_spKsControl->KsProperty(pProperty, ulPropertyLength, pPropertyData, ulDataLength, pulBytesReturned); + if (hr == S_OK) + { + pExtendedHeader->Capability |= (KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_DIGITALWINDOW | KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_CREATIVEFILTER); + } + else + { + pExtendedHeader->Capability = (KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_DIGITALWINDOW | KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_CREATIVEFILTER); + } + pExtendedHeader->Flags = 0; + pExtendedHeader->Result = 0; + pExtendedHeader->Size = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE); + pExtendedHeader->Version = 1; + pExtendedHeader->PinId = 0xFFFFFFFF; + + *pulBytesReturned = pExtendedHeader->Size; + } + else + { + return E_INVALIDARG; + } + } + // --SET-- + else if (pProperty->Flags & KSPROPERTY_TYPE_SET) + { + ULONG expectedSize = sizeof(KSCAMERA_EXTENDEDPROP_HEADER); + if (ulDataLength < expectedSize) + { + *pulBytesReturned = expectedSize; + return HRESULT_FROM_WIN32(ERROR_MORE_DATA); + } + else if (!pPropertyData) + { + return E_INVALIDARG; + } + + // send the payload to the next driver component, in this case we don't care about the return value + (void)m_spKsControl->KsProperty(pProperty, ulPropertyLength, pPropertyData, ulDataLength, pulBytesReturned); + + // parse the payload + BYTE* pPayload = (BYTE*)pPropertyData; + KSCAMERA_EXTENDEDPROP_HEADER* pExtendedHeader = (KSCAMERA_EXTENDEDPROP_HEADER*)pPayload; + + // if this is a SET notification we care about + if (KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_DIGITALWINDOW == pExtendedHeader->Flags) + { + // we expect the size of the payload for a DigitalWindow SET call, see spec for KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW + expectedSize += (sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_DIGITALWINDOW_SETTING)); + if (ulDataLength < expectedSize) + { + *pulBytesReturned = expectedSize; + return HRESULT_FROM_WIN32(ERROR_MORE_DATA); + } + + KSCAMERA_EXTENDEDPROP_HEADER* pDigitalWindowPayloadExtendedHeader = pExtendedHeader + 1; + KSCAMERA_EXTENDEDPROP_DIGITALWINDOW_SETTING* pDigitalWindowPaySettings = (KSCAMERA_EXTENDEDPROP_DIGITALWINDOW_SETTING*)(pDigitalWindowPayloadExtendedHeader + 1); + + // read the parts of the payload and take action accordingly + if (pDigitalWindowPayloadExtendedHeader->Flags == KSCAMERA_EXTENDEDPROP_DIGITALWINDOW_AUTOFACEFRAMING) + { + // Automatic Framing has been enabled, do something differently internally + } + else + { + // Automatic Framing is disabled, read the window bounds used i.e. + // pDigitalWindowPaySettings->OriginX, pDigitalWindowPaySettings->OriginY, pDigitalWindowPaySettings->WindowSize + // and do something differently internally + } + } + else if (KSCAMERA_WINDOWSSTUDIO_SETNOTIFICATION_CREATIVEFILTER == pExtendedHeader->Flags) + { + // we expect the size of the payload for a CreativeFilter SET call, see spec for KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER + expectedSize += (sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE)); + if (ulDataLength < expectedSize) + { + *pulBytesReturned = expectedSize; + return HRESULT_FROM_WIN32(ERROR_MORE_DATA); + } + + KSCAMERA_EXTENDEDPROP_HEADER* pCreativeFilterPayloadExtendedHeader = pExtendedHeader + 1; + // read the parts of the payload and if creative filter is enabled, take action accordingly + if (pCreativeFilterPayloadExtendedHeader->Flags != KSCAMERA_WINDOWSSTUDIO_CREATIVEFILTER_OFF) + { + // Creative filter has been enabled, do something differently internally + } + else + { + // Creative filter is disabled, do something differently internally + } + } + else + { + return E_INVALIDARG; + } + } + // --GETPAYLOAD-- + else if (pProperty->Flags & KSPROPERTY_TYPE_GETPAYLOADSIZE) + { + *pulBytesReturned = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE); + } + } + else + { + return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED); + } + } + else + { + return HRESULT_FROM_WIN32(ERROR_SET_NOT_FOUND); + } + return S_OK; +} +~~~ + +---------- + + +---------- +## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP Control + +>**This DDI is intended for other camera driver component to implement, not application.** + +The KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP DDI allows *WSE* to query from a driver component the current sensor center crop performed by the ISP to produce the current output MediaType resolution set on the streams exposed by the source. It is possible that upon starting a stream, the ISP may choose to change the exercised area of the sensor in order to produce frames on 1 or more streams concurrently. If that’s the case, in order to perform correctly both lens distortion correction and the new Cinematic Director type of Automatic Framing, *WSE* need to pull this information. The following DDI is the way for the camera driver to declare this sensor crop information. + +Implementing this DDI is optional: +- If this DDI is not supported by the source driver but the above device property keys are exposed (DEVPKEY_WindowsStudio_PinholeIntrinsics, DEVPKEY_WindowsStudio_LensDistortionModel_6k or DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear) then it is assumed that all MediaTypes are sampled using the maximum area of the sensor allowed by their aspect ratio. + + +| Example of possible sensor crop | Example values returned via DDI assuming SensorWidth = 3840, SensorHeight = 2640 for the sensor center crop (green) | Example sensor area leveraged to service MediaType (red) | +| -- | -- | -- | +| ![sensor crop example 1](./DocAssets/SensorCrop1.png) | Size.cx = 3840
Size.cy = 2160 | Mediatype is of resolution 1920 x 1440, We infer then that the sensor area covered is:
2880 x 2160 | +| ![sensor crop example 2](./DocAssets/SensorCrop2.png) | Size.cx = 3520
Size.cy = 2160 | Mediatype is of resolution 1920 x 1080, We infer then that the sensor area covered is:
2880 x 2160 | +| ![sensor crop example 3](./DocAssets/SensorCrop3.png) | Size.cx = 1600
Size.cy = 1600 | Mediatype is of resolution 800 x 800, We infer then that the sensor area covered is:
1600 x 1600 | + +>**A driver component such as a DMFT that does not implement this DDI call shall relay it to the subsequent component** ### Usage Summary | Scope | Control | Type | GET | SET | | ----- | ----- | ----- | ----- | ----- | -| Version 1 | Filter | Synchronous| ✔️ | ❌ | +| Version 1 | Filter | Synchronous| ✔️ | ❌ | -The SET call of this control will fail. +The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of a [KSCAMERA_EXTENDEDPROP_VALUE](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_value) that actualy contains only a struct of type [SIZE](https://learn.microsoft.com/en-us/windows/win32/api/windef/ns-windef-size) (64 bits, see *windef.h*) detailing the current center crop width and height in pixels given the output MediaType(s) set on the available stream(s). -The KSPROPERTY payload follows the same layout as traditional KSPROPERTY_CAMERACONTROL_EXTENDED_PROPERTY, comprised of a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content of size of *sizeof(KSPROPERTY) \** ***n*** -where ***“n”*** is the count of KSPROPERTY reported. +This is a GET only DDI, a SET call of this control will fail. +#### KSCAMERA_EXTENDEDPROP_HEADER +| **Member** | **Description** | +| ----- | ----- | +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | +| **Result** | Unused, must be 0 | +| **Capability** | Unused, must be 0 | +| **Flags** | Unused, must be 0 | +#### SIZE +| **Member** | **Description** | +| ----- | ----- | +| **cx** | Sensor center crop width in pixels | +| **cy** | Sensor center crop height in pixels | + + +This is an example of how a device MFT may report support for and relay the current KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP payload. ~~~cpp -// PerformanceMitigation possible flags values -#define KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_NONE 0x0000000000000000 -#define KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_EYEGAZECORRECTION_STARE 0x0000000000000001 +// .h +// redefine locally GUID defined in OS SDK ksmedia.h for convenience +static GUID KSPROPERTYSETID_WindowsStudioCameraControl = { 0x1666d655, 0x21a6, 0x4982, 0x97, 0x28, 0x52, 0xc3, 0x9e, 0x86, 0x9f, 0x90 }; + +// redefine locally +typedef enum { + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED = 0, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT = 1, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER = 2, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION = 3, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA = 5, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP = 6, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND = 7, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT = 8 +} KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PROPERTY; + +class SampleDMFT + : public IMFDeviceTransform, +// … +{ +public: + HRESULT InitializeTransform(_In_ IMFAttributes* pAttributes) override; + HRESULT KSProperty( + _In_reads_bytes_(ulPropertyLength) PKSPROPERTY pProperty, + _In_ ULONG ulPropertyLength, + _Inout_updates_to_(ulDataLength, *pulBytesReturned) LPVOID pPropertyData, + _In_ ULONG ulDataLength, + _Out_ ULONG* pulBytesReturned) override; + wil::com_ptr_nothrow m_spKsControl; +// … +} + +// .cpp + +/// +/// This function is the entry point of the transform. The +/// following things may be initialized here: +/// 1) Query for MF_DEVICEMFT_CONNECTED_FILTER_KSCONTROL on the +/// attributes supplied. +HRESULT SampleDMFT::InitializeTransform(_In_ IMFAttributes *pAttributes) +{ + HRESULT hr = S_OK; + wil::com_ptr_nothrow spFilterUnk; +// … + hr = pAttributes->GetUnknown(MF_DEVICEMFT_CONNECTED_FILTER_KSCONTROL, IID_PPV_ARGS(&spFilterUnk)); + if(hr != S_OK) + { + return hr; + } + + hr = spFilterUnk.query_to(IID_PPV_ARGS(&m_spKsControl)); + if(hr != S_OK) + { + return hr; + } +// … +} + +/// +/// This function is the intercept point for SET/GET KSProperty. +/// This sample shows how to intercept the KSPROPERTY_CAMERACONTROL_WINDOWSTUDIO_SENSORCENTERCROP DDI from a DMFT standpoint that wants to declare the current sensor center crop exercised by the ISP. +HRESULT SampleDMFT::KsProperty( + _In_reads_bytes_(ulPropertyLength) PKSPROPERTY pProperty, + _In_ ULONG ulPropertyLength, + _Inout_updates_to_(ulDataLength, *pulBytesReturned) LPVOID pPropertyData, + _In_ ULONG ulDataLength, + _Out_ ULONG* pulBytesReturned +) +{ + RETURN_HR_IF_NULL(E_POINTER, pProperty); + if (ulPropertyLength < sizeof(KSPROPERTY)) + { + RETURN_IF_FAILED(E_INVALIDARG); + } + + wil::com_ptr_nothrow m_spKsControl; + + if (IsEqualCLSID(pProperty->Set, KSPROPERTYSETID_WindowsCameraEffect)) + { + if (pProperty->Id == KSPROPERTY_CAMERACONTROL_WINDOWSTUDIO_SENSORCENTERCROP) + { + // --GET-- + if (pProperty->Flags & KSPROPERTY_TYPE_GET) + { + if (ulDataLength < sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE)) + { + *pulBytesReturned = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE); + return HRESULT_FROM_WIN32(ERROR_MORE_DATA); + } + else if (pPropertyData) + { + KSCAMERA_EXTENDEDPROP_HEADER* pExtendedHeader = (KSCAMERA_EXTENDEDPROP_HEADER*)pPropertyData; + SIZE* pSensorCropDimension = (SIZE*)(pExtendedHeader + 1); + + // return the current sensor crop exercised to produce frames + pExtendedHeader->Capability 0; + pExtendedHeader->Flags = 0; + pExtendedHeader->Result = 0; + pExtendedHeader->Size = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE); + pExtendedHeader->Version = 1; + pExtendedHeader->PinId = 0xFFFFFFFF; + + // in this example we return that ISP uses a 16:9 sensor center crop (3200x1800) + *pSensorCropDimension = {3200, 1800}; + + *pulBytesReturned = pExtendedHeader->Size; + } + else + { + return E_INVALIDARG; + } + } + // --SET-- + else + { + return E_INVALIDARG; + } + } + // other property id than KSPROPERTY_CAMERACONTROL_WINDOWSTUDIO_SENSORCENTERCROP.. + else + { + return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED); + } + } + // other property set than KSPROPERTYSETID_WindowsCameraEffect.. + else + { + return HRESULT_FROM_WIN32(ERROR_SET_NOT_FOUND); + } + return S_OK; +} +~~~ + +---------- + +---------- +## KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT Control + +>**This DDI is intended for other camera driver component to implement, not application.** + +This DDI allows a driver component running prior to *WSE* to be notified of a viewport update. *WSE* may crop its input frames using the provided set of corners in order to produce its output frames. This is the case such as when applying Automatic Framing, correcting lens distortion or when adjusting field of view, pan or tilt values. The viewport is a quadrilateral and not necessarily a rectangle. *WSE* will query for the support of this DDI and if the driver reports support, then *WSE* will send down the viewport corners from its input image accordingly where each relative coordinate of a corner is in floating point in Q31 format. + +Implementing this DDI is optional: +- If this DDI is not supported by the source driver but the above device property keys are exposed (DEVPKEY_WindowsStudio_PinholeIntrinsics, DEVPKEY_WindowsStudio_LensDistortionModel_6k or DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear) then it is assumed that all MediaTypes are sampled using the maximum area of the sensor allowed by their aspect ratio. + +| Input image to *WSE*, green cropping quadrilateral corners in red. Those red points coordinates are sent down to driver if the DDI is supported. | WSE output image resulting from cropping the input image using the green quadrilateral | +| -- | -- | +| ![input to *WSE* with viewport updated control supported](./DocAssets/ViewportUpdated1.png) | ![output to *WSE* with viewport updated control supported](./DocAssets/ViewportUpdated2.png) | + +>**If a driver component intercepts this KSProperty DDI, it shall relay any SET payload if supported to the subsequent component in which case it needs to adjust coordinates according to its own manipulation of the viewport if there are any** + +### Usage Summary +| Scope | Control | Type | GET | SET | +| ----- | ----- | ----- | ----- | ----- | +| Version 1 | Filter | Synchronous| ✔️ | ✔️ | + +The KSPROPERTY payload follows the following layout: a [KSCAMERA_EXTENDEDPROP_HEADER](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ksmedia/ns-ksmedia-tagkscamera_extendedprop_header) followed by a content payload of type ***WSEViewportCorners***. + +~~~cpp +// custom struct containing the coordinates of the corners for the updated viewport sampled by WSE from its input frame to produce its output frame +struct WSEViewportCorners +{ + POINT TopLeft; + POINT TopRight; + POINT BottomRight; + POINT BottomLeft; +}; ~~~ #### KSCAMERA_EXTENDEDPROP_HEADER | **Member** | **Description** | | ----- | ----- | -| **Version** | Must be 1 | -| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). -| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + n * sizeof(KSPROPERTY) where “n” is the count of KSPROPERTY reported | -| **Result** | Unused, must be 0 -| **Capability** | ***GET call:*** Bitmask of supported flag values (KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_*). Must at least contain a different valid potential flag value than KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_NONE. -| **Flags** | ***GET call:*** Bitmask of current flag value in operation **(KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_\*)**. For example, if the flags value is KSCAMERA_WINDOWSSTUDIO_PERFORMANCEMITIGATION_NONE, then it means no performance mitigation are currently applied. \ No newline at end of file +| **Version** | Must be 1 | +| **PinId** | This must be KSCAMERA_EXTENDEDPROP_FILTERSCOPE (0xFFFFFFFF). | +| **Size** | sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(KSCAMERA_EXTENDEDPROP_VALUE) | +| **Result** | Unused, must be 0 | +| **Capability** | Unused, must be 0 | +| **Flags** | Unused, must be 0 | +#### WSEViewportCorners +| **Member** | **Description** | +| ----- | ----- | +| **TopLeft** | (LONG x, LONG y) top left corner in relative coordinates, floating points in Q31 format | +| **TopRight** | (LONG x, LONG y) top right corner in relative coordinates, floating points in Q31 format | +| **BottomRight** | (LONG x, LONG y) bottom right corner in relative coordinates, floating points in Q31 format | +| **BottomLeft** | (LONG x, LONG y) bottom left corner in relative coordinates, floating points in Q31 format | + + +This is an example of how a DMFT would support KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT: +~~~cpp +// .h +// redefine locally GUID defined in OS SDK ksmedia.h for convenience +static GUID KSPROPERTYSETID_WindowsStudioCameraControl = { 0x1666d655, 0x21a6, 0x4982, 0x97, 0x28, 0x52, 0xc3, 0x9e, 0x86, 0x9f, 0x90 }; + +typedef enum { + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SUPPORTED = 0, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT = 1, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER = 2, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION = 3, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA = 5, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP = 6, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND = 7, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT = 8 + // … +} KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PROPERTY; + +struct WSEViewportCorners +{ + POINT TopLeft; + POINT TopRight; + POINT BottomRight; + POINT BottomLeft; +}; + +// helper function to derive floating point representation in Q(n) +constexpr +LONGLONG +BASE_Q(ULONG n) +{ + return (((LONGLONG)1) << n); +} + +// Returns the value from x in fixed - point Q31 format. +constexpr +float +FROM_Q31(LONGLONG x) +{ + return (float)x / (float)BASE_Q(31); +} + +class SampleDMFT + : public IMFDeviceTransform, +// … +{ +public: + HRESULT InitializeTransform(_In_ IMFAttributes* pAttributes) override; + HRESULT KSProperty( + _In_reads_bytes_(ulPropertyLength) PKSPROPERTY pProperty, + _In_ ULONG ulPropertyLength, + _Inout_updates_to_(ulDataLength, *pulBytesReturned) LPVOID pPropertyData, + _In_ ULONG ulDataLength, + _Out_ ULONG* pulBytesReturned) override; + wil::com_ptr_nothrow m_spKsControl; + + ULONG m_imageWidth; + ULONG m_imageHeight; + +// … +} + +// .cpp + +/// +/// This function is the entry point of the transform. The +/// following things may be initialized here: +/// 1) Query for MF_DEVICEMFT_CONNECTED_FILTER_KSCONTROL on the +/// attributes supplied. +HRESULT SampleDMFT::InitializeTransform(_In_ IMFAttributes *pAttributes) +{ + HRESULT hr = S_OK; + wil::com_ptr_nothrow spFilterUnk; +// … + hr = pAttributes->GetUnknown(MF_DEVICEMFT_CONNECTED_FILTER_KSCONTROL, IID_PPV_ARGS(&spFilterUnk)); + if(hr != S_OK) + { + return hr; + } + + hr = spFilterUnk.query_to(IID_PPV_ARGS(&m_spKsControl)); + if(hr != S_OK) + { + return hr; + } +// … +} + +/// +/// This function is the intercept point for SET/GET KSProperty. +/// This sample shows how to intercept the KSPROPERTY_CAMERACONTROL_WINDOWSTUDIO_UPDATED_VIEWPORT DDI from a DMFT standpoint. +HRESULT SampleDMFT::KsProperty( + _In_reads_bytes_(ulPropertyLength) PKSPROPERTY pProperty, + _In_ ULONG ulPropertyLength, + _Inout_updates_to_(ulDataLength, *pulBytesReturned) LPVOID pPropertyData, + _In_ ULONG ulDataLength, + _Out_ ULONG* pulBytesReturned +) +{ + RETURN_HR_IF_NULL(E_POINTER, pProperty); + if (ulPropertyLength < sizeof(KSPROPERTY)) + { + RETURN_IF_FAILED(E_INVALIDARG); + } + + // obtain IKsControl for the driver stack below.. + wil::com_ptr_nothrow m_spKsControl; + + if (IsEqualCLSID(pProperty->Set, KSPROPERTYSETID_WindowsCameraEffect)) + { + if (pProperty->Id == KSPROPERTY_CAMERACONTROL_WINDOWSTUDIO_UPDATED_VIEWPORT) + { + if (ulDataLength < sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(WSEViewportCorners)) + { + *pulBytesReturned = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(WSEViewportCorners); + return HRESULT_FROM_WIN32(ERROR_MORE_DATA); + } + if (pPropertyData == nullptr) + { + return E_INVALIDARG; + } + + // --GET-- + if (pProperty->Flags & KSPROPERTY_TYPE_GET) + { + if (pPropertyData) + { + KSCAMERA_EXTENDEDPROP_HEADER* pExtendedHeader = (KSCAMERA_EXTENDEDPROP_HEADER*)pPropertyData; + + pExtendedHeader->Flags = 0; + pExtendedHeader->Result = 0; + pExtendedHeader->Capability = 0; + pExtendedHeader->Size = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(WSEViewportCorners); + pExtendedHeader->Version = 1; + pExtendedHeader->PinId = 0xFFFFFFFF; + + *pulBytesReturned = pExtendedHeader->Size; + } + } + // --SET-- + else if (pProperty->Flags & KSPROPERTY_TYPE_SET) + { + // parse the payload + BYTE* pPayload = (BYTE*)pPropertyData; + KSCAMERA_EXTENDEDPROP_HEADER* pExtendedHeader = (KSCAMERA_EXTENDEDPROP_HEADER*)pPayload; + WSEViewportCorners* pViewportCorners = (WSEViewportCorners *)pPayload; + + // handle the corners relative coordinates in Q31 format, and retrieve absolute image coordinates + ULONG topLeftAbsoluteX = FROM_Q31(pViewportCorners->TopLeft.x) * m_imageWidth; + ULONG topLeftAbsoluteY = FROM_Q31(pViewportCorners->TopLeft.y) * m_imageHeight; + ULONG topRightAbsoluteX = FROM_Q31(pViewportCorners->TopRight.x) * m_imageWidth; + ULONG topRightAbsoluteY = FROM_Q31(pViewportCorners->TopRight.y) * m_imageHeight; + ULONG bottomRightAbsoluteX = FROM_Q31(pViewportCorners->BottomRight.x) * m_imageWidth; + ULONG bottomRightAbsoluteY = FROM_Q31(pViewportCorners->BottomRight.y) * m_imageHeight; + ULONG bottomLeftAbsoluteX = FROM_Q31(pViewportCorners->BottomLeft.x) * m_imageWidth; + ULONG bottomLeftAbsoluteY = FROM_Q31(pViewportCorners->BottomLeft.y) * m_imageHeight; + + // … + + } + // --GETPAYLOAD-- + else if (pProperty->Flags & KSPROPERTY_TYPE_GETPAYLOADSIZE) + { + *pulBytesReturned = sizeof(KSCAMERA_EXTENDEDPROP_HEADER) + sizeof(WSEViewportCorners); + } + } + else + { + return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED); + } + } + else + { + return HRESULT_FROM_WIN32(ERROR_SET_NOT_FOUND); + } + + // relay to the next driver component + (void)m_spKsControl->KsProperty(pProperty, ulPropertyLength, pPropertyData, ulDataLength, pulBytesReturned); + + return S_OK; +} +~~~ + +---------- + +---------- + +# *WSE* Device properties +Windows Studio camera will also behave differently given a certain set of custom device properties. This section details those device properties and their effect. + +## High resolution modes + +| | | +| -- | -- | +| **Name** | DEVPKEY_WindowsStudio_ScaleFromHigherResolution | +| **Property GUID** | AA3E8B1E-B590-4E50-90C6-780AEC4EB4D9 | +| **Property ID** | 2 | +| **Type** | DEVPROP_TYPE_UINT32 | +| **Required** | Optional | +| **Values** | 0 – disabled
1 – HighRes mode 1 enabled
2 – HighRes mode 2 enabled
3 – HighRes mode 3 enabled | + +#### C++ definition: + +~~~cpp +// define locally GUID for Windows Studio DEVPROPKEY +DEFINE_DEVPROPKEY(DEVPKEY_WindowsStudio_ScaleFromHigherResolution, + 0xaa3e8b1e, 0xb590, 0x4e50, 0x90, 0xc6, 0x78, 0xa, 0xec, 0x4e, 0xb4, 0xd9, 2); /* DEVPROP_TYPE_UINT32 */ +~~~ + +#### .inf excerpt to define the above property: +~~~ +[Camera.AddProperty] +; Set property to operate in high resolution mode, potentially starting the stream at a higher resolution than the output from WSE. In this example high resolution mode is set to 3 + +{AA3E8B1E-B590-4E50-90C6-780AEC4EB4D9},2,7,,3 +~~~ + + +The optional device property *DEVPKEY_WindowsStudio_ScaleFromHigherResolution* is a UINT32 value that enables the Windows Studio camera component to attempt to stream from its source and process at a higher input resolution than what it outputs. This can enhance image quality when applying effects such as Automatic Framing. Therefore, if the app’s desired resolution triggers a higher resolution from the source to be used under the hood by *WSE* as input, it may consume more compute resources which comes as a tradeoff for better image quality. The higher resolution chosen correlates with the desired framerate and aspect ratio of the MediaType requested by the application at up to the maximum input capability in *WSE* i.e. up to 1440p (see below diagram for each presets) + +There are 3 types of high resolution mode that can be specified: + +1. When the DEVPROP_TYPE_UINT32 value is set to "1", it requests *WSE* to attempt to use the highest resolution supported at all time. For example, if application desires 640x360@30fps, if the driver advertises a MediaType of 2560x1440@30fps, then WSE would stream at 2560x1440@30fps from the base camera, process all effects at that resolution then scale and output the frame at 640x360@30fps: +![MediaType leveraged with High Res mode set to 1](./DocAssets/HighRes1.png) + +2. When the DEVPROP_TYPE_UINT32 value is set to "2", it requests MEP to attempt to use the highest resolution that is closest to 4 times the pixel resolution to the desired output (i.e. closest to 2xWidth and 2xHeight). For example, if application desires 640x360@30fps, if the driver advertises a MediaType of 1280x720@30fps, then WSE would stream at 1280x720@30fps from the base camera, process all effects at that resolution then scale and output the frame at 640x360@30fps: +![MediaType leveraged with High Res mode set to 2](./DocAssets/HighRes2.png) + +3. When the DEVPROP_TYPE_UINT32 value is set to "3", MEP will change its behavior dynamically given the state of the Automatic Framing effect. + 1. It will behave as if the value was set to "2" when the Automatic Framing effect is turned ON. + 2. If the Automatic Framing is turned OFF, it will behave as if the value was "0", meaning it will not attempt to stream at a higher resolution. + + This mode of operation will therefore potentially switch the base camera MediaType on the fly when Automatic Framing is engaged or disengaged (i.e. if the MediaType requested by the app is not what MEP would leverage from the camera driver in "High Res mode 2") causing a stream restart which may have some drawbacks that demand careful consideration: + - While switching MediaType, the stream needs to be restarted which may drop some frames, appearing as if the frame froze for that amount of time. + - When switching MediaType, the camera driver may reset its 3A (auto-white balance, auto-focus, auto-exposure) causing a "flicker" image quality artifact + - The hardware LED that notifies when the camera is turned on may be turned off then on again when engaging or disengaging Automatic Framing due to the stream restarting +This "High Res mode 3" is only supported on Windows build 26100 and above with latest MEP driver. + +---------- + +## High resolution mode conditional on power state + +| | | +| -- | -- | +| **Name** | DEVPKEY_WindowsStudio_HighResolutionModeOverrideInDC | +| **Property GUID** | AA3E8B1E-B590-4E50-90C6-780AEC4EB4D9 | +| **Property ID** | 3 | +| **Type** | DEVPROP_TYPE_UINT32 | +| **Required** | Optional, has no effect if the other property *DEVPKEY_WindowsStudio_ScaleFromHigherResolution* is set to 0 or not specified | +| **Values** | 0 – disabled
1 – enabled | + +#### C++ definition: + +~~~cpp +// define locally GUID for Windows Studio DEVPROPKEY +DEFINE_DEVPROPKEY(DEVPKEY_WindowsStudio_HighResolutionModeOverrideInDC, + 0xaa3e8b1e, 0xb590, 0x4e50, 0x90, 0xc6, 0x78, 0xa, 0xec, 0x4e, 0xb4, 0xd9, 3); /* DEVPROP_TYPE_UINT32 */ +~~~ + +#### .inf excerpt to define the above property: +~~~ +[Camera.AddProperty] +; Set property to operate in the high resolution mode specified by the value of DEVPKEY_WindowsStudio_ScaleFromHigherResolution only when the device is plugged in when the stream is started. If the device is using its battery instead at the moment the stream is started, then the high resolution mode is disabled (DEVPKEY_WindowsStudio_ScaleFromHigherResolution == 0). + +{AA3E8B1E-B590-4E50-90C6-780AEC4EB4D9},3,7,,1 +~~~ + +The optional device property *DEVPKEY_WindowsStudio_HighResolutionModeOverrideInDC* is a UINT32 value that adds a condition for Windows Studio camera component when attempting to leverage the high resolution mode prescribed by the other device property named *DEVPKEY_WindowsStudio_ScaleFromHigherResolution* when starting a stream. +- if the device is in DC power (battery) when the stream starts, this property causes an override and disables the high resolution mode. +- Similarly, if the device is feeding from AC power (plugged-in) when the stream is started then the device may be streaming at a higher resolution. + +**This property will not cause a dynamic change of the stream resolution if the power state changes while the stream is running; this condition only applies upon starting the stream.** + +---------- + +## Face detection DDI support and frequency of face-based ROIs for 3A driven by *WSE* + +| | | +| -- | -- | +| **Name** | DEVPKEY_WindowsStudio_ImplementFaceDetectionFor3A | +| **Property GUID** | EAC24D1C-F801-4EC7-BBA6-A8B38314FFE5 | +| **Property ID** | 2 | +| **Type** | DEVPROP_TYPE_UINT32 | +| **Required** | Optional | +| **Values** | 0 – disabled
1 – enabled | + + +| | | +| -- | -- | +| **Name** | DEVPKEY_WindowsStudio_HighResolutionModeOverrideInDC | +| **Property GUID** | EAC24D1C-F801-4EC7-BBA6-A8B38314FFE5 | +| **Property ID** | 3 | +| **Type** | DEVPROP_TYPE_UINT32 | +| **Required** | Optional, requires *DEVPKEY_WindowsStudio_ImplementFaceDetectionFor3A* to be specified | +| **Values** | Interval in milliseconds at which face-based ROI(s) are evaluated and sent down to adjust 3A. This interval is then aligned with framerate where once a frame with timestamp ‘t’ is used to calculate ROI, the next frame with timestamp > ‘t + value’ is used.
This serves also as the minimum interval at which face detection is run and therefore the sample metadata is refreshed. | + + +#### C++ definition: + +~~~cpp +// define locally GUID for Windows Studio DEVPROPKEY +DEFINE_DEVPROPKEY(DEVPKEY_WindowsStudio_ImplementFaceDetectionFor3A, + 0xeac24d1c, 0xf801, 0x4ec7, 0xbb, 0xa6, 0xa8, 0xb3, 0x83, 0x14, 0xff, 0xe5, 2); /* DEVPROP_TYPE_UINT32 */ + +DEFINE_DEVPROPKEY(DEVPKEY_WindowsStudio_3AFaceROIIntervalInMs, + 0xeac24d1c, 0xf801, 0x4ec7, 0xbb, 0xa6, 0xa8, 0xb3, 0x83, 0x14, 0xff, 0xe5, 3); /* DEVPROP_TYPE_UINT32 */ +~~~ + +#### .inf excerpt to define the above property: +~~~ +[Camera.AddProperty] +; Set property to operate face detection in Windows Studio and execute face detection at least every 200ms. If the driver supports the DDI for receiving ROI for ISP 3A, face-based ROIs will also be refreshed every 200ms + +{EAC24D1C-F801-4EC7-BBA6-A8B38314FFE5},2,7,,1 +{EAC24D1C-F801-4EC7-BBA6-A8B38314FFE5},3,7,,200 +~~~ + +The optional device property *DEVPKEY_WindowsStudio_ImplementFaceDetectionFor3A* is a UINT32 value that enables (value = 1) or disables (value = 0) standalone face detection capability in Windows Studio camera component applied to any one of video record or video preview stream. +- If this device property is not specified in the .inf of a camera opting into Windows Studio Effect, its default value is 0. +- However, if the camera is opted into Windows Studio Effect by other means than .inf such as manually by the user, the default value is instead 1. + +When enabled, this has the effect that Windows Studio camera component now supports the camera extended control [KSPROPERTY_CAMERACONTROL_EXTENDED_FACEDETECTION](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-facedetection). Per specification, if the client enables face detection via this DDI, the detected face regions of interest (ROIs) are then attached to each sample flowing out of the Windows Studio camera component via a supported stream (see [MF_CAPTURE_METADATA_FACEROIS](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/mf-capture-metadata#mf_capture_metadata_facerois)). + +Additionally, similar to the [OS Platform DMFT](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/uvc-camera-implementation-guide#platform-device-mft) , if the base driver supports [KSPROPERTY_CAMERACONTROL_EXTENDED_ROI_ISPCONTROL](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-facedetection), Windows Studio camera component will drive 3A using up to 2 predominant face ROIs. If both Platform DMFT and Windows Studio are opted-in and this device property is enabled, then only Windows Studio would perform this role, deactivating the redundant logic in Platform DMFT. + +The optional device property *DEVPKEY_WindowsStudio_3AFaceROIIntervalInMs* prescribes the minimum time interval in milliseconds at which the Windows Studio camera component will send down the ROI(s) via KSPROPERTY_CAMERACONTROL_EXTENDED_ROI_ISPCONTROL. This interval is then aligned with framerate where once a frame with timestamp ‘t’ is used to calculate face ROI(s), the next frame with timestamp > ‘t + value’ is used to reassess the detection of face(s) and send down accordingly either a new set of ROI(s) or a reset if no face is detected. + +The property *DEVPKEY_WindowsStudio_ImplementFaceDetectionFor3A* must be specified if the property *DEVPKEY_WindowsStudio_3AFaceROIIntervalInMs* is specified. +If *DEVPKEY_WindowsStudio_3AFaceROIIntervalInMs* is not specified, the implicit default value shall be 200ms. + + +---------- + +## Camera intrinsics and lens distortion model + +### Lens distortion correction +Lens distortion correction improves the visual quality of images captured by cameras with wide field of view (WFOV). This functionality aims to reduce the "fish-eye" effect that can occur with such cameras, providing a more natural and accurate representation of the scene. + +- Lens distortion correction requires both device properties described below: pinhole intrinsics camera parameters and the lens distortion model (which can be expressed in one of the 2 defined formats) + - If deemed necessary in the camera driver stack running prior to *WSE*, we also provide a mean to obtain the active sensor region that may have been center-cropped further than the full sensor area cropped to fit the aspect ratio of the frame (see [sensor crop mode DDI](#ksproperty_cameracontrol_windowsstudio_sensorcentercrop-control)). *WSE* can also notify that driver stack of which portion of its input frame is leveraged to produce its output frame (i.e. the viewport leveraged when applying lens distortion correction, field of view adjustments such as zooming, panning and tilting) (see [viewport update DDI](#ksproperty_cameracontrol_windowsstudio_updated_viewport-control)) + - If the camera driver prefers to perform its own lens distortion correction, then this feature can be left unsupported, else this functionality should only be performed by *WSE* to avoid incorrect redundant processing. + - Lens distortion correction is applied on all color streams and always active. This processing is not an effect that the user toggles ON or OFF via the Windows settings UIs. + +- When *WSE* is enabled to perform lens distortion correction, it will also implement the camera DDIs for zooming by adjusting the field of view (see [KSPROPERTY_CAMERACONTROL_EXTENDED_FIELDOFVIEW2](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-fieldofview2) and [KSPROPERTY_CAMERACONTROL_EXTENDED_FIELDOFVIEW2_CONFIGCAPS](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-extended-fieldofview2-configcaps)) as well as panning and tilting (see [KSPROPERTY_CAMERACONTROL_PAN](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-pan) and [KSPROPERTY_CAMERACONTROL_TILT](https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/ksproperty-cameracontrol-tilt)) + - If the camera driver stack running prior to *WSE* already implemented these DDIs, *WSE* will override their implementation with its own to ensure that lens distortion correction is correctly aligned. + + +Microsoft strongly recommends enabling Lens Distortion Correction on Wide and Ultra-Wide FOV integrated front-facing cameras: + +- Wide: 16-24mm | Diagonal FOV >= 84° and < 110° +- Ultrawide: < 16mm | Diagonal FOV >= 110° + +### Pinhole intrinsics device property key +The following device property key defines the format to declare the camera intrinsic parameters in a driver: *DEVPKEY_WindowsStudio_PinholeIntrinsics*. +- If this information is provided, *WSE* can perform [Cinematic automatic framing](#ksproperty_cameracontrol_windowsstudio_automaticframingkind-control) more faithfully, especially when the subject is located at the edges of the camera field of view. This is done by applying a perspective transform mimicking a camera pivoting towards the subject as opposed to simply cropping the area of the frame like traditional automatic framing method. +- This device property also allows *WSE* to derive the actual field of view of the camera and optimize other effects around it such as blur. +- This device property is **required** in order for *WSE* to exercise Lens distortion correction. + +| | | +| -- | -- | +| **Name** | DEVPKEY_WindowsStudio_PinholeIntrinsics | +| **Property GUID** | 0A6244AF-D455-4A0F-B43E-F29507A7DEB6 | +| **Property ID** | 2 | +| **Type** | DEVPROP_TYPE_STRING | +| **Required** | Optional, required to support Lens Distortion Correction, highly encouraged to perform Cinematic Automatic framing and background blur optimally | +| **Values** | "\,\,\" | + + +#### *SensorWidthInPixels* and *SensorHeightInPixels* +The camera sensor dimensions are defined by the values *\* and *\*. + +This information relates to the camera sensor; for example, for a camera that captures frames at up to 4K resolution, this would usually be 3840 for *SensorWidthInPixels* and 2160 for *SensorHeightInPixels*. Some cameras have larger resolutions to accommodate both 16:9 and 4:3 mode. The resolution of the sensor is required for proper lens distortion correction. + +#### FocalLengthInPixels +The focal length of the camera lens is defined by the value *\*. + +The value provided needs to be measured in size of pixel on the sensor. For example, if a lens focal length is 2.508 mm and the physical size of a single pixel on the sensor is 1.55 micron (0.00155 mm), the focal length in pixels will be: + +$FocalLengthInPixels = 2.508 mm ÷ 0.00155 mm × 1 pixel = 1618 pixels$ + +The combined values of *\*, *\* and *\* describe basic camera geometry and allows *WSE* to calculate the camera field of view. This is the minimal information required to create a virtual camera rotation when performing [Cinematic automatic framing](#ksproperty_cameracontrol_windowsstudio_automaticframingkind-control) or when panning or tilting while performing lens distortion correction. + +### Lens distortion model device property keys +The following device property keys define two possible formats to declare the lens distortion model of the camera system in a driver: +- *DEVPKEY_WindowsStudio_LensDistortionModel_6k* +- *DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear* + +Therefore, a driver is expected to declare at most either one or the other, but not both. In the event that both are declared, the value of *DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear* would take precedence. **If either of these device property key is defined, it requires that the *DEVPKEY_WindowsStudio_PinholeIntrinsics* device property key is also defined in order to provide the intended lens distortion correction functionality.** + +| | | +| -- | -- | +| **Name** | DEVPKEY_WindowsStudio_LensDistortionModel_6k | +| **Property GUID** | 0A6244AF-D455-4A0F-B43E-F29507A7DEB6 | +| **Property ID** | 3 | +| **Type** | DEVPROP_TYPE_STRING | +| **Required** | Optional, requires DEVPKEY_WindowsStudio_PinholeIntrinsics to be defined. Required to support Lens Distortion Correction if DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear is not defined. | +| **Values** | "\,\,\,\,\,\" | + + +| | | +| -- | -- | +| **Name** | DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear | +| **Property GUID** | 0A6244AF-D455-4A0F-B43E-F29507A7DEB6 | +| **Property ID** | 4 | +| **Type** | DEVPROP_TYPE_STRING | +| **Required** | Optional, requires DEVPKEY_WindowsStudio_PinholeIntrinsics to be defined, supersedes DEVPKEY_WindowsStudio_LensDistortionModel_6k.
Required to support Lens Distortion Correction if DEVPKEY_WindowsStudio_LensDistortionModel_6k is not defined. | +| **Values** | Up to 200 pairs, 400 entries, in pixels:
"\,\, …, \,\" | + +If the lens distortion is not deemed significant enough to be corrected, then these values are not required, and the lens distortion correction functionality will not be applied by WSE. Usually, cameras with narrow field of view (less than 90^0) do not require lens distortion correction, while camera with wider field of view do. + +### DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear +For our application in *WSE*, we think about lens distortion as the impact of refraction on a ray that goes through the center of lens. + +In the figure below, the green ray originates outside of camera and would go straight through a pinhole and therefore If it was just a pinhole camera system, then the ray would continue as a line and touch the sensor. With a lens however, the green ray became refracted by the lens and continued its course towards the sensor as the red line. + +Performing lens distortion correction requires moving image pixels from position of *RadialPositionAfterLensDistortion* to *RadialPositionBeforeLensDistortion*. +![ray distortion](./DocAssets/RayDistortion.png) + +Here is an example on how to create a pair of values mapping *RadialPositionAfterLensDistortion* to *RadialPositionBeforeLensDistortion* using optical characteristic of the lens. + +Profesionnals working with camera lens usually provide mapping from angle α to position on sensor *RadialPositionAfterLensDistortion*. Usually, this information is provided by camera/lens design software such as Zemax OpticStudio. + +In this example, we presume the software outputs two columns: one is angle of the ray (α), and the other is the position on an axis where the way would meet sensor (*RadialPositionAfterLensDistortion*), in millimeters. + +| Offset of ray on sensor (*RadialPositionAfterLensDistortion*) in millimeters | Angle α in degrees | +| -- | -- | +| 0.022997392 | 0.902308601 | +| 0.045979278 | 1.804008861 | +| 0.068930936 | 2.704523059 | +| … | … | +|1.033159104 | 40.53626426 | + +Let’s presume that this table is generated for a particular lens with focal length + +$F = 1.460mm$ + +We assume then that the lens is mounted at distance of F from the sensor. From a camera geometry then we can write: + +$RadialPositionBefoerLensDistortion = F × \tan{(α)}$ + +Using this formula, we can add third column *RadialPosition**Before**LensDistortion*: + +| Offset of ray on sensor (*RadialPositionAfterLensDistortion*) in millimeters | Angle α in degrees | *RadialPositionBeforeLensDistortion*
(F × tan(α)) in mm | +| -- | -- | -- | +| 0.022997392 | 0.902308601 | 0.02299436 | +| 0.045979278 | 1.804008861 | 0.0459846 | +| 0.068930936 | 2.704523059 | 0.068967 +| … | … | … | +| 1.219 | 40.53626426 | 1.248557 | + + +Now that we have both *RadialPositionAfterLensDistortion* and *RadialPositionBeforeLensDistortion* values in mm, we can convert them into pixel metric values. For this particular camera the sensor pixel size was 1.55 micron (0.00155 mm). We need to divide all distances in mm by the sensor pixel size: + +$RadialPositionAfterLensDistortionInPixels = RadialPositionAfterLensDistortion / 0.00155mm * 1pixel$ + + + +| RadialPositionAfterLensDistortion in pixels | RadialPositionBeforeLensDistortion (F × tan(α)) in pixels | +| -- | -- | +| 14.835 | 14.835 | +| 29.664 | 29.66748 | +| 44.47157 | 44.4948 | +| … | … | +| 786.4516 | 805.52064 | + +As we see from the table there is practically no distortion in the center, but it increases towards the edges of the frame. We need the values in the table to cover up to the corner of the frame, which means last entry for *RadialPositionAfterLensDistortion* should be at least half of the sensor diagonal. Therefore, the maximum distance from the optical center that needs to be populated as a value of *RadialPositionAfterLensDistortion*: + +$MaximumDistance = \frac{\sqrt{(SensorWidthInPixels^2+ SensorHeightInPixels^2)}}{2}$ + +![lens maximum distance](./DocAssets/LensMaxDist.png) + + +This table allows *WSE* to generate a mapping of the offset between *RadialPositionAfterLensDistortion* from center to *RadialPositionBeforeLensDistortion* for the whole frame and therefore correct lens distortion in real time. + +### DEVPKEY_WindowsStudio_LensDistortionModel_6k +Alternatively, the driver can provide the ‘6k’ coefficients that describe the lens distortion profile from which we can generate a map. +These coefficients are used in the standard lens correction formula: + +$x_{dist} = x\frac{(1+k_1 r^2+k_2 r^4+k_3 r^6)}{(1+k_4 r^2+k_5 r^4+k_6 r^6 )}$ + +$y_{dist} = y\frac{(1+k_1 r^2+k_2 r^4+k_3 r^6)}{(1+k_4 r^2+k_5 r^4+k_6 r^6 )}$ + +Where $r=\sqrt{(x^2+y^2)}$ + +The $x_{dist}$ and $y_{dist}$ are positions after lens distortion, $x$ and $y$ are desired position on the rectilinear image. During lens correction processing, we take the pixel data at position ($x_{dist},y_{dist}$) and move it to position ($x,y$). In order to create a corrected image, we iterate over ($x,y$), look for data pixel at position ($x_{dist},y_{dist}$) and place it at ($x,y$). + +To obtain coefficients $k_1,k_2,k_3,k_4,k_5,k_6$, OpenCV has a convenient function called calibrateCameraRO(). This function takes a set of points before and after lens distortion (usually a perfect grid). The coefficients $k_1$ to $k_6$ have dimensionality: their value depends on the unit of measure. We measure all coordinates in terms of pixels which is relative to the size of a pixel on the sensor. Position of ($x,y$) and ($x_{dist},y_{dist}$) are measured in sensor pixels (same as image pixels if no down/up scaling is applied). Therefore all coordinates of points provided to calibrateCameraRO() should be in coordinates of pixels on the sensor. Then the function will output coefficient values for $k_1,k_2,k_3,k_4,k_5,k_6$ in a coherent metric to the rest of the device property parameters. + +If the characteristics of the lens are not known, Microsoft will provide a tool that calculates lens distortion profile from images of rectangular chart with grid dots. The image of the chart would look like the below which is a real image captured with a wide field of view lens from which we can then either calculate the $k_1,k_2,k_3,k_4,k_5,k_6$ lens distortion coefficients or generate a map of *RadialPositionBeforeLensDistortion* and *RadialPositionAfterLensDistortion* when paired with the pinhole intrinsics values from DEVPKEY_WindowsStudio_PinholeIntrinsics. The lens distortion model values can then be shared back with the partner device manufacturer for populating accordingly the device property key in their driver. + +![example distortion calibration frame](./DocAssets/DistortionCalibrationExample.png) + +#### C++ definition: + +~~~cpp +// define locally GUID for Windows Studio DEVPROPKEY +DEFINE_DEVPROPKEY(DEVPKEY_WindowsStudio_PinholeIntrinsics, +0x0a6244af,, 0xd455, 0x4a0f, 0xb4, 0x3e, 0xf2, 0x95, 0x07, 0xa7, 0xde, 0xb6, 2); /* DEVPROP_TYPE_STRING_LIST */ + +DEFINE_DEVPROPKEY(DEVPKEY_WindowsStudio_LensDistortionModel_6k, +0x0a6244af,, 0xd455, 0x4a0f, 0xb4, 0x3e, 0xf2, 0x95, 0x07, 0xa7, 0xde, 0xb6, 3); /* DEVPROP_TYPE_STRING_LIST */ + +DEFINE_DEVPROPKEY(DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear, +0x0a6244af,, 0xd455, 0x4a0f, 0xb4, 0x3e, 0xf2, 0x95, 0x07, 0xa7, 0xde, 0xb6, 4); /* DEVPROP_TYPE_STRING_LIST */ + +~~~ + +#### .inf excerpt to define the above property: +~~~ +[Camera.AddProperty] +; Set property to define DEVPKEY_WindowsStudio_PinholeIntrinsics, specifying sensor size in pixels of 3840x2160 as well as focal length in pixels of 1618. + +{0A6244AF-D455-4A0F-B43E-F29507A7DEB6}, 2, 18,, “3840.0,2160.0,1618.0” + + +; Set property to define DEVPKEY_WindowsStudio_LensDistortionModel_6k, specifying the 6 ‘k’ radial distortion coefficients. +; Note that you need to either provide this property or the DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear. If both are defined, the DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear takes precedence over DEVPKEY_WindowsStudio_LensDistortionModel_6k. + +{0A6244AF-D455-4A0F-B43E-F29507A7DEB6}, 3, 18,, ”0.234567,0.142537,0.112233,0.050607,0.045678,0.019876” + + +; Set property to define DEVPKEY_WindowsStudio_LensDistortionModel_PolyLinear, specifying 9 pairs of sensor position after lens distortion and before lens distortion. (There can be up to 200 pairs defined) + +{0A6244AF-D455-4A0F-B43E-F29507A7DEB6}, 3, 18,, “46.0,45.9846,69.0,68.967,92.0,91.94,115.0,114.9,138.0,137.858,1219.0,1248.557,1679.0,1749.0,2162.0,2543.8,2231.0,2714.0” + +~~~ + +---------- \ No newline at end of file diff --git a/Samples/WindowsStudio/WindowsStudioSample.sln b/Samples/WindowsStudio/WindowsStudioSample.sln index ad26766..710f8f1 100644 --- a/Samples/WindowsStudio/WindowsStudioSample.sln +++ b/Samples/WindowsStudio/WindowsStudioSample.sln @@ -1,9 +1,9 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.9.34728.123 +# Visual Studio Version 18 +VisualStudioVersion = 18.1.11304.174 d18.0 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ControlMonitorHelperWinRT", "..\ControlMonitorApp\ControlMonitorHelper\ControlMonitorHelperWinRT.vcxproj", "{3AC253A5-A383-49D8-ACC0-102C1E46E42F}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ControlMonitorHelperWinRT", "..\ControlMonitorApp\ControlMonitorHelper\ControlMonitorHelperWinRT.vcxproj", "{5A835B38-9B32-43CE-90FA-EDC3E4FC8070}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WindowsStudioSample_WinUI", "WindowsStudioSample_WinUI\WindowsStudioSample_WinUI.csproj", "{80DC76D0-9378-4D68-A097-998F601F0F18}" EndProject @@ -19,20 +19,20 @@ Global Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Debug|Any CPU.ActiveCfg = Debug|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Debug|ARM64.Build.0 = Debug|ARM64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Debug|x64.ActiveCfg = Debug|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Debug|x64.Build.0 = Debug|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Debug|x86.ActiveCfg = Debug|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Debug|x86.Build.0 = Debug|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Release|Any CPU.ActiveCfg = Release|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Release|ARM64.ActiveCfg = Release|ARM64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Release|ARM64.Build.0 = Release|ARM64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Release|x64.ActiveCfg = Release|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Release|x64.Build.0 = Release|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Release|x86.ActiveCfg = Release|x64 - {3AC253A5-A383-49D8-ACC0-102C1E46E42F}.Release|x86.Build.0 = Release|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Debug|Any CPU.ActiveCfg = Debug|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Debug|ARM64.Build.0 = Debug|ARM64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Debug|x64.ActiveCfg = Debug|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Debug|x64.Build.0 = Debug|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Debug|x86.ActiveCfg = Debug|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Debug|x86.Build.0 = Debug|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Release|Any CPU.ActiveCfg = Release|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Release|ARM64.ActiveCfg = Release|ARM64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Release|ARM64.Build.0 = Release|ARM64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Release|x64.ActiveCfg = Release|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Release|x64.Build.0 = Release|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Release|x86.ActiveCfg = Release|x64 + {5A835B38-9B32-43CE-90FA-EDC3E4FC8070}.Release|x86.Build.0 = Release|x64 {80DC76D0-9378-4D68-A097-998F601F0F18}.Debug|Any CPU.ActiveCfg = Debug|x64 {80DC76D0-9378-4D68-A097-998F601F0F18}.Debug|Any CPU.Build.0 = Debug|x64 {80DC76D0-9378-4D68-A097-998F601F0F18}.Debug|Any CPU.Deploy.0 = Debug|x64 diff --git a/Samples/WindowsStudio/WindowsStudioSample_WinUI/KsHelper.cs b/Samples/WindowsStudio/WindowsStudioSample_WinUI/KsHelper.cs index 704c4ed..9b5c70d 100644 --- a/Samples/WindowsStudio/WindowsStudioSample_WinUI/KsHelper.cs +++ b/Samples/WindowsStudio/WindowsStudioSample_WinUI/KsHelper.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Windows.Graphics.Imaging; using Windows.Media.Capture.Frames; @@ -33,7 +34,10 @@ public enum ExtendedControlKind : uint { KSPROPERTY_CAMERACONTROL_EXTENDED_EYEGAZECORRECTION = 40, KSPROPERTY_CAMERACONTROL_EXTENDED_BACKGROUNDSEGMENTATION = 41, - KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW = 43 + KSPROPERTY_CAMERACONTROL_EXTENDED_DIGITALWINDOW = 43, + KSPROPERTY_CAMERACONTROL_EXTENDED_FRAMERATE_THROTTLE = 44, + KSPROPERTY_CAMERACONTROL_EXTENDED_FIELDOFVIEW2_CONFIGCAPS = 45, + KSPROPERTY_CAMERACONTROL_EXTENDED_FIELDOFVIEW2 = 46 }; public enum BackgroundSegmentationCapabilityKind : uint @@ -50,6 +54,12 @@ public enum EyeGazeCorrectionCapabilityKind : uint KSCAMERA_EXTENDEDPROP_EYEGAZECORRECTION_ON = 1, KSCAMERA_EXTENDEDPROP_EYEGAZECORRECTION_STARE = 2, }; + + public enum FramerateThrottleCapabilityKind : uint + { + KSCAMERA_EXTENDEDPROP_FRAMERATE_THROTTLE_OFF = 0, + KSCAMERA_EXTENDEDPROP_FRAMERATE_THROTTLE_ON = 1 + }; // --> Windows Studio Effects custom KsProperties public static readonly Guid KSPROPERTYSETID_WindowsCameraEffect = @@ -62,7 +72,11 @@ public enum KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS : uint KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_STAGELIGHT = 1, KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_CREATIVEFILTER = 2, KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SETNOTIFICATION = 3, - KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4 + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_PERFORMANCEMITIGATION = 4, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA = 5, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_SENSORCENTERCROP = 6, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND = 7, + KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_UPDATED_VIEWPORT = 8 }; public enum StageLightCapabilityKind : uint @@ -78,6 +92,66 @@ public enum CreativeFilterCapabilityKind : uint KSCAMERA_WINDOWSSTUDIO_CREATIVEFILTER_ANIMATED = 2, KSCAMERA_WINDOWSSTUDIO_CREATIVEFILTER_WATERCOLOR = 4 }; + + public static readonly Guid MF_WINDOWSSTUDIO_METADATA_FACETRACKING = + Guid.Parse("F008811B-70C6-4A4F-9424-D576AC159905"); + + public static readonly Guid MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS = + Guid.Parse("1239D90E-6920-49A9-B72E-5F6D77944675"); + + public static readonly Guid MF_WINDOWSSTUDIO_METADATA_FACEPOSE = + Guid.Parse("397757AA-B630-43CE-AD47-AB3AC5FE4728"); + + public enum WindowsStudioFaceMetadataCapabilityKind : uint + { + KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_OFF = 0, + KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACETRACKING = 1, + KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACELANDMARKS = 2, + KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACEPOSE = 4 + }; + + public enum WindowsStudioAutomaticFramingKindCapabilityKind : uint + { + KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_WINDOW = 1, + KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_CINEMATIC = 2 + }; + + public struct MF_FLOAT2 + { + public float x; + public float y; + }; + + struct WSEFaceMetadataHeader + { + public uint Count; + public uint Size; + }; + + public struct WSEFaceTrackingMetadata + { + public MF_FLOAT2 TopLeft; // Top left corner of the bounding box of face in image relative coordinates [0, 1] + public MF_FLOAT2 BoxSize; // Width and Height of the bounding box of face in image relative coordinates [0, 1] + public float Confidence; // Confidence of this region being an actual face (0..1) + public uint TrackId; // Corresponding track id + }; + + public const int MAX_NUM_LANDMARKS = 70; + public unsafe struct WSEFaceLandmarksMetadata + { + // maximum number of landmarks + public fixed float Landmarks2D[MAX_NUM_LANDMARKS * 2]; // landmark location of faces in image relative coordinates [0, 1]. + public fixed float Confidence[MAX_NUM_LANDMARKS]; // individual confidence for each landmark (0..1) + public uint TrackId; // corresponding track id + }; + + const int EULER_ANGLE_COUNT = 3; // number of angles in pose + public unsafe struct WSEFacePoseMetadata + { + public fixed float Pose[EULER_ANGLE_COUNT]; // yaw, pitch, roll + public float Confidence; // overall confidence of pose detection results (0..1) + public uint TrackId; // corresponding track id + } // <-- Windows Studio Effects custom KsProperties [StructLayout(LayoutKind.Sequential)] @@ -194,6 +268,15 @@ public struct KSCAMERA_METADATA_BACKGROUNDSEGMENTATIONMASK //public byte[] MaskData; }; + [StructLayout(LayoutKind.Sequential)] + public unsafe struct KSCAMERA_EXTENDEDPROP_FIELDOFVIEW2_CONFIGCAPS + { + public short DefaultDiagonalFieldOfViewInDegrees; // The default FoV value for the driver/device + public short DiscreteFoVStopsCount; // Count of use FoV entries in DiscreteFoVStops array + public fixed short DiscreteFoVStops[360]; // Descending list of FoV Stops in degrees + public ushort Reserved; + } + // Lookup table to match a known camera profile ID with a legible name public const string LegacyProfileNameStr = "Legacy"; public static readonly Dictionary CameraProfileIdLUT = @@ -394,5 +477,115 @@ public static SoftwareBitmap ExtractBackgroundSegmentationMaskMetadataAsBitmap(M return maskSoftwareBitmap; } + + public static string ExtractFaceTrackMetadata(MediaFrameReference frame, out WSEFaceTrackingMetadata? metadataOut) + { + // face tracking + string result = ""; + metadataOut = null; + foreach (var s in frame.Properties) + { + result += $"{s.Key} : {s.Value}\n"; + } + if (frame.Properties.TryGetValue(MFSampleExtension_CaptureMetadata, out var captureMetadata)) + { + if (captureMetadata is IReadOnlyDictionary captureMetadataLookUp) + { + if (captureMetadataLookUp.TryGetValue(MF_WINDOWSSTUDIO_METADATA_FACETRACKING, out var metadataBlob)) + { + byte[] payloadBytes = (byte[])metadataBlob; + int byteOffset = Marshal.SizeOf(); + WSEFaceMetadataHeader payloadHeader = FromBytes(payloadBytes, byteOffset, 0); + int WSEFaceTrackingMetadataSize = Marshal.SizeOf(); + + result += $"ft({payloadHeader.Count}):"; + + for (int i = 0; i < payloadHeader.Count; i++) + { + metadataOut = FromBytes(payloadBytes, WSEFaceTrackingMetadataSize, byteOffset); + result += $"t:{metadataOut?.TrackId} {metadataOut?.TopLeft.x,4:0.00},{metadataOut?.TopLeft.y,4:0.00} -> w:{metadataOut?.BoxSize.x,4:0.00} h:{metadataOut?.BoxSize.y,4:0.00}"; + byteOffset += WSEFaceTrackingMetadataSize; + } + + result += $"\n"; + } + } + } + + return result; + } + public static string ExtractFaceLandmarksMetadata(MediaFrameReference frame, out WSEFaceLandmarksMetadata? metadataOut) + { + string result = ""; + metadataOut = null; + + // landmarks + if (frame.Properties.TryGetValue(MFSampleExtension_CaptureMetadata, out var captureMetadata)) + { + if (captureMetadata is IReadOnlyDictionary captureMetadataLookUp) + { + if (captureMetadataLookUp.TryGetValue(MF_WINDOWSSTUDIO_METADATA_FACELANDMARKS, out var metadataBlob)) + { + byte[] payloadBytes = (byte[])metadataBlob; + int byteOffset = Marshal.SizeOf(); + WSEFaceMetadataHeader payloadHeader = FromBytes(payloadBytes, byteOffset, 0); + int WSEFaceLandmarksMetadataSize = Marshal.SizeOf(); + + result += $"fl({payloadHeader.Count}):"; + + for (int i = 0; i < payloadHeader.Count; i++) + { + WSEFaceLandmarksMetadata faceMetadata = FromBytes(payloadBytes, WSEFaceLandmarksMetadataSize, byteOffset); + metadataOut = faceMetadata; + unsafe + { + result += $"t:{faceMetadata.TrackId} {faceMetadata.Landmarks2D[0],4:0.00},{faceMetadata.Landmarks2D[1],4:0.00}"; + } + byteOffset += WSEFaceLandmarksMetadataSize; + } + result += $"\n"; + } + } + } + + return result; + } + + public static string ExtractFacePoseMetadata(MediaFrameReference frame, out WSEFacePoseMetadata? metadataOut) + { + string result = ""; + metadataOut = null; + + // pose + if (frame.Properties.TryGetValue(MFSampleExtension_CaptureMetadata, out var captureMetadata)) + { + if (captureMetadata is IReadOnlyDictionary captureMetadataLookUp) + { + if (captureMetadataLookUp.TryGetValue(MF_WINDOWSSTUDIO_METADATA_FACEPOSE, out var metadataBlob)) + { + byte[] payloadBytes = (byte[])metadataBlob; + int byteOffset = Marshal.SizeOf(); + WSEFaceMetadataHeader payloadHeader = FromBytes(payloadBytes, byteOffset, 0); + int WSEFacePoseMetadataSize = Marshal.SizeOf(); + + result += $"fp({payloadHeader.Count}):"; + + for (int i = 0; i < payloadHeader.Count; i++) + { + WSEFacePoseMetadata faceMetadata = FromBytes(payloadBytes, WSEFacePoseMetadataSize, byteOffset); + metadataOut = faceMetadata; + unsafe + { + result += $"t:{faceMetadata.TrackId} {faceMetadata.Pose[0],4:0.00},{faceMetadata.Pose[1],4:0.00}, {faceMetadata.Pose[2],4:0.00}"; + } + byteOffset += WSEFacePoseMetadataSize; + } + result += $"\n"; + } + } + } + + return result; + } } } diff --git a/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml b/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml index 5c7dd0b..d9b17ef 100644 --- a/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml +++ b/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml @@ -6,114 +6,132 @@ xmlns:local="using:WindowsStudioSample_WinUI" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - mc:Ignorable="d"> + mc:Ignorable="d" + SizeChanged="Window_SizeChanged" + Closed="Window_Closed"> + + + - - - - - - - + - - - - - - + + - - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + - - + + + + - - - - - - - - - - + + + + + + + + + + + + - - + - + + + + diff --git a/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml.cs b/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml.cs index cedd45f..eaf2348 100644 --- a/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml.cs +++ b/Samples/WindowsStudio/WindowsStudioSample_WinUI/MainWindow.xaml.cs @@ -1,26 +1,30 @@ // Copyright (c) Microsoft. All rights reserved. +using ControlMonitorHelperWinRT; +using Microsoft.UI.Dispatching; +using Microsoft.UI.Windowing; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Media; +using Microsoft.UI.Xaml.Media.Imaging; +using Microsoft.UI.Xaml.Shapes; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; -using Microsoft.UI.Xaml; -using Microsoft.UI.Xaml.Controls; using System.Threading; -using Windows.Media.Capture.Frames; -using Windows.Media.Capture; -using Windows.Media.Playback; -using Microsoft.UI.Dispatching; using System.Threading.Tasks; -using static WindowsStudioSample_WinUI.KsHelper; using Windows.Devices.Enumeration; +using Windows.Foundation; +using Windows.Graphics.Imaging; +using Windows.Media.Capture; +using Windows.Media.Capture.Frames; +using Windows.Media.Core; using Windows.Media.Devices; using Windows.Media.MediaProperties; -using Windows.Media.Core; -using ControlMonitorHelperWinRT; -using Windows.Graphics.Imaging; -using Microsoft.UI.Xaml.Media.Imaging; +using Windows.Media.Playback; +using static WindowsStudioSample_WinUI.KsHelper; namespace WindowsStudioSample_WinUI; @@ -46,6 +50,17 @@ public sealed partial class MainWindow : Window private SoftwareBitmapSource m_backgroundMaskImageSource = new SoftwareBitmapSource(); private bool m_isShowingMask = false; + // canvas + private SemaphoreSlim m_canvasRefreshLock = new SemaphoreSlim(1); + private double m_canvasOffsetX = 0, m_canvasOffsetY = 0; + private Rectangle m_trackingRectUI = new Rectangle() { StrokeThickness = 3, Stroke = new SolidColorBrush(Microsoft.UI.Colors.Blue) }; + private Ellipse[] m_landmarksPointsUI = null; + private Line[] m_poseLinesUI = new Line[2] + { + new Line() { Stroke = new SolidColorBrush(Microsoft.UI.Colors.Green), StrokeThickness = 8, X1=-50, X2=50 }, + new Line() { Stroke = new SolidColorBrush(Microsoft.UI.Colors.Pink), StrokeThickness = 8, Y1=-50, Y2=50 }, + }; + // this DevPropKey signifies if the system is capable of exposing a Windows Studio camera private const string DEVPKEY_DeviceInterface_IsWindowsCameraEffectAvailable = "{6EDC630D-C2E3-43B7-B2D1-20525A1AF120} 4"; @@ -57,6 +72,7 @@ private enum AppState } private AppState m_appState = AppState.Unitialized; + private Point m_pointerPosition; // Lookup table for Background Segmentation DDI flag values and UI names readonly Dictionary m_possibleBackgroundSegmentationFlagValues = new() @@ -130,12 +146,40 @@ private enum AppState } }; + // Lookup table for Automatic Framing Kind DDI flag values and UI names + readonly Dictionary m_possibleAutomaticFramingKindFlagValues = new() + { + { + "Cinematic", + (ulong)WindowsStudioAutomaticFramingKindCapabilityKind.KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_CINEMATIC + }, + { + "Standard", + (ulong)WindowsStudioAutomaticFramingKindCapabilityKind.KSCAMERA_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND_WINDOW + } + }; + public MainWindow() { this.InitializeComponent(); - this.ExtendsContentIntoTitleBar = true;; + this.ExtendsContentIntoTitleBar = true; this.SetTitleBar(UITitle); - + + this.AppWindow.Changed += AppWindow_Changed; + m_landmarksPointsUI = new Ellipse[70]; + for (int i = 0; i < MAX_NUM_LANDMARKS; i++) + { + m_landmarksPointsUI[i] = new Ellipse() + { + StrokeThickness = 3, + Stroke = new SolidColorBrush(i < 68 ? Microsoft.UI.Colors.Red : Microsoft.UI.Colors.Yellow), + Fill = new SolidColorBrush(Microsoft.UI.Colors.Transparent), + Width = 5, + Height = 5, + Visibility = Visibility.Collapsed + }; + } + m_uiDispatcherQueue = DispatcherQueue.GetForCurrentThread(); var fireAndForget = InitializeCameraAndUI(); @@ -144,7 +188,7 @@ public MainWindow() /// /// Uninitialize the camera /// - private void UninitializeCamera() + private async Task UninitializeCamera() { // disable UI @@ -171,13 +215,14 @@ private void UninitializeCamera() } m_frameReadingLock.Wait(); - m_backgroundSegmentationImageRefreshLock.Wait(); try { if (m_mediaFrameReader != null) { m_mediaFrameReader.FrameArrived -= MediaFrameReader_FrameArrived; - m_mediaFrameReader.StopAsync().Wait(); + m_backgroundSegmentationImageRefreshLock.Wait(); + m_canvasRefreshLock.Wait(); + await m_mediaFrameReader.StopAsync(); m_mediaFrameReader = null; } @@ -191,6 +236,7 @@ private void UninitializeCamera() } finally { + m_canvasRefreshLock.Release(); m_backgroundSegmentationImageRefreshLock.Release(); m_frameReadingLock.Release(); } @@ -213,6 +259,16 @@ private async Task InitializeCameraAndUI() { m_initLock.Wait(); + UIPreviewCanvas.Children.Clear(); + + UIPreviewCanvas.Children.Add(m_trackingRectUI); + UIPreviewCanvas.Children.Add(m_poseLinesUI[0]); + UIPreviewCanvas.Children.Add(m_poseLinesUI[1]); + for (int i = 0; i < m_landmarksPointsUI.Count(); i++) + { + UIPreviewCanvas.Children.Add(m_landmarksPointsUI[i]); + } + // Attempt to find a Windows Studio camera.. if (m_cameraDeviceInfo == null) { @@ -233,7 +289,7 @@ private async Task InitializeCameraAndUI() // reinterpret the byte array as an extended property payload KSCAMERA_EXTENDEDPROP_HEADER payloadHeader = FromBytes(byteResultPayload); int sizeofHeader = Marshal.SizeOf(); - int sizeofKsProperty = Marshal.SizeOf(); ; + int sizeofKsProperty = Marshal.SizeOf(); int supportedControls = ((int)payloadHeader.Size - sizeofHeader) / sizeofKsProperty; for (int i = 0; i < supportedControls; i++) @@ -250,6 +306,14 @@ private async Task InitializeCameraAndUI() { RefreshCreativeFilterUI(); } + else if (payloadKsProperty.Id == (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA) + { + RefreshFaceMetadataUI(); + } + else if (payloadKsProperty.Id == (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND) + { + RefreshAutomaticFramingKindUI(); + } } } } @@ -291,7 +355,7 @@ private async Task InitializeCameraAndUI() catch (Exception ex) { NotifyUser(ex.Message, NotifyType.ErrorMessage); - UninitializeCamera(); + await UninitializeCamera(); m_appState = AppState.Error; } finally @@ -507,6 +571,69 @@ private void MediaFrameReader_FrameArrived(MediaFrameReader sender, MediaFrameAr m_backgroundSegmentationImageRefreshLock.Release(); }); } + + // Attempt to extract the face metadata + string faceTrackMetadataOutput = ExtractFaceTrackMetadata(frame, out var faceTrackMetadataOut); + string faceLandmarksMetadataOutput = ExtractFaceLandmarksMetadata(frame, out var faceLandmarksMetadataOut); + string facePoseMetadataOutput = ExtractFacePoseMetadata(frame, out var facePoseMetadataOut); + + if (faceTrackMetadataOut != null || faceLandmarksMetadataOut != null || facePoseMetadataOut != null) + { + m_uiDispatcherQueue.TryEnqueue(() => + { + m_canvasRefreshLock.Wait(0); + try + { + UIMetadataText.Text = faceTrackMetadataOutput + faceLandmarksMetadataOutput + facePoseMetadataOutput; + if (faceTrackMetadataOut != null) + { + m_trackingRectUI.Width = (double)(faceTrackMetadataOut?.BoxSize.x * (UIPreviewElement.ActualWidth - 2 * m_canvasOffsetX)); + m_trackingRectUI.Height = (double)(faceTrackMetadataOut?.BoxSize.y * (UIPreviewCanvas.ActualHeight - 2 * m_canvasOffsetY)); + Canvas.SetLeft(m_trackingRectUI, (double)(faceTrackMetadataOut?.TopLeft.x * (UIPreviewCanvas.ActualWidth - 2 * m_canvasOffsetX) + m_canvasOffsetX)); + Canvas.SetTop(m_trackingRectUI, (double)(faceTrackMetadataOut?.TopLeft.y * (UIPreviewCanvas.ActualHeight - 2 * m_canvasOffsetY) + m_canvasOffsetY)); + } + if (faceLandmarksMetadataOut != null) + { + WSEFaceLandmarksMetadata landmarkMetadata = (WSEFaceLandmarksMetadata)faceLandmarksMetadataOut; + for (int i = 0; i < 70; i++) unsafe + { + + Canvas.SetLeft(m_landmarksPointsUI[i], (double)(landmarkMetadata.Landmarks2D[2 * i] * (UIPreviewElement.ActualWidth - 2 * m_canvasOffsetX) + m_canvasOffsetX)); + Canvas.SetTop(m_landmarksPointsUI[i], (double)(landmarkMetadata.Landmarks2D[2 * i + 1] * (UIPreviewElement.ActualHeight - 2 * m_canvasOffsetY) + m_canvasOffsetY)); + } + } + if (facePoseMetadataOut != null) + { + WSEFacePoseMetadata poseMetadata = (WSEFacePoseMetadata)facePoseMetadataOut; + for (int i = 0; i < 2; i++) unsafe + { + m_poseLinesUI[i].Projection = new PlaneProjection() + { + RotationX = (double)poseMetadata.Pose[0], + RotationY = -(double)poseMetadata.Pose[1], + RotationZ = -(double)poseMetadata.Pose[2] + }; + if (faceTrackMetadataOut == null) + { + Canvas.SetLeft(m_poseLinesUI[i], UIPreviewCanvas.ActualWidth / 2); + Canvas.SetTop(m_poseLinesUI[i], UIPreviewCanvas.ActualHeight / 2); + } + else + { + Canvas.SetLeft(m_poseLinesUI[i], (double)(faceTrackMetadataOut?.TopLeft.x * (UIPreviewCanvas.ActualWidth - 2 * m_canvasOffsetX) + m_canvasOffsetX)); + Canvas.SetTop(m_poseLinesUI[i], (double)(faceTrackMetadataOut?.TopLeft.y * (UIPreviewCanvas.ActualHeight - 2 * m_canvasOffsetY) + m_canvasOffsetY)); + } + } + } + } + finally + { + m_canvasRefreshLock.Release(); + } + + }); + } + m_frameReadingLock.Release(); } } @@ -521,7 +648,7 @@ private void MediaCapture_Failed(MediaCapture sender, MediaCaptureFailedEventArg { m_initLock.Wait(); NotifyUser($"MediaCapture_Failed: {errorEventArgs.Message}", NotifyType.ErrorMessage); - UninitializeCamera(); + var t = UninitializeCamera(); m_appState = AppState.Error; m_initLock.Release(); } @@ -569,6 +696,14 @@ private void CameraControlMonitor_ControlChanged(object sender, ControlData cont m_uiDispatcherQueue.TryEnqueue(DispatcherQueuePriority.Normal, () => RefreshCreativeFilterUI()); break; + case (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA: + m_uiDispatcherQueue.TryEnqueue(DispatcherQueuePriority.Normal, () => RefreshFaceMetadataUI()); + break; + + case (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND: + m_uiDispatcherQueue.TryEnqueue(DispatcherQueuePriority.Normal, () => RefreshAutomaticFramingKindUI()); + break; + default: throw new Exception("unhandled Windows Studio Effects control change, implement or allow through at your convenience"); } @@ -609,7 +744,7 @@ private void RefreshEyeGazeUI() /// private void RefreshBackgroundSegmentationUI() { - // send a GET call for the eye gaze DDI and retrieve the result payload + // send a GET call for the background segmentation DDI and retrieve the result payload byte[] byteResultPayload = GetExtendedControlPayload( m_mediaCapture.VideoDeviceController, KSPROPERTYSETID_ExtendedCameraControl, @@ -701,6 +836,66 @@ private void RefreshCreativeFilterUI() UICreativeFilterModes.SelectionChanged += UICreativeFilterModes_SelectionChanged; } + private void RefreshAutomaticFramingKindUI() + { + // send a GET call for the AutomaticFramingKind DDI and retrieve the result payload + byte[] byteResultPayload = GetExtendedControlPayload( + m_mediaCapture.VideoDeviceController, + KSPROPERTYSETID_WindowsCameraEffect, + (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND); + + // reinterpret the byte array as an extended property payload + KsBasicCameraExtendedPropPayload payload = FromBytes(byteResultPayload); + + // refresh the list of values displayed for this control + UIAutomaticFramingKind.SelectionChanged -= UIAutomaticFramingKind_SelectionChanged; + + var automaticFramingKindCapabilities = m_possibleAutomaticFramingKindFlagValues.Where(x => (x.Value & payload.header.Capability) == x.Value).ToList(); + UIAutomaticFramingKind.ItemsSource = automaticFramingKindCapabilities.Select(x => x.Key); + + // reflect in UI what is the current value + var currentFlag = payload.header.Flags; + var currentIndexToSelect = automaticFramingKindCapabilities.FindIndex(x => x.Value == currentFlag); + UIAutomaticFramingKind.SelectedIndex = currentIndexToSelect; + + UIAutomaticFramingKind.IsEnabled = true; + UIAutomaticFramingKind.SelectionChanged += UIAutomaticFramingKind_SelectionChanged; + } + + private void RefreshFaceMetadataUI() + { + // send a GET call for the FaceMetadata DDI and retrieve the result payload + byte[] byteResultPayload = GetExtendedControlPayload( + m_mediaCapture.VideoDeviceController, + KSPROPERTYSETID_WindowsCameraEffect, + (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA); + + // reinterpret the byte array as an extended property payload + KsBasicCameraExtendedPropPayload payload = FromBytes(byteResultPayload); + + // refresh the list of values displayed for this control + UIFaceTracking.Checked -= UIFaceTracking_Checked; + UIFaceTracking.Unchecked -= UIFaceTracking_Checked; + UIFaceLandmarks.Checked -= UIFaceLandmarks_Checked; + UIFaceLandmarks.Unchecked -= UIFaceLandmarks_Checked; + UIFacePose.Checked -= UIFacePose_Checked; + UIFacePose.Unchecked -= UIFacePose_Checked; + + UIFaceTracking.IsEnabled = ((uint)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACETRACKING & (payload.header.Capability)) > 0; + UIFaceTracking.IsChecked = ((uint)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACETRACKING & (payload.header.Flags)) > 0; + UIFaceLandmarks.IsEnabled = ((uint)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACELANDMARKS & (payload.header.Capability)) > 0; + UIFaceLandmarks.IsChecked = ((uint)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACELANDMARKS & (payload.header.Flags)) > 0; + UIFacePose.IsEnabled = ((uint)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACEPOSE & (payload.header.Capability)) > 0; + UIFacePose.IsChecked = ((uint)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACEPOSE & (payload.header.Flags)) > 0; + + UIFaceTracking.Checked += UIFaceTracking_Checked; + UIFaceTracking.Unchecked += UIFaceTracking_Checked; + UIFaceLandmarks.Checked += UIFaceLandmarks_Checked; + UIFaceLandmarks.Unchecked += UIFaceLandmarks_Checked; + UIFacePose.Checked += UIFacePose_Checked; + UIFacePose.Unchecked += UIFacePose_Checked; + } + #region UICallbacks private void UIEyeGazeCorrectionModes_SelectionChanged(object sender, SelectionChangedEventArgs e) { @@ -782,6 +977,26 @@ private void UICreativeFilterModes_SelectionChanged(object sender, SelectionChan flagToSet); } + private void UIAutomaticFramingKind_SelectionChanged(object sender, SelectionChangedEventArgs e) + { + var selectedIndex = UIAutomaticFramingKind.SelectedIndex; + if (selectedIndex < 0) + { + return; + } + + // find the flags value associated with the current mode selected + string selection = UIAutomaticFramingKind.SelectedItem.ToString(); + ulong flagToSet = m_possibleAutomaticFramingKindFlagValues.FirstOrDefault(x => x.Key == selection).Value; + + // set the flags value for the corresponding extended control + SetExtendedControlFlags( + m_mediaCapture.VideoDeviceController, + KSPROPERTYSETID_WindowsCameraEffect, + (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_AUTOMATICFRAMINGKIND, + flagToSet); + } + private void UILaunchSettingsPage_Click(object sender, RoutedEventArgs e) { // launch Windows Settings page for the camera identified by the specified Id @@ -815,6 +1030,102 @@ private void UIProfilesAvailable_SelectionChanged(object sender, SelectionChange } } + private void UIFaceTracking_Checked(object sender, RoutedEventArgs e) + { + m_trackingRectUI.Visibility = UIFaceTracking.IsChecked == true ? Visibility.Visible : Visibility.Collapsed; + UIFaceMetadataSwitch_Toggled(null, null); + } + + private void UIFaceLandmarks_Checked(object sender, RoutedEventArgs e) + { + for (int i = 0; i < 70; i++) + { + m_landmarksPointsUI[i].Visibility = UIFaceLandmarks.IsChecked == true ? Visibility.Visible : Visibility.Collapsed; + } + UIFaceMetadataSwitch_Toggled(null, null); + } + + private void UIFacePose_Checked(object sender, RoutedEventArgs e) + { + for (int i = 0; i < 2; i++) + { + m_poseLinesUI[i].Visibility = UIFacePose.IsChecked == true ? Visibility.Visible : Visibility.Collapsed; + } + UIFaceMetadataSwitch_Toggled(null, null); + } + + private void UIFaceMetadataSwitch_Toggled(object sender, RoutedEventArgs e) + { + // find the flags value associated with the current toggle value selected + ulong flagToSet = + ((UIFaceTracking.IsChecked == true ? (ulong)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACETRACKING : (ulong)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_OFF) + | (UIFaceLandmarks.IsChecked == true ? (ulong)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACELANDMARKS : (ulong)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_OFF) + | (UIFacePose.IsChecked == true ? (ulong)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_FACEPOSE : (ulong)WindowsStudioFaceMetadataCapabilityKind.KSCAMERA_WINDOWSSTUDIO_FACEMETADATA_OFF)); + + // set the flags value for the corresponding extended control + SetExtendedControlFlags( + m_mediaCapture.VideoDeviceController, + KSPROPERTYSETID_WindowsCameraEffect, + (uint)KSPROPERTY_CAMERACONTROL_WINDOWS_EFFECTS.KSPROPERTY_CAMERACONTROL_WINDOWSSTUDIO_FACEMETADATA, + flagToSet); + + UIPreviewCanvas.Width = UIPreviewElement.ActualWidth; + UIPreviewCanvas.Height = UIPreviewElement.ActualHeight; + + if (((float)m_selectedFormat.VideoFormat.Width / m_selectedFormat.VideoFormat.Height) > (float)(UIPreviewElement.ActualWidth / UIPreviewElement.ActualHeight)) + { + m_canvasOffsetY = (UIPreviewElement.ActualHeight - UIPreviewElement.ActualWidth * m_selectedFormat.VideoFormat.Height / m_selectedFormat.VideoFormat.Width) / 2; + } + else + { + m_canvasOffsetX = (UIPreviewElement.ActualWidth - UIPreviewElement.ActualHeight * m_selectedFormat.VideoFormat.Width / m_selectedFormat.VideoFormat.Height) / 2; + } + + UIMetadataText.Visibility = flagToSet > 0 ? Visibility.Visible : Visibility.Collapsed; + UIPreviewCanvas.Visibility = flagToSet > 0 ? Visibility.Visible : Visibility.Collapsed; + } + + private async void Window_Closed(object sender, WindowEventArgs args) + { + await UninitializeCamera(); + } + + private void UIPreviewCanvas_PointerMoved(object sender, Microsoft.UI.Xaml.Input.PointerRoutedEventArgs e) + { + m_pointerPosition = e.GetCurrentPoint(UIPreviewCanvas).Position; + } + + private void AppWindow_Changed(AppWindow sender, AppWindowChangedEventArgs args) + { + var presenter = sender.Presenter as OverlappedPresenter; + if (presenter != null) + { + if (presenter.State == OverlappedPresenterState.Restored || presenter.State == OverlappedPresenterState.Maximized) + { + Window_SizeChanged(null, null); + } + } + } + + private void Window_SizeChanged(object sender, WindowSizeChangedEventArgs args) + { + if (m_selectedFormat == null) + { + return; + } + if (((float)m_selectedFormat.VideoFormat.Width / m_selectedFormat.VideoFormat.Height) > (float)(UIPreviewElement.ActualWidth / UIPreviewElement.ActualHeight)) + { + m_canvasOffsetY = (UIPreviewElement.ActualHeight - UIPreviewElement.ActualWidth * m_selectedFormat.VideoFormat.Height / m_selectedFormat.VideoFormat.Width) / 2; + } + else + { + m_canvasOffsetX = (UIPreviewElement.ActualWidth - UIPreviewElement.ActualHeight * m_selectedFormat.VideoFormat.Width / m_selectedFormat.VideoFormat.Height) / 2; + } + + UIPreviewCanvas.Width = UIPreviewElement.ActualWidth; + UIPreviewCanvas.Height = UIPreviewElement.ActualHeight; + } + #endregion UICallbacks @@ -861,7 +1172,7 @@ private void UpdateStatus(string strMessage, NotifyType type) UIStatusBar.Severity = InfoBarSeverity.Error; break; } - UIStatusBar.Message= strMessage; + UIStatusBar.Message = strMessage; Debug.WriteLine($"{type}: {strMessage}"); // Collapse the StatusBlock if it has no text to conserve UI real estate. diff --git a/Samples/WindowsStudio/WindowsStudioSample_WinUI/WindowsStudioSample_WinUI.csproj b/Samples/WindowsStudio/WindowsStudioSample_WinUI/WindowsStudioSample_WinUI.csproj index 3e1a8bf..ae098ac 100644 --- a/Samples/WindowsStudio/WindowsStudioSample_WinUI/WindowsStudioSample_WinUI.csproj +++ b/Samples/WindowsStudio/WindowsStudioSample_WinUI/WindowsStudioSample_WinUI.csproj @@ -1,15 +1,15 @@  WinExe - net8.0-windows10.0.22621.0 + net8.0-windows10.0.26100.0 10.0.17763.0 WindowsStudioSample_WinUI app.manifest x86;x64;ARM64 win-x86;win-x64;win-arm64 - win10-$(Platform).pubxml true true + Properties\PublishProfiles\win10-$(Platform).pubxml 10.0.22621.0 WindowsStudioSample_WinUI_TemporaryKey.pfx True @@ -33,9 +33,9 @@ - - - + + +