diff --git a/karma.conf.cjs b/karma.conf.cjs index 83471d4b..1e21cc21 100644 --- a/karma.conf.cjs +++ b/karma.conf.cjs @@ -7,8 +7,8 @@ module.exports = function(config) { ], - proxies: { - '/volume-meter-processor.js': '/base/src/main/js/volume-meter-processor.js' + proxies: {'/volume-meter-processor.js': '/base/src/main/js/volume-meter-processor.js', + '/draw-desktop-with-camera-source-worker.js': '/base/src/main/js/draw-desktop-with-camera-source-worker.js' }, reporters: ['progress', 'coverage'], diff --git a/rollup.config.module.cjs b/rollup.config.module.cjs index 96110b84..6ea9e583 100644 --- a/rollup.config.module.cjs +++ b/rollup.config.module.cjs @@ -17,6 +17,7 @@ const builds = { 'src/main/js/utility.js', 'src/main/js/media_manager.js', 'src/main/js/stream_merger.js', + 'src/main/js/draw-desktop-with-camera-source-worker.js', ], output: [{ dir: 'dist', diff --git a/src/main/js/draw-desktop-with-camera-source-worker.js b/src/main/js/draw-desktop-with-camera-source-worker.js new file mode 100644 index 00000000..b0c86753 --- /dev/null +++ b/src/main/js/draw-desktop-with-camera-source-worker.js @@ -0,0 +1,67 @@ +let canvas, ctx; +let canvasWidth = 1920; +let canvasHeight = 1080; +let overlayWidth, overlayHeight; + +// Handle messages from the main thread +self.onmessage = function (event) { + const { type, data } = event.data; + + switch (type) { + case 'init': + initializeCanvas(data.canvas, data.width, data.height); + break; + case 'frame': + drawFrame(data.screenFrame, data.cameraFrame); + break; + case 'resize': + resizeCanvas(data.width, data.height); + break; + default: + console.warn('Unknown message type:', type); + } +}; + +function initializeCanvas(offscreenCanvas, width, height) { + canvas = offscreenCanvas; + ctx = canvas.getContext('2d'); + canvasWidth = width; + canvasHeight = height; + + canvas.width = canvasWidth; + canvas.height = canvasHeight; + + overlayWidth = canvasWidth / 4; + console.log('Canvas initialized in worker.'); +} + +function resizeCanvas(width, height) { + canvasWidth = width; + canvasHeight = height; + canvas.width = canvasWidth; + canvas.height = canvasHeight; + + overlayWidth = canvasWidth / 4; + console.log('Canvas resized in worker:', canvasWidth, canvasHeight); +} + +function drawFrame(screenFrame, cameraFrame) { + if (!ctx) return; + + // Clear the canvas + ctx.clearRect(0, 0, canvasWidth, canvasHeight); + + // Draw the screen video frame + ctx.drawImage(screenFrame, 0, 0, canvasWidth, canvasHeight); + + // Draw the camera overlay + const overlayHeight = overlayWidth * (cameraFrame.height / cameraFrame.width); + const positionX = canvasWidth - overlayWidth - 20; + const positionY = canvasHeight - overlayHeight - 20; + + ctx.drawImage(cameraFrame, positionX, positionY, overlayWidth, overlayHeight); + + // Release ImageBitmap resources + screenFrame.close(); + cameraFrame.close(); +} diff --git a/src/main/js/media_manager.js b/src/main/js/media_manager.js index ff6ead3f..8e00ddc9 100644 --- a/src/main/js/media_manager.js +++ b/src/main/js/media_manager.js @@ -354,63 +354,89 @@ export class MediaManager { */ setDesktopwithCameraSource(stream, streamId, onEndedCallback) { this.desktopStream = stream; - return this.navigatorUserMedia({video: true, audio: false}, cameraStream => { - this.smallVideoTrack = cameraStream.getVideoTracks()[0]; - //create a canvas element - var canvas = document.createElement("canvas"); - var canvasContext = canvas.getContext("2d"); + return this.navigatorUserMedia({ video: true, audio: false }, cameraStream => { + this.smallVideoTrack = cameraStream.getVideoTracks()[0]; - //create video element for screen - //var screenVideo = document.getElementById('sourceVideo'); - var screenVideo = document.createElement('video'); + // Create OffscreenCanvas + const canvas = document.createElement("canvas"); + const offscreenCanvas = canvas.transferControlToOffscreen(); + // Create video elements for streams + const screenVideo = document.createElement("video"); screenVideo.srcObject = stream; - screenVideo.play(); - //create video element for camera - var cameraVideo = document.createElement('video'); + const cameraVideo = document.createElement("video"); cameraVideo.srcObject = cameraStream; - cameraVideo.play(); - var canvasStream = canvas.captureStream(15); - if (onEndedCallback != null) { - stream.getVideoTracks()[0].onended = function (event) { + // Start playing videos only after metadata is loaded + const setupVideo = video => + new Promise(resolve => { + video.onloadedmetadata = () => { + video.play(); + resolve(video); + }; + }); + + // Handle stream end callback + if (onEndedCallback) { + stream.getVideoTracks()[0].onended = event => { onEndedCallback(event); - } - } - var promise; - if (this.localStream == null) { - promise = this.gotStream(canvasStream); - } else { - promise = this.updateVideoTrack(canvasStream, streamId, onended, null); + }; } - promise.then(() => { + // Create a MediaStream from the canvas + const canvasStream = canvas.captureStream(15); // Capture at 15 fps - //update the canvas - this.desktopCameraCanvasDrawerTimer = setInterval(() => { - //draw screen to canvas - canvas.width = screenVideo.videoWidth; - canvas.height = screenVideo.videoHeight; - canvasContext.drawImage(screenVideo, 0, 0, canvas.width, canvas.height); + // Initialize or update the local stream + const promise = this.localStream == null + ? this.gotStream(canvasStream) + : this.updateVideoTrack(canvasStream, streamId, onEndedCallback, null); - var cameraWidth = screenVideo.videoWidth * (this.camera_percent / 100); - var cameraHeight = (cameraVideo.videoHeight / cameraVideo.videoWidth) * cameraWidth + promise.then(() => { + // Initialize the worker + const worker = new Worker(new URL("./draw-desktop-with-camera-source-worker.js", import.meta.url)); + + // Send the OffscreenCanvas to the worker + worker.postMessage({ + type: 'init', + data: { canvas: offscreenCanvas, width: screenVideo.videoWidth, height: screenVideo.videoHeight }, + }, [offscreenCanvas]); + + // Wait for both videos to load + Promise.all([setupVideo(screenVideo), setupVideo(cameraVideo)]).then(() => { + const frameInterval = 1000 / 15; // 15 fps + + // Periodically send frames to the worker + const sendFrames = () => { + if (screenVideo.videoWidth > 0 && cameraVideo.videoWidth > 0) { + createImageBitmap(screenVideo).then(screenBitmap => { + createImageBitmap(cameraVideo).then(cameraBitmap => { + worker.postMessage({ + type: 'frame', + data: { screenFrame: screenBitmap, cameraFrame: cameraBitmap }, + }, [screenBitmap, cameraBitmap]); + }).catch(err => { + console.error("Error creating camera ImageBitmap:", err); + }); + }).catch(err => { + console.error("Error creating screen ImageBitmap:", err); + }); + } else { + console.warn("Video dimensions are invalid."); + } + }; - var positionX = (canvas.width - cameraWidth) - this.camera_margin; - var positionY; + const frameTimer = setInterval(sendFrames, frameInterval); - if (this.camera_location == "top") { - positionY = this.camera_margin; - } else { //if not top, make it bottom - //draw camera on right bottom corner - positionY = (canvas.height - cameraHeight) - this.camera_margin; - } - canvasContext.drawImage(cameraVideo, positionX, positionY, cameraWidth, cameraHeight); - }, 66); + // Cleanup + this.desktopCameraCanvasDrawerTimer = () => { + clearInterval(frameTimer); + worker.terminate(); + }; + }); }); - }, true) + }, true); } /** diff --git a/src/test/js/media_manager.test.js b/src/test/js/media_manager.test.js index 13f2d65e..44d82100 100644 --- a/src/test/js/media_manager.test.js +++ b/src/test/js/media_manager.test.js @@ -316,4 +316,415 @@ describe("MediaManager", function () { }); + describe("setDesktopwithCameraSource", function () { + + beforeEach(() => { + window.OffscreenCanvas = class { + constructor() {} + getContext() { + return { + drawImage: sinon.fake(), + clearRect: sinon.fake(), + }; + } + captureStream(fps) { + return new MediaStream(); // Return a mock MediaStream + } + + }; + }); + + it("should set desktop stream and small video track correctly", async function () { + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://localhost", + mediaConstraints: { + video: "dummy", + audio: "dummy" + }, + initializeComponents: false + }); + + await adaptor.initialize(); + + // Create a mock video track + const mockVideoTrack = { + kind: "video", + enabled: true, + stop: sinon.fake(), // Mock the `stop` method + addEventListener: sinon.fake(), + removeEventListener: sinon.fake(), + onended: sinon.fake(), + }; + + // Create a fake `MediaStream` and add the mock video track + const cameraStream = new MediaStream(); + cameraStream.getVideoTracks = () => [mockVideoTrack]; // Override `getVideoTracks` + + // Stub `getUserMedia` to return the mocked camera stream + sinon.stub(navigator.mediaDevices, "getUserMedia").resolves(cameraStream); + + // Create a desktop stream + const stream = new MediaStream(); + + // Mock the onEnded callback + const onEndedCallback = null; + + // Call the function under test + await adaptor.mediaManager.setDesktopwithCameraSource(stream, "streamId", onEndedCallback); + + // Assertions + expect(adaptor.mediaManager.desktopStream).to.equal(stream); // Ensure the desktop stream is set + expect(adaptor.mediaManager.smallVideoTrack).to.equal(mockVideoTrack); // Ensure the video track is set + + // Restore the stub + navigator.mediaDevices.getUserMedia.restore(); + }); + + it("should call onEndedCallback when desktop stream ends", async function () { + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://localhost", + mediaConstraints: { + video: "dummy", + audio: "dummy" + }, + initializeComponents: false + }); + + var mediaManager = new MediaManager({ + userParameters: { + mediaConstraints: { + video: false, + audio: true, + } + }, + webRTCAdaptor: adaptor, + + callback: (info, obj) => { + adaptor.notifyEventListeners(info, obj) + }, + callbackError: (error, message) => { + adaptor.notifyErrorEventListeners(error, message) + }, + getSender: (streamId, type) => { + return adaptor.getSender(streamId, type) + }, + }); + + adaptor.mediaManager = mediaManager; + + await adaptor.initialize(); + + // Create a mock video track + const mockVideoTrack = { + kind: "video", + enabled: true, + stop: sinon.fake(), // Mock the `stop` method + addEventListener: sinon.fake(), + removeEventListener: sinon.fake(), + onended: sinon.fake(), + }; + + // Create a desktop stream + const stream = new MediaStream(); + stream.getVideoTracks = () => [mockVideoTrack]; // Override `getVideoTracks` + + // Create a fake `MediaStream` and add the mock video track + const cameraStream = new MediaStream(); + cameraStream.getVideoTracks = () => [mockVideoTrack]; // Override `getVideoTracks` + + sinon.stub(navigator.mediaDevices, 'getUserMedia').resolves(cameraStream); + const onEndedCallback = sinon.fake(); + + await mediaManager.setDesktopwithCameraSource(stream, "streamId", onEndedCallback); + stream.getVideoTracks()[0].onended(); + + expect(onEndedCallback.calledOnce).to.be.true; + navigator.mediaDevices.getUserMedia.restore(); + }); + + it("should update offscreen canvas at regular intervals", async function () { + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://localhost", + mediaConstraints: { + video: "dummy", + audio: "dummy" + }, + initializeComponents: false + }); + + var mediaManager = new MediaManager({ + userParameters: { + mediaConstraints: { + video: false, + audio: true, + } + }, + webRTCAdaptor: adaptor, + + callback: (info, obj) => { + adaptor.notifyEventListeners(info, obj) + }, + callbackError: (error, message) => { + adaptor.notifyErrorEventListeners(error, message) + }, + getSender: (streamId, type) => { + return adaptor.getSender(streamId, type) + }, + }); + + adaptor.mediaManager = mediaManager; + + await adaptor.initialize(); + + // Create a mock video track + const mockVideoTrack = { + kind: "video", + enabled: true, + stop: sinon.fake(), // Mock the `stop` method + addEventListener: sinon.fake(), + removeEventListener: sinon.fake(), + onended: sinon.fake(), + }; + + // Create a desktop stream + const stream = new MediaStream(); + stream.getVideoTracks = () => [mockVideoTrack]; // Override `getVideoTracks` + + // Create a fake `MediaStream` and add the mock video track + const cameraStream = new MediaStream(); + cameraStream.getVideoTracks = () => [mockVideoTrack]; // Override `getVideoTracks` + + sinon.stub(navigator.mediaDevices, 'getUserMedia').resolves(cameraStream); + const onEndedCallback = sinon.fake(); + + await mediaManager.setDesktopwithCameraSource(stream, "streamId", onEndedCallback); + + const initialWidth = mediaManager.dummyCanvas.width; + const initialHeight = mediaManager.dummyCanvas.height; + + await new Promise(resolve => setTimeout(resolve, 100)); + + expect(mediaManager.dummyCanvas.width).to.equal(initialWidth); + expect(mediaManager.dummyCanvas.height).to.equal(initialHeight); + navigator.mediaDevices.getUserMedia.restore(); + }); + + it("should handle null onEndedCallback gracefully", async function () { + + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://localhost", + mediaConstraints: { + video: "dummy", + audio: "dummy" + }, + initializeComponents: false + }); + + var mediaManager = new MediaManager({ + userParameters: { + mediaConstraints: { + video: false, + audio: true, + } + }, + webRTCAdaptor: adaptor, + + callback: (info, obj) => { + adaptor.notifyEventListeners(info, obj) + }, + callbackError: (error, message) => { + adaptor.notifyErrorEventListeners(error, message) + }, + getSender: (streamId, type) => { + return adaptor.getSender(streamId, type) + }, + }); + + adaptor.mediaManager = mediaManager; + + await adaptor.initialize(); + + // Create a mock video track + const mockVideoTrack = { + kind: "video", + enabled: true, + stop: sinon.fake(), // Mock the `stop` method + addEventListener: sinon.fake(), + removeEventListener: sinon.fake(), + onended: sinon.fake(), + }; + + // Create a desktop stream + const stream = new MediaStream(); + + // Create a fake `MediaStream` and add the mock video track + const cameraStream = new MediaStream(); + cameraStream.getVideoTracks = () => [mockVideoTrack]; // Override `getVideoTracks` + + sinon.stub(navigator.mediaDevices, 'getUserMedia').resolves(cameraStream); + + await mediaManager.setDesktopwithCameraSource(stream, "streamId", null); + + expect(mediaManager.desktopStream).to.equal(stream); + expect(mediaManager.smallVideoTrack).to.equal(mockVideoTrack); + navigator.mediaDevices.getUserMedia.restore(); + }); + + it("should initialize local stream with video and audio", async function () { + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://example.com", + }); + + var mediaManager = new MediaManager({ + userParameters: { + mediaConstraints: { + video: true, + audio: true, + } + }, + webRTCAdaptor: adaptor, + + callback: (info, obj) => { + adaptor.notifyEventListeners(info, obj) + }, + callbackError: (error, message) => { + adaptor.notifyErrorEventListeners(error, message) + }, + getSender: (streamId, type) => { + return adaptor.getSender(streamId, type) + }, + }); + + await mediaManager.initLocalStream(); + + expect(mediaManager.localStream.getAudioTracks().length).to.be.equal(1); + expect(mediaManager.localStream.getVideoTracks().length).to.be.equal(1); + }); + + it("should handle error when initializing local stream", async function () { + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://example.com", + }); + + var mediaManager = new MediaManager({ + userParameters: { + mediaConstraints: { + video: true, + audio: true, + } + }, + webRTCAdaptor: adaptor, + + callback: (info, obj) => { + adaptor.notifyEventListeners(info, obj) + }, + callbackError: (error, message) => { + adaptor.notifyErrorEventListeners(error, message) + }, + getSender: (streamId, type) => { + return adaptor.getSender(streamId, type) + }, + }); + + sinon.stub(navigator.mediaDevices, 'getUserMedia').rejects(new Error("Permission denied")); + + try { + await mediaManager.initLocalStream(); + } catch (error) { + expect(error.message).to.be.equal("Permission denied"); + } + + navigator.mediaDevices.getUserMedia.restore(); + }); + + it("should switch video camera capture", async function () { + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://example.com", + mediaConstraints: { + video: true, + audio: true + } + }); + + await adaptor.mediaManager.initLocalStream(); + + const deviceId = "testDeviceId"; + await adaptor.mediaManager.switchVideoCameraCapture("streamId", deviceId); + + expect(adaptor.mediaManager.localStream.getVideoTracks()[0].getSettings().deviceId).to.not.be.equal(deviceId); + }); + + it("should handle error when switching video camera capture", async function () { + var adaptor = new WebRTCAdaptor({ + websocketURL: "ws://example.com", + mediaConstraints: { + video: true, + audio: true + } + }); + + await adaptor.mediaManager.initLocalStream(); + + const deviceId = "invalidDeviceId"; + sinon.stub(navigator.mediaDevices, 'getUserMedia').rejects(new Error("Device not found")); + + try { + await adaptor.mediaManager.switchVideoCameraCapture("streamId", deviceId); + } catch (error) { + expect(error.message).to.be.equal("Device not found"); + } + + navigator.mediaDevices.getUserMedia.restore(); + }); + + it("should play video and resolve promise when metadata is loaded", async function() { + const video = document.createElement("video"); + const playStub = sinon.stub(video, "play").resolves(); + const resolveSpy = sinon.spy(); + + const promise = new Promise(resolve => { + video.onloadedmetadata = () => { + video.play(); + resolve(video); + }; + }); + + video.onloadedmetadata(); + + await promise; + + expect(playStub.calledOnce).to.be.true; + expect(resolveSpy.calledOnceWith(video)).to.be.false; + + playStub.restore(); + }); + + it("should handle error when video play fails", async function() { + const video = document.createElement("video"); + const playStub = sinon.stub(video, "play").rejects(new Error("play error")); + const resolveSpy = sinon.spy(); + + const promise = new Promise((resolve, reject) => { + video.onloadedmetadata = () => { + video.play().catch(reject); + resolve(video); + }; + }); + + video.onloadedmetadata(); + + try { + await promise; + } catch (error) { + expect(error.message).to.be.equal("play error"); + } + + expect(playStub.calledOnce).to.be.true; + expect(resolveSpy.calledOnceWith(video)).to.be.false; + + playStub.restore(); + }); + + }); + });