diff --git a/src/main/js/webrtc_adaptor.js b/src/main/js/webrtc_adaptor.js index c6a29390..84b3fc05 100644 --- a/src/main/js/webrtc_adaptor.js +++ b/src/main/js/webrtc_adaptor.js @@ -387,6 +387,32 @@ export class WebRTCAdaptor { if (this.initializeComponents) { this.initialize(); } + + /** + * Automatic AV Sync Recovery Configurations + */ + this.autoResyncOnFrameDrop = initialValues.autoResyncOnFrameDrop ?? false; + this.autoResyncCooldownMs = initialValues.autoResyncCooldownMs ?? 10000; + this._lastAutoResyncTime = {}; // { streamId: timestamp } + // FPS fluctuation-based auto-resync config + this.fpsFluctuationWindowSize = initialValues.fpsFluctuationWindowSize ?? 3; // e.g., last 5 seconds + this.fpsDropPercentThreshold = initialValues.fpsDropPercentThreshold ?? 0.1; // 10% drop + this.fpsFluctuationStdDevThreshold = initialValues.fpsFluctuationStdDevThreshold ?? 1; // e.g., 5 FPS + this.fpsFluctuationConsecutiveCount = initialValues.fpsFluctuationConsecutiveCount ?? 1; + this._fpsHistory = {}; // { streamId: [fps] } + this._fpsFluctuationCount = {}; // { streamId: count } + this._lastFramesReceived = {}; // { streamId: last framesReceived } + this._lastStatsTime = {}; // { streamId: last timestamp } + // Improved fluctuation handling + this.fpsStableStdDevThreshold = initialValues.fpsStableStdDevThreshold ?? 2; + // Dynamic healthy FPS baseline and stabilization window + this.fpsHealthyBaselineWindow = initialValues.fpsHealthyBaselineWindow ?? 5; // samples for healthy baseline + this.fpsStableWindow = initialValues.fpsStableWindow ?? 3; // samples for stabilization + this.fpsStablePercentOfBaseline = initialValues.fpsStablePercentOfBaseline ?? 0.8; // 80% + this._fpsHealthyBaseline = {}; // { streamId: [fps] } + this._lastHealthyAvgFps = {}; // { streamId: number } + this.healthyWindowSize = initialValues.healthyWindowSize ?? 5; + this._fpsFluctuating = {}; // { streamId: boolean } } /** @@ -1744,10 +1770,100 @@ export class WebRTCAdaptor { this.remotePeerConnectionStats[streamId].audioPacketsReceived = audioPacketsReceived; this.remotePeerConnectionStats[streamId].videoPacketsReceived = videoPacketsReceived; + const now = Date.now(); + const lastFrames = this._lastFramesReceived[streamId] ?? framesReceived; + const lastTime = this._lastStatsTime[streamId] ?? now; + const calculatedFps = (framesReceived - lastFrames) / ((now - lastTime) / 1000); + this._lastFramesReceived[streamId] = framesReceived; + this._lastStatsTime[streamId] = now; + + this.checkAndHandleFpsFluctuation(streamId, calculatedFps, now); + return this.remotePeerConnectionStats[streamId]; } + checkAndHandleFpsFluctuation(streamId, calculatedFps, now) { + if (!this._fpsHistory[streamId]) this._fpsHistory[streamId] = []; + this._fpsHistory[streamId].push(calculatedFps); + if (this._fpsHistory[streamId].length > this.fpsFluctuationWindowSize + 1) { + this._fpsHistory[streamId].shift(); + } + + if (this._fpsHistory[streamId].length > this.fpsFluctuationWindowSize) { + Logger.debug(`FPS Fluctuation Detection - Stream: ${streamId}, Current FPS: ${calculatedFps.toFixed(2)}`); + // Exclude the current FPS from the average of previous values + const prevFpsValues = this._fpsHistory[streamId].slice(0, -1); + const prevAvgFps = prevFpsValues.reduce((a, b) => a + b, 0) / prevFpsValues.length; + const prevStdDev = Math.sqrt(prevFpsValues.reduce((a, b) => a + Math.pow(b - prevAvgFps, 2), 0) / prevFpsValues.length); + + // Update healthy baseline and last healthy avg if not fluctuating + if (!this._fpsFluctuating[streamId]) { + if (!this._fpsHealthyBaseline[streamId]) this._fpsHealthyBaseline[streamId] = []; + this._fpsHealthyBaseline[streamId].push(calculatedFps); + if (this._fpsHealthyBaseline[streamId].length > this.healthyWindowSize) { + this._fpsHealthyBaseline[streamId].shift(); + } + // Update last healthy avg + const healthyArr = this._fpsHealthyBaseline[streamId]; + if (healthyArr.length === this.healthyWindowSize) { + this._lastHealthyAvgFps[streamId] = healthyArr.reduce((a, b) => a + b, 0) / healthyArr.length; + } + } + // Fluctuation detection: use last healthy avg + const healthyAvg = this._lastHealthyAvgFps[streamId] ?? calculatedFps; + const fluctuationDetected = ( + calculatedFps < healthyAvg * (1 - this.fpsDropPercentThreshold) && + prevStdDev > this.fpsFluctuationStdDevThreshold + ); + if (fluctuationDetected) { + this._fpsFluctuationCount[streamId] = (this._fpsFluctuationCount[streamId] || 0) + 1; + Logger.debug(`FPS Fluctuation Detection - Stream: ${streamId}, Current FPS: ${calculatedFps.toFixed(2)}, Count: ${this._fpsFluctuationCount[streamId]}`); + if (this._fpsFluctuationCount[streamId] >= this.fpsFluctuationConsecutiveCount) { + this._fpsFluctuating[streamId] = true; + } + } else { + this._fpsFluctuationCount[streamId] = 0; + } + + // If we are in a fluctuating state, check for stabilization + if (this._fpsFluctuating[streamId]) { + // Use only the last N samples for stabilization + const stableWindow = this._fpsHistory[streamId].slice(-this.fpsStableWindow); + const stableAvg = stableWindow.reduce((a, b) => a + b, 0) / stableWindow.length; + const stableStdDev = Math.sqrt(stableWindow.reduce((a, b) => a + Math.pow(b - stableAvg, 2), 0) / stableWindow.length); + // Use the healthy baseline + const healthyBaselineArr = this._fpsHealthyBaseline[streamId] || []; + const healthyBaseline = healthyBaselineArr.length > 0 ? (healthyBaselineArr.reduce((a, b) => a + b, 0) / healthyBaselineArr.length) : 0; + Logger.debug(`FPS Stabilization Check - Stream: ${streamId}, StableAvg: ${stableAvg.toFixed(2)}, StableStdDev: ${stableStdDev.toFixed(2)}, HealthyBaseline: ${healthyBaseline.toFixed(2)}`); + if (stableStdDev < this.fpsStableStdDevThreshold && stableAvg > healthyBaseline * this.fpsStablePercentOfBaseline) { + // Now stable, trigger restart + if (this.autoResyncOnFrameDrop && this.playStreamId && this.playStreamId.includes(streamId)) { + if (!this._lastAutoResyncTime[streamId] || now - this._lastAutoResyncTime[streamId] > this.autoResyncCooldownMs) { + this._lastAutoResyncTime[streamId] = now; + Logger.warn(`Auto-resync triggered for stream ${streamId} after FPS stabilized. StableAvg: ${stableAvg.toFixed(2)}, StableStdDev: ${stableStdDev.toFixed(2)}, HealthyBaseline: ${healthyBaseline.toFixed(2)}`); + this.notifyEventListeners("auto_resync_triggered", { streamId, calculatedFps, stableAvg, stableStdDev, healthyBaseline, stabilized: true }); + this.stop(streamId); + setTimeout(() => { + this.play( + streamId, + this.playToken, + this.playRoomId, + this.playEnableTracks, + this.playSubscriberId, + this.playSubscriberCode, + this.playMetaData, + this.playRole + ); + }, 500); + } + } + this._fpsFluctuating[streamId] = false; // Reset + this._fpsFluctuationCount[streamId] = 0; + } + } + } + } /** * Called to start a periodic timer to get statistics periodically (5 seconds) for a specific stream. diff --git a/src/test/js/webrtc_adaptor.test.js b/src/test/js/webrtc_adaptor.test.js index 106d5d0f..c92fdbe8 100644 --- a/src/test/js/webrtc_adaptor.test.js +++ b/src/test/js/webrtc_adaptor.test.js @@ -2101,6 +2101,140 @@ describe("WebRTCAdaptor", function() { expect(initPeerConnection.calledWithExactly(streamId, "play")).to.be.true; }); + describe("checkAndHandleFpsFluctuation", function() { + let adaptor; + let streamId = "testStream"; + let now = Date.now(); + let loggerDebugStub, loggerWarnStub; + let stopStub, playStub; + let notifyEventListenersStub; + beforeEach(function() { + adaptor = new WebRTCAdaptor({ + websocketURL: "ws://example.com", + initializeComponents: false, + }); + // Set up stubs for Logger + loggerDebugStub = sinon.stub(window.log, "debug"); + loggerWarnStub = sinon.stub(window.log, "warn"); + // Set up stubs for stop/play and event notification + stopStub = sinon.stub(adaptor, "stop"); + playStub = sinon.stub(adaptor, "play"); + notifyEventListenersStub = sinon.stub(adaptor, "notifyEventListeners"); + // Set up playStreamId for auto-resync + adaptor.playStreamId = [streamId]; + // Set up thresholds for easier testing + adaptor.fpsFluctuationWindowSize = 2; + adaptor.fpsFluctuationStdDevThreshold = 1; + adaptor.fpsDropPercentThreshold = 0.1; + adaptor.fpsFluctuationConsecutiveCount = 2; + adaptor.fpsStableWindow = 2; + adaptor.fpsStableStdDevThreshold = 1; + adaptor.fpsStablePercentOfBaseline = 0.8; + adaptor.healthyWindowSize = 2; + adaptor.autoResyncOnFrameDrop = true; + adaptor.autoResyncCooldownMs = 1000; + }); + + afterEach(function() { + sinon.restore(); + }); + + it("should update healthy baseline and not trigger fluctuation or resync on stable FPS", function() { + adaptor._fpsFluctuating = {}; + adaptor._fpsHealthyBaseline = {}; + adaptor._lastHealthyAvgFps = {}; + adaptor._fpsHistory = {}; + adaptor._fpsFluctuationCount = {}; + adaptor._lastAutoResyncTime = {}; + // Push stable FPS values + adaptor.checkAndHandleFpsFluctuation(streamId, 30, now); + adaptor.checkAndHandleFpsFluctuation(streamId, 31, now + 1000); + adaptor.checkAndHandleFpsFluctuation(streamId, 32, now + 2000); + // Should not be fluctuating + expect(adaptor._fpsFluctuating[streamId]).to.not.be.true; + expect(adaptor._fpsFluctuationCount[streamId]).to.equal(0); + // Should update healthy baseline + expect(adaptor._fpsHealthyBaseline[streamId].length).to.be.at.most(adaptor.healthyWindowSize); + }); + + it("should detect FPS fluctuation and set fluctuating state after consecutive drops", function() { + const streamId = 'testStream'; + const now = Date.now(); + // Fill healthy baseline with stable values + adaptor._fpsHealthyBaseline[streamId] = [30, 30, 30, 30, 30]; + adaptor._lastHealthyAvgFps[streamId] = 30; + adaptor._fpsFluctuationCount[streamId] = 0; + // Initialize FPS history with enough values to trigger fluctuation detection + adaptor._fpsHistory[streamId] = [30, 30, 30, 30, 30, 30, 30, 30, 30, 30]; + // Add values that will trigger fluctuation detection + adaptor.checkAndHandleFpsFluctuation(streamId, 20, now - 1000); + adaptor.checkAndHandleFpsFluctuation(streamId, 15, now - 500); + adaptor.checkAndHandleFpsFluctuation(streamId, 10, now - 250); + // Simulate a dramatic drop + adaptor.checkAndHandleFpsFluctuation(streamId, 5, now); + // Should now be fluctuating + expect(adaptor._fpsFluctuating[streamId]).to.be.true; + expect(adaptor._fpsFluctuationCount[streamId]).to.be.at.least(1); + }); + + it("should trigger auto-resync and reset fluctuation state when FPS stabilizes", function() { + // Simulate fluctuating state + adaptor._fpsFluctuating[streamId] = true; + adaptor._fpsHistory[streamId] = [20, 20, 30, 31]; // last two are stable + adaptor._fpsHealthyBaseline[streamId] = [30, 30]; + adaptor._lastHealthyAvgFps[streamId] = 30; + adaptor._fpsFluctuationCount[streamId] = 2; + adaptor.playToken = "token"; + adaptor.playRoomId = "room"; + adaptor.playEnableTracks = []; + adaptor.playSubscriberId = "subid"; + adaptor.playSubscriberCode = "subcode"; + adaptor.playMetaData = "meta"; + adaptor.playRole = "role"; + // Should trigger auto-resync + adaptor.checkAndHandleFpsFluctuation(streamId, 32, now + 2000); + expect(loggerWarnStub.called).to.be.true; + expect(notifyEventListenersStub.calledWith("auto_resync_triggered", sinon.match.object)).to.be.true; + expect(stopStub.calledWith(streamId)).to.be.true; + // Simulate setTimeout + clock.tick(600); + expect(playStub.calledWith( + streamId, + "token", + "room", + [], + "subid", + "subcode", + "meta", + "role" + )).to.be.true; + // Should reset fluctuation state + expect(adaptor._fpsFluctuating[streamId]).to.be.false; + expect(adaptor._fpsFluctuationCount[streamId]).to.equal(0); + }); + + it("should not trigger auto-resync if cooldown not passed", function() { + adaptor._fpsFluctuating[streamId] = true; + adaptor._fpsHistory[streamId] = [20, 20, 30, 31]; + adaptor._fpsHealthyBaseline[streamId] = [30, 30]; + adaptor._lastHealthyAvgFps[streamId] = 30; + adaptor._fpsFluctuationCount[streamId] = 2; + adaptor._lastAutoResyncTime[streamId] = now; + adaptor.playToken = "token"; + adaptor.playRoomId = "room"; + adaptor.playEnableTracks = []; + adaptor.playSubscriberId = "subid"; + adaptor.playSubscriberCode = "subcode"; + adaptor.playMetaData = "meta"; + adaptor.playRole = "role"; + // Should not trigger auto-resync due to cooldown + adaptor.checkAndHandleFpsFluctuation(streamId, 32, now + 500); + expect(loggerWarnStub.called).to.be.false; + expect(notifyEventListenersStub.calledWith("auto_resync_triggered", sinon.match.object)).to.be.false; + expect(stopStub.called).to.be.false; + expect(playStub.called).to.be.false; + }); + }); });