From 2ee872f1dccb3285fe6e69dd40c4fcb7a7f10225 Mon Sep 17 00:00:00 2001 From: jonadimovska Date: Mon, 6 Apr 2026 15:02:30 +0200 Subject: [PATCH 1/3] feat(client): expose blocked autoplay audio state and resume API --- packages/client/src/Call.ts | 7 + .../client/src/helpers/DynascaleManager.ts | 80 +++++++++++- .../__tests__/DynascaleManager.test.ts | 122 ++++++++++++++++++ .../src/hooks/callStateHooks.ts | 12 ++ 4 files changed, 219 insertions(+), 2 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 92ac8c5a67..225f8cf44b 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -2922,6 +2922,13 @@ export class Call { }; }; + /** + * Plays all audio elements blocked by the browser's autoplay policy. + */ + resumeAudio = () => { + return this.dynascaleManager.resumeAudio(); + }; + /** * Binds a DOM element to this call's thumbnail (if enabled in settings). * diff --git a/packages/client/src/helpers/DynascaleManager.ts b/packages/client/src/helpers/DynascaleManager.ts index 1b46fdee94..c887faf5ad 100644 --- a/packages/client/src/helpers/DynascaleManager.ts +++ b/packages/client/src/helpers/DynascaleManager.ts @@ -27,7 +27,7 @@ import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signa import { CallState } from '../store'; import type { StreamSfuClient } from '../StreamSfuClient'; import { SpeakerManager } from '../devices'; -import { getCurrentValue, setCurrentValue } from '../store/rxUtils'; +import { getCurrentValue, Patch, setCurrentValue } from '../store/rxUtils'; import { videoLoggerSystem } from '../logger'; import { Tracer } from '../stats'; @@ -79,6 +79,50 @@ export class DynascaleManager { private pendingSubscriptionsUpdate: NodeJS.Timeout | null = null; readonly audioBindingsWatchdog: AudioBindingsWatchdog | undefined; + /** + * Audio elements that were blocked by the browser's autoplay policy. + * These can be retried by calling `resumeAudio()` from a user gesture. + */ + private blockedAudioElementsSubject = new BehaviorSubject< + Set + >(new Set()); + + blockedAudioElements$ = this.blockedAudioElementsSubject.asObservable(); + + /** + * Whether the browser's autoplay policy is blocking audio playback. + * Will be `true` when the browser blocks autoplay (e.g., no prior user interaction). + * Use `resumeAudio()` within a user gesture to unblock. + */ + autoplayBlocked$ = this.blockedAudioElements$.pipe( + map((elements) => elements.size > 0), + distinctUntilChanged(), + ); + + get blockedAudioElements() { + return getCurrentValue(this.blockedAudioElements$); + } + + private setBlockedAudioElements = (update: Patch>) => { + return setCurrentValue(this.blockedAudioElementsSubject, update); + }; + + private addBlockedAudioElement = (audioElement: HTMLAudioElement) => { + this.setBlockedAudioElements((elements) => { + const next = new Set(elements); + next.add(audioElement); + return next; + }); + }; + + private removeBlockedAudioElement = (audioElement: HTMLAudioElement) => { + this.setBlockedAudioElements((elements) => { + const nextElements = new Set(elements); + nextElements.delete(audioElement); + return nextElements; + }); + }; + private videoTrackSubscriptionOverridesSubject = new BehaviorSubject({}); @@ -136,6 +180,7 @@ export class DynascaleManager { clearTimeout(this.pendingSubscriptionsUpdate); } this.audioBindingsWatchdog?.dispose(); + this.setBlockedAudioElements(new Set()); const context = this.audioContext; if (context && context.state !== 'closed') { document.removeEventListener('click', this.resumeAudioContext); @@ -575,7 +620,10 @@ export class DynascaleManager { setTimeout(() => { audioElement.srcObject = source ?? null; - if (!source) return; + if (!source) { + this.removeBlockedAudioElement(audioElement); + return; + } // Safari has a special quirk that prevents playing audio until the user // interacts with the page or focuses on the tab where the call happens. @@ -599,6 +647,9 @@ export class DynascaleManager { audioElement.muted = false; audioElement.play().catch((e) => { this.tracer.trace('audioPlaybackError', e.message); + if (e.name === 'NotAllowedError') { + this.addBlockedAudioElement(audioElement); + } this.logger.warn(`Failed to play audio stream`, e); }); } @@ -628,6 +679,7 @@ export class DynascaleManager { return () => { this.audioBindingsWatchdog?.unregister(sessionId, trackType); + this.removeBlockedAudioElement(audioElement); sinkIdSubscription?.unsubscribe(); volumeSubscription.unsubscribe(); updateMediaStreamSubscription.unsubscribe(); @@ -637,6 +689,30 @@ export class DynascaleManager { }; }; + /** + * Plays all audio elements blocked by the browser's autoplay policy. + * Must be called from within a user gesture (e.g., click handler). + * + * @returns a promise that resolves when all blocked elements have been retried. + */ + resumeAudio = async () => { + const blocked = new Set(); + await Promise.all( + Array.from(this.blockedAudioElements, async (el) => { + try { + if (el.srcObject) { + await el.play(); + } + } catch { + this.logger.warn(`Can't resume audio for element: `, el); + blocked.add(el); + } + }), + ); + + this.setBlockedAudioElements(blocked); + }; + private getOrCreateAudioContext = (): AudioContext | undefined => { if (!this.useWebAudio) return; if (this.audioContext) return this.audioContext; diff --git a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts index 0bdeaa6f6d..a1a99411a9 100644 --- a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts +++ b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts @@ -647,6 +647,128 @@ describe('DynascaleManager', () => { expect(unregisterSpy).toHaveBeenCalledWith('session-id', 'audioTrack'); }); + it('audio: should track blocked audio elements on NotAllowedError', async () => { + vi.useFakeTimers(); + const audioElement = document.createElement('audio'); + Object.defineProperty(audioElement, 'srcObject', { writable: true }); + const notAllowedError = new DOMException('', 'NotAllowedError'); + vi.spyOn(audioElement, 'play').mockRejectedValue(notAllowedError); + + // @ts-expect-error incomplete data + call.state.updateOrAddParticipant('session-id', { + userId: 'user-id', + sessionId: 'session-id', + publishedTracks: [], + }); + + const cleanup = dynascaleManager.bindAudioElement( + audioElement, + 'session-id', + 'audioTrack', + ); + + const mediaStream = new MediaStream(); + call.state.updateParticipant('session-id', { + audioStream: mediaStream, + }); + + vi.runAllTimers(); + await vi.advanceTimersByTimeAsync(0); + + expect(dynascaleManager.blockedAudioElements.size).toBe(1); + expect(dynascaleManager.blockedAudioElements.has(audioElement)).toBe( + true, + ); + + cleanup?.(); + expect(dynascaleManager.blockedAudioElements.size).toBe(0); + }); + + it('audio: should unblock audio elements on explicit resumeAudio call', async () => { + vi.useFakeTimers(); + const audioElement = document.createElement('audio'); + Object.defineProperty(audioElement, 'srcObject', { writable: true }); + const playSpy = vi + .spyOn(audioElement, 'play') + .mockRejectedValueOnce(new DOMException('', 'NotAllowedError')) + .mockResolvedValue(undefined); + + // @ts-expect-error incomplete data + call.state.updateOrAddParticipant('session-id', { + userId: 'user-id', + sessionId: 'session-id', + publishedTracks: [], + }); + + const cleanup = dynascaleManager.bindAudioElement( + audioElement, + 'session-id', + 'audioTrack', + ); + + const mediaStream = new MediaStream(); + call.state.updateParticipant('session-id', { + audioStream: mediaStream, + }); + + vi.runAllTimers(); + await vi.advanceTimersByTimeAsync(0); + + expect(dynascaleManager.blockedAudioElements.size).toBe(1); + + await dynascaleManager.resumeAudio(); + await vi.advanceTimersByTimeAsync(0); + + expect(playSpy).toHaveBeenCalledTimes(2); + expect(dynascaleManager.blockedAudioElements.size).toBe(0); + + cleanup?.(); + }); + + it('audio: should clear blocked state when the audio stream is removed', async () => { + vi.useFakeTimers(); + const audioElement = document.createElement('audio'); + Object.defineProperty(audioElement, 'srcObject', { writable: true }); + vi.spyOn(audioElement, 'play').mockRejectedValue( + new DOMException('', 'NotAllowedError'), + ); + + // @ts-expect-error incomplete data + call.state.updateOrAddParticipant('session-id', { + userId: 'user-id', + sessionId: 'session-id', + publishedTracks: [], + }); + + const cleanup = dynascaleManager.bindAudioElement( + audioElement, + 'session-id', + 'audioTrack', + ); + + const mediaStream = new MediaStream(); + call.state.updateParticipant('session-id', { + audioStream: mediaStream, + }); + + vi.runAllTimers(); + await vi.advanceTimersByTimeAsync(0); + + expect(dynascaleManager.blockedAudioElements.size).toBe(1); + + call.state.updateParticipant('session-id', { + audioStream: undefined, + }); + + vi.runAllTimers(); + await vi.advanceTimersByTimeAsync(0); + + expect(audioElement.srcObject).toBeNull(); + expect(dynascaleManager.blockedAudioElements.size).toBe(0); + + cleanup?.(); + }); + it('audio: should warn when binding an already-bound session', () => { const watchdog = dynascaleManager.audioBindingsWatchdog!; // @ts-expect-error private property diff --git a/packages/react-bindings/src/hooks/callStateHooks.ts b/packages/react-bindings/src/hooks/callStateHooks.ts index 550c021a5b..c53e4e0548 100644 --- a/packages/react-bindings/src/hooks/callStateHooks.ts +++ b/packages/react-bindings/src/hooks/callStateHooks.ts @@ -496,6 +496,18 @@ export const useIncomingVideoSettings = () => { return useObservableValue(call.dynascaleManager.incomingVideoSettings$); }; +/** + * Returns whether the browser's autoplay policy is blocking audio playback. + * + * When the browser blocks audio autoplay (e.g., no prior user interaction), + * this hook returns `true`. Use `call.resumeAudio()` inside a click handler + * to unblock audio playback. + */ +export const useIsAutoplayBlocked = (): boolean => { + const call = useCall() as Call; + return useObservableValue(call.dynascaleManager.autoplayBlocked$); +}; + /** * Returns the current call's closed captions queue. */ From fe502bad42723f4ec84635cb1657e13308c4782b Mon Sep 17 00:00:00 2001 From: jonadimovska Date: Mon, 6 Apr 2026 15:18:15 +0200 Subject: [PATCH 2/3] feat(client): expose blocked autoplay audio state and resume API --- .../client/src/helpers/DynascaleManager.ts | 43 ++++++++----------- .../__tests__/DynascaleManager.test.ts | 16 +++---- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/packages/client/src/helpers/DynascaleManager.ts b/packages/client/src/helpers/DynascaleManager.ts index c887faf5ad..4f84c2df34 100644 --- a/packages/client/src/helpers/DynascaleManager.ts +++ b/packages/client/src/helpers/DynascaleManager.ts @@ -27,7 +27,7 @@ import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signa import { CallState } from '../store'; import type { StreamSfuClient } from '../StreamSfuClient'; import { SpeakerManager } from '../devices'; -import { getCurrentValue, Patch, setCurrentValue } from '../store/rxUtils'; +import { getCurrentValue, setCurrentValue } from '../store/rxUtils'; import { videoLoggerSystem } from '../logger'; import { Tracer } from '../stats'; @@ -87,28 +87,18 @@ export class DynascaleManager { Set >(new Set()); - blockedAudioElements$ = this.blockedAudioElementsSubject.asObservable(); - /** * Whether the browser's autoplay policy is blocking audio playback. * Will be `true` when the browser blocks autoplay (e.g., no prior user interaction). * Use `resumeAudio()` within a user gesture to unblock. */ - autoplayBlocked$ = this.blockedAudioElements$.pipe( + autoplayBlocked$ = this.blockedAudioElementsSubject.pipe( map((elements) => elements.size > 0), distinctUntilChanged(), ); - get blockedAudioElements() { - return getCurrentValue(this.blockedAudioElements$); - } - - private setBlockedAudioElements = (update: Patch>) => { - return setCurrentValue(this.blockedAudioElementsSubject, update); - }; - private addBlockedAudioElement = (audioElement: HTMLAudioElement) => { - this.setBlockedAudioElements((elements) => { + setCurrentValue(this.blockedAudioElementsSubject, (elements) => { const next = new Set(elements); next.add(audioElement); return next; @@ -116,7 +106,7 @@ export class DynascaleManager { }; private removeBlockedAudioElement = (audioElement: HTMLAudioElement) => { - this.setBlockedAudioElements((elements) => { + setCurrentValue(this.blockedAudioElementsSubject, (elements) => { const nextElements = new Set(elements); nextElements.delete(audioElement); return nextElements; @@ -180,7 +170,7 @@ export class DynascaleManager { clearTimeout(this.pendingSubscriptionsUpdate); } this.audioBindingsWatchdog?.dispose(); - this.setBlockedAudioElements(new Set()); + setCurrentValue(this.blockedAudioElementsSubject, new Set()); const context = this.audioContext; if (context && context.state !== 'closed') { document.removeEventListener('click', this.resumeAudioContext); @@ -698,19 +688,22 @@ export class DynascaleManager { resumeAudio = async () => { const blocked = new Set(); await Promise.all( - Array.from(this.blockedAudioElements, async (el) => { - try { - if (el.srcObject) { - await el.play(); + Array.from( + getCurrentValue(this.blockedAudioElementsSubject), + async (el) => { + try { + if (el.srcObject) { + await el.play(); + } + } catch { + this.logger.warn(`Can't resume audio for element: `, el); + blocked.add(el); } - } catch { - this.logger.warn(`Can't resume audio for element: `, el); - blocked.add(el); - } - }), + }, + ), ); - this.setBlockedAudioElements(blocked); + setCurrentValue(this.blockedAudioElementsSubject, blocked); }; private getOrCreateAudioContext = (): AudioContext | undefined => { diff --git a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts index a1a99411a9..96c63037d0 100644 --- a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts +++ b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts @@ -18,6 +18,7 @@ import { DynascaleManager } from '../DynascaleManager'; import { Call } from '../../Call'; import { StreamClient } from '../../coordinator/connection/client'; import { StreamVideoWriteableStateStore } from '../../store'; +import { getCurrentValue } from '../../store/rxUtils'; import { VisibilityState } from '../../types'; import { noopComparator } from '../../sorting'; import { TrackType } from '../../gen/video/sfu/models/models'; @@ -675,13 +676,10 @@ describe('DynascaleManager', () => { vi.runAllTimers(); await vi.advanceTimersByTimeAsync(0); - expect(dynascaleManager.blockedAudioElements.size).toBe(1); - expect(dynascaleManager.blockedAudioElements.has(audioElement)).toBe( - true, - ); + expect(getCurrentValue(dynascaleManager.autoplayBlocked$)).toBe(true); cleanup?.(); - expect(dynascaleManager.blockedAudioElements.size).toBe(0); + expect(getCurrentValue(dynascaleManager.autoplayBlocked$)).toBe(false); }); it('audio: should unblock audio elements on explicit resumeAudio call', async () => { @@ -714,13 +712,13 @@ describe('DynascaleManager', () => { vi.runAllTimers(); await vi.advanceTimersByTimeAsync(0); - expect(dynascaleManager.blockedAudioElements.size).toBe(1); + expect(getCurrentValue(dynascaleManager.autoplayBlocked$)).toBe(true); await dynascaleManager.resumeAudio(); await vi.advanceTimersByTimeAsync(0); expect(playSpy).toHaveBeenCalledTimes(2); - expect(dynascaleManager.blockedAudioElements.size).toBe(0); + expect(getCurrentValue(dynascaleManager.autoplayBlocked$)).toBe(false); cleanup?.(); }); @@ -754,7 +752,7 @@ describe('DynascaleManager', () => { vi.runAllTimers(); await vi.advanceTimersByTimeAsync(0); - expect(dynascaleManager.blockedAudioElements.size).toBe(1); + expect(getCurrentValue(dynascaleManager.autoplayBlocked$)).toBe(true); call.state.updateParticipant('session-id', { audioStream: undefined, @@ -764,7 +762,7 @@ describe('DynascaleManager', () => { await vi.advanceTimersByTimeAsync(0); expect(audioElement.srcObject).toBeNull(); - expect(dynascaleManager.blockedAudioElements.size).toBe(0); + expect(getCurrentValue(dynascaleManager.autoplayBlocked$)).toBe(false); cleanup?.(); }); From c9c931b7d4bf9ec73bce24bd67c65563de1d3623 Mon Sep 17 00:00:00 2001 From: jonadimovska Date: Mon, 6 Apr 2026 16:18:31 +0200 Subject: [PATCH 3/3] feat(client): expose blocked autoplay audio state and resume API --- packages/client/src/helpers/DynascaleManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/client/src/helpers/DynascaleManager.ts b/packages/client/src/helpers/DynascaleManager.ts index 4f84c2df34..aa00b9933b 100644 --- a/packages/client/src/helpers/DynascaleManager.ts +++ b/packages/client/src/helpers/DynascaleManager.ts @@ -638,6 +638,7 @@ export class DynascaleManager { audioElement.play().catch((e) => { this.tracer.trace('audioPlaybackError', e.message); if (e.name === 'NotAllowedError') { + this.tracer.trace('audioPlaybackBlocked', null); this.addBlockedAudioElement(audioElement); } this.logger.warn(`Failed to play audio stream`, e); @@ -686,6 +687,7 @@ export class DynascaleManager { * @returns a promise that resolves when all blocked elements have been retried. */ resumeAudio = async () => { + this.tracer.trace('resumeAudio', null); const blocked = new Set(); await Promise.all( Array.from(