Optimize mobile media processing

frontend/src/features/call/hooks/__tests__/useLocalMedia.switchCamera.test.ts

@@ -109,8 +109,7 @@ describe('useLocalMedia.switchCamera', () => {
         value: originalMediaDevices,
       });
     } else {
-      // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-      delete (navigator as { mediaDevices?: MediaDevices }).mediaDevices;
+      Reflect.deleteProperty(navigator as { mediaDevices?: MediaDevices }, 'mediaDevices');
     }
     vi.restoreAllMocks();
   });

frontend/src/features/call/hooks/useLocalMedia.ts

@@ -236,9 +236,13 @@ export const useLocalMedia = ({ roomId, messageApi }: UseLocalMediaOptions) => {
 
   const setupLocalStream = useCallback(async () => {
     try {
+      const videoConstraints = buildVideoConstraints(
+        activeVideoDeviceId,
+        desiredFacingModeRef.current ?? 'user',
+      );
       const stream = await navigator.mediaDevices.getUserMedia({
         audio: true,
-        video: { facingMode: 'user' },
+        video: videoConstraints,
       });
       await applyLocalStream(stream, { micEnabled: isMicEnabled, cameraEnabled: isCameraEnabled });
     } catch (error) {
@@ -250,7 +254,14 @@ export const useLocalMedia = ({ roomId, messageApi }: UseLocalMediaOptions) => {
     }
 
     await refreshDevices();
-  }, [applyLocalStream, isCameraEnabled, isMicEnabled, messageApi, refreshDevices]);
+  }, [
+    activeVideoDeviceId,
+    applyLocalStream,
+    isCameraEnabled,
+    isMicEnabled,
+    messageApi,
+    refreshDevices,
+  ]);
 
   const toggleMicrophone = useCallback(() => {
     const stream = localStreamRef.current;

frontend/src/features/call/utils/mediaConstraints.ts

@@ -20,15 +20,110 @@ export const guessFacingMode = (
   return null;
 };
 
+const MOBILE_VIDEO_CONSTRAINTS: MediaTrackConstraints = {
+  width: { ideal: 960, max: 1280 },
+  height: { ideal: 540, max: 720 },
+  frameRate: { ideal: 24, max: 30 },
+};
+
+const DEFAULT_VIDEO_CONSTRAINTS: MediaTrackConstraints = {
+  width: { ideal: 1280, max: 1920 },
+  height: { ideal: 720, max: 1080 },
+  frameRate: { ideal: 30, max: 30 },
+};
+
+const CONSERVATIVE_VIDEO_CONSTRAINTS: MediaTrackConstraints = {
+  width: { ideal: 640, max: 960 },
+  height: { ideal: 360, max: 540 },
+  frameRate: { ideal: 20, max: 24 },
+};
+
+const isMobileDevice = () => {
+  if (typeof navigator === 'undefined') {
+    return false;
+  }
+
+  const userAgent = navigator.userAgent || '';
+  return /android|iphone|ipad|ipod|mobile/i.test(userAgent);
+};
+
+type NetworkQuality = 'slow' | 'unknown' | 'fast';
+
+type NavigatorConnection = Navigator & {
+  connection?: {
+    effectiveType?: string;
+    saveData?: boolean;
+  };
+};
+
+const getNetworkQuality = (): NetworkQuality => {
+  if (typeof navigator === 'undefined') {
+    return 'unknown';
+  }
+
+  const connection = (navigator as NavigatorConnection).connection;
+
+  if (!connection) {
+    return 'unknown';
+  }
+
+  if ('saveData' in connection && connection.saveData) {
+    return 'slow';
+  }
+
+  const effectiveType = connection.effectiveType;
+  if (!effectiveType) {
+    return 'unknown';
+  }
+
+  if (['slow-2g', '2g', '3g'].includes(effectiveType)) {
+    return 'slow';
+  }
+
+  if (effectiveType === '4g') {
+    return 'fast';
+  }
+
+  return 'unknown';
+};
+
+const pickBaseVideoConstraints = (): MediaTrackConstraints => {
+  const mobile = isMobileDevice();
+  const networkQuality = getNetworkQuality();
+
+  if (networkQuality === 'slow') {
+    return CONSERVATIVE_VIDEO_CONSTRAINTS;
+  }
+
+  if (mobile) {
+    return MOBILE_VIDEO_CONSTRAINTS;
+  }
+
+  return DEFAULT_VIDEO_CONSTRAINTS;
+};
+
 export const buildVideoConstraints = (
   deviceId?: string | null,
   facingMode?: 'user' | 'environment' | null,
 ): MediaTrackConstraints => {
+  const baseConstraints = { ...pickBaseVideoConstraints() };
+
   if (deviceId) {
-    return { deviceId: { exact: deviceId } };
+    return {
+      ...baseConstraints,
+      deviceId: { exact: deviceId },
+    };
   }
+
   if (facingMode) {
-    return { facingMode: { exact: facingMode } };
+    return {
+      ...baseConstraints,
+      facingMode: { exact: facingMode },
+    };
   }
-  return { facingMode: 'user' };
+
+  return {
+    ...baseConstraints,
+    facingMode: 'user',
+  };
 };

frontend/src/lib/audioMixer.ts

@@ -54,9 +54,9 @@ export class AudioStreamMixer {
 
   private systemGain: GainNode | null = null;
 
-  private scheduleNextRaf = false;
+  private volumeIntervalId: number | null = null;
 
-  private rafId: number | null = null;
+  private readonly volumeCheckIntervalMs = 150;
 
   private options: AudioMixerOptions;
 
@@ -94,37 +94,53 @@ export class AudioStreamMixer {
       this.analyser = getAnalyser(this.context);
       micSource.connect(this.analyser);
 
-      this.scheduleNextRaf = true;
-      const listen = () => {
-        if (!this.context || !this.analyser || !this.systemGain) {
-          return;
-        }
-        const average = getAverageVolume(this.analyser);
-
-        if (average > microAverageVolume) {
-          this.systemGain.gain.setTargetAtTime(0.32, this.context.currentTime, 0.05);
-        } else {
-          this.systemGain.gain.setTargetAtTime(1, this.context.currentTime, 0.05);
-        }
-
-        if (this.scheduleNextRaf) {
-          this.rafId = requestAnimationFrame(listen);
-        }
-      };
-
-      listen();
+      this.startVolumeMonitoring(microAverageVolume);
     }
   }
 
+  private startVolumeMonitoring(microAverageVolume: number) {
+    if (typeof window === 'undefined') {
+      return;
+    }
+
+    if (this.volumeIntervalId !== null) {
+      window.clearInterval(this.volumeIntervalId);
+    }
+
+    const checkVolume = () => {
+      if (!this.context || !this.analyser || !this.systemGain) {
+        return;
+      }
+
+      const hasActiveMic = this.options.userAudioStream
+        ?.getAudioTracks()
+        .some((track) => track.enabled && track.readyState === 'live');
+
+      if (!hasActiveMic) {
+        this.systemGain.gain.setTargetAtTime(1, this.context.currentTime, 0.05);
+        return;
+      }
+
+      const average = getAverageVolume(this.analyser);
+      if (average > microAverageVolume) {
+        this.systemGain.gain.setTargetAtTime(0.32, this.context.currentTime, 0.05);
+      } else {
+        this.systemGain.gain.setTargetAtTime(1, this.context.currentTime, 0.05);
+      }
+    };
+
+    checkVolume();
+    this.volumeIntervalId = window.setInterval(checkVolume, this.volumeCheckIntervalMs);
+  }
+
   getAudioStream() {
     return this.audioStream;
   }
 
   stop() {
-    this.scheduleNextRaf = false;
-    if (this.rafId) {
-      cancelAnimationFrame(this.rafId);
-      this.rafId = null;
+    if (typeof window !== 'undefined' && this.volumeIntervalId !== null) {
+      window.clearInterval(this.volumeIntervalId);
+      this.volumeIntervalId = null;
     }
   }
 

frontend/src/lib/videoMixer.ts

@@ -38,10 +38,34 @@ export class VideoStreamMixer {
 
   private rafId: number | null = null;
 
-  private scheduleNextFrame = false;
+  private isLoopEnabled = false;
+
+  private isVisibilityPaused = false;
+
+  private lastFrameTimestamp = 0;
+
+  private readonly frameIntervalMs = 1000 / 30;
 
   private options: MixerOptions;
 
+  private readonly handleVisibilityChange = () => {
+    if (typeof document === 'undefined') {
+      return;
+    }
+
+    this.isVisibilityPaused = document.visibilityState === 'hidden';
+
+    if (this.isVisibilityPaused && this.rafId) {
+      cancelAnimationFrame(this.rafId);
+      this.rafId = null;
+      return;
+    }
+
+    if (!this.isVisibilityPaused) {
+      this.requestNextFrame();
+    }
+  };
+
   constructor(options: MixerOptions) {
     this.options = options;
   }
@@ -80,9 +104,15 @@ export class VideoStreamMixer {
       this.prepareVideoElement(this.options.secondStream);
     }
 
-    this.scheduleNextFrame = true;
-    this.computeFrame();
-    this.videoStream = canvas.captureStream(60);
+    this.isLoopEnabled = true;
+    this.isVisibilityPaused = typeof document !== 'undefined' && document.visibilityState === 'hidden';
+    this.lastFrameTimestamp = 0;
+    this.requestNextFrame();
+    this.videoStream = canvas.captureStream(30);
+
+    if (typeof document !== 'undefined') {
+      document.addEventListener('visibilitychange', this.handleVisibilityChange);
+    }
   }
 
   private prepareVideoElement(streamConfig: StreamConfig) {
@@ -99,11 +129,41 @@ export class VideoStreamMixer {
     }
   }
 
-  private computeFrame = () => {
+  private requestNextFrame() {
+    if (!this.isLoopEnabled || this.isVisibilityPaused || this.rafId !== null) {
+      return;
+    }
+
+    this.rafId = requestAnimationFrame(this.computeFrame);
+  }
+
+  private computeFrame = (timestamp?: number) => {
     if (!this.ctx || !this.canvas) {
       return;
     }
 
+    if (!this.isLoopEnabled) {
+      if (this.rafId) {
+        cancelAnimationFrame(this.rafId);
+        this.rafId = null;
+      }
+      return;
+    }
+
+    if (this.isVisibilityPaused) {
+      this.rafId = null;
+      return;
+    }
+
+    const now = typeof timestamp === 'number' ? timestamp : performance.now();
+    if (now - this.lastFrameTimestamp < this.frameIntervalMs) {
+      this.rafId = null;
+      this.requestNextFrame();
+      return;
+    }
+
+    this.lastFrameTimestamp = now;
+
     const { firstStream, secondStream, sizes } = this.options;
     const firstVideo = firstStream.videoElement;
     const firstWidth = firstStream.width ?? sizes.width;
@@ -122,25 +182,28 @@ export class VideoStreamMixer {
       this.ctx.drawImage(secondVideo, secondLeft, secondTop, secondWidth, secondHeight);
     }
 
-    if (this.scheduleNextFrame) {
-      this.rafId = requestAnimationFrame(this.computeFrame);
-    }
+    this.rafId = null;
+    this.requestNextFrame();
   };
 
   getVideoStream() {
     return this.videoStream;
   }
 
   stop() {
-    this.scheduleNextFrame = false;
+    this.isLoopEnabled = false;
     if (this.rafId) {
       cancelAnimationFrame(this.rafId);
       this.rafId = null;
     }
+    this.isVisibilityPaused = false;
   }
 
   destroy() {
     this.stop();
+    if (typeof document !== 'undefined') {
+      document.removeEventListener('visibilitychange', this.handleVisibilityChange);
+    }
     if (this.canvas) {
       this.canvas.remove();
       this.canvas = null;