From 6ec001c1b7f86c98140974b729a1d34ab5f9ed46 Mon Sep 17 00:00:00 2001 From: Brazol Date: Fri, 20 Feb 2026 15:10:43 +0100 Subject: [PATCH] video moderation blur --- packages/stream_video/CHANGELOG.md | 5 ++ packages/stream_video/lib/src/call/call.dart | 83 +++++++++++++++++++ .../state/mixins/state_coordinator_mixin.dart | 22 +++++ packages/stream_video/lib/src/call_state.dart | 9 ++ .../lib/src/models/call_preferences.dart | 12 +++ .../stream_video/lib/src/models/models.dart | 1 + .../src/models/moderation_blur_config.dart | 71 ++++++++++++++++ .../lib/src/retry/retry_manager.dart | 25 +++++- .../StreamVideoFiltersPlugin.kt | 9 ++ .../factories/FullFrameBlurFactory.kt | 30 +++++++ .../FullFrameBlurVideoFrameProcessor.swift | 25 ++++++ .../StreamVideoFiltersPlugin.swift | 9 ++ .../lib/stream_video_filters.dart | 5 ++ .../stream_video_filters_method_channel.dart | 7 ++ ...ream_video_filters_platform_interface.dart | 6 ++ .../lib/video_effects_manager.dart | 31 ++++++- 16 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 packages/stream_video/lib/src/models/moderation_blur_config.dart create mode 100644 packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/factories/FullFrameBlurFactory.kt create mode 100644 packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/FullFrameBlurVideoFrameProcessor.swift diff --git a/packages/stream_video/CHANGELOG.md b/packages/stream_video/CHANGELOG.md index be9741d16..6d1eb63b9 100644 --- a/packages/stream_video/CHANGELOG.md +++ b/packages/stream_video/CHANGELOG.md @@ -1,3 +1,8 @@ +## Upcoming + +### ✅ Added +* Added video moderation support by providing `VideoModerationConfig` in `CallPreferences`. Chech [cookbook](https://getstream.io/video/docs/flutter/ui-cookbook/call-moderation/) for more details. + ## 1.2.4 ### 🐞 Fixed diff --git a/packages/stream_video/lib/src/call/call.dart b/packages/stream_video/lib/src/call/call.dart index 5638d1350..a58060654 100644 --- a/packages/stream_video/lib/src/call/call.dart +++ b/packages/stream_video/lib/src/call/call.dart @@ -296,6 +296,9 @@ class Call { final Map _reactionTimers = {}; final Map _captionsTimers = {}; + Timer? _videoModerationTimer; + void Function()? _onModerationBlurApply; + void Function()? _onModerationBlurClear; final List> _sfuStatsTimers = []; final Set _sfuClientCapabilities = { SfuClientCapability.subscriberVideoPause, // on by default @@ -623,11 +626,77 @@ class Call { event.metadata, updateMembers: false, ); + case StreamCallModerationWarningEvent _: + return _handleModerationWarningEvent(event); + case StreamCallModerationBlurEvent _: + return _handleModerationBlurEvent(event); default: break; } } + void _handleModerationWarningEvent( + StreamCallModerationWarningEvent event, + ) { + final config = state.value.preferences.videoModerationConfig; + if (config.isDisabled || event.userId != _streamVideo.currentUser.id) { + return; + } + + config.onWarning?.call(event.message); + } + + Future _handleModerationBlurEvent( + StreamCallModerationBlurEvent event, + ) async { + final config = state.value.preferences.videoModerationConfig; + if (config.isDisabled) return; + + _stateManager.coordinatorCallModerationBlur(event.userId); + + _videoModerationTimer?.cancel(); + _videoModerationTimer = null; + if (config.duration != null) { + _videoModerationTimer = Timer(config.duration!, clearModerationBlur); + } + + if (config.muteAudio) await setMicrophoneEnabled(enabled: false); + if (config.muteVideo) await setCameraEnabled(enabled: false); + if (config.applyBlur) _onModerationBlurApply?.call(); + config.onApply?.call(); + } + + /// Clears the moderation action, restoring normal operation. + /// + /// When [VideoModerationConfig.muteAudio] / [VideoModerationConfig.muteVideo] + /// were active, re-enabling mic/camera is allowed again but they stay off + /// until the user manually re-enables them. + void clearModerationBlur() { + _videoModerationTimer?.cancel(); + _videoModerationTimer = null; + + if (!state.value.isVideoModerated) return; + + final config = state.value.preferences.videoModerationConfig; + _stateManager.clearModerationBlur(); + + if (config.applyBlur) _onModerationBlurClear?.call(); + config.onClear?.call(); + } + + /// Registers handlers for the native blur effect pipeline. + /// + /// Called automatically by `StreamVideoEffectsManager` from + /// `stream_video_filters` when [VideoModerationConfig.applyBlur] is true. + @internal + void setModerationBlurEffectHandlers({ + required void Function() onApply, + required void Function() onClear, + }) { + _onModerationBlurApply = onApply; + _onModerationBlurClear = onClear; + } + @internal void traceSessionLog(String tag, dynamic data) { _session?.trace(tag, data); @@ -1851,6 +1920,8 @@ class Call { ]) { timer.cancel(); } + _videoModerationTimer?.cancel(); + _videoModerationTimer = null; for (final operation in _sfuStatsTimers) { await operation.cancel(); @@ -2930,6 +3001,12 @@ class Call { required bool enabled, CameraConstraints? constraints, }) async { + if (enabled && + state.value.isVideoModerated && + state.value.preferences.videoModerationConfig.muteVideo) { + _logger.w(() => '[setCameraEnabled] blocked by video moderation'); + return Result.error('Blocked by video moderation'); + } if (enabled && !hasPermission(CallPermission.sendVideo)) { return Result.error('Missing permission to send video'); } @@ -3006,6 +3083,12 @@ class Call { required bool enabled, AudioConstraints? constraints, }) async { + if (enabled && + state.value.isVideoModerated && + state.value.preferences.videoModerationConfig.muteAudio) { + _logger.w(() => '[setMicrophoneEnabled] blocked by video moderation'); + return Result.error('Blocked by video moderation'); + } if (enabled && !hasPermission(CallPermission.sendAudio)) { return Result.error('Missing permission to send audio'); } diff --git a/packages/stream_video/lib/src/call/state/mixins/state_coordinator_mixin.dart b/packages/stream_video/lib/src/call/state/mixins/state_coordinator_mixin.dart index ac4c339c2..cc5f14d75 100644 --- a/packages/stream_video/lib/src/call/state/mixins/state_coordinator_mixin.dart +++ b/packages/stream_video/lib/src/call/state/mixins/state_coordinator_mixin.dart @@ -518,4 +518,26 @@ mixin StateCoordinatorMixin on StateNotifier { .toList(), ); } + + void coordinatorCallModerationBlur( + String userId, + ) { + if (userId != state.currentUserId) { + _logger.i( + () => '[coordinatorCallModeration] rejected (not current user)', + ); + return; + } + + state = state.copyWith( + isVideoModerated: true, + ); + } + + void clearModerationBlur() { + _logger.i(() => '[clearModerationBlur]'); + state = state.copyWith( + isVideoModerated: false, + ); + } } diff --git a/packages/stream_video/lib/src/call_state.dart b/packages/stream_video/lib/src/call_state.dart index e608e54d8..827b334f1 100644 --- a/packages/stream_video/lib/src/call_state.dart +++ b/packages/stream_video/lib/src/call_state.dart @@ -52,6 +52,7 @@ class CallState extends Equatable { anonymousParticipantCount: 0, iOSMultitaskingCameraAccessEnabled: false, custom: const {}, + isVideoModerated: false, ); } @@ -92,6 +93,7 @@ class CallState extends Equatable { required this.anonymousParticipantCount, required this.iOSMultitaskingCameraAccessEnabled, required this.custom, + required this.isVideoModerated, }); final CallPreferences preferences; @@ -131,6 +133,9 @@ class CallState extends Equatable { final bool iOSMultitaskingCameraAccessEnabled; final Map custom; + /// Whether the local user's video is currently blurred by moderation. + final bool isVideoModerated; + String get callId => callCid.id; StreamCallType get callType => callCid.type; @@ -201,6 +206,7 @@ class CallState extends Equatable { int? anonymousParticipantCount, bool? iOSMultitaskingCameraAccessEnabled, Map? custom, + bool? isVideoModerated, }) { return CallState._( preferences: preferences ?? this.preferences, @@ -242,6 +248,7 @@ class CallState extends Equatable { iOSMultitaskingCameraAccessEnabled ?? this.iOSMultitaskingCameraAccessEnabled, custom: custom ?? this.custom, + isVideoModerated: isVideoModerated ?? this.isVideoModerated, ); } @@ -314,6 +321,7 @@ class CallState extends Equatable { anonymousParticipantCount, iOSMultitaskingCameraAccessEnabled, custom, + isVideoModerated, ]; @override @@ -321,6 +329,7 @@ class CallState extends Equatable { return 'CallState(status: $status, currentUserId: $currentUserId,' ' callCid: $callCid, createdByUser: $createdByUser,' ' sessionId: $sessionId, isRecording: $isRecording,' + ' isVideoModerated: $isVideoModerated,' ' settings: $settings, egress: $egress, ' ' videoInputDevice: $videoInputDevice,' ' audioInputDevice: $audioInputDevice,' diff --git a/packages/stream_video/lib/src/models/call_preferences.dart b/packages/stream_video/lib/src/models/call_preferences.dart index ab3e8cd53..63fef252c 100644 --- a/packages/stream_video/lib/src/models/call_preferences.dart +++ b/packages/stream_video/lib/src/models/call_preferences.dart @@ -1,4 +1,5 @@ import 'call_client_publish_options.dart'; +import 'moderation_blur_config.dart'; abstract class CallPreferences { /// The maximum duration to wait when establishing a connection to the call. @@ -49,6 +50,10 @@ abstract class CallPreferences { /// The maximum number of closed caption lines that can be visible /// simultaneously on screen. int get closedCaptionsVisibleCaptions; + + /// Configuration for how the SDK handles call moderation events. + /// Defaults to [VideoModerationConfig.disabled]. + VideoModerationConfig get videoModerationConfig; } class DefaultCallPreferences implements CallPreferences { @@ -62,6 +67,7 @@ class DefaultCallPreferences implements CallPreferences { this.clientPublishOptions, this.closedCaptionsVisibilityDurationMs = 2700, this.closedCaptionsVisibleCaptions = 2, + this.videoModerationConfig = const VideoModerationConfig.disabled(), }); /// The maximum duration to wait when establishing a connection to the call. @@ -137,4 +143,10 @@ class DefaultCallPreferences implements CallPreferences { /// Defaults to 2 lines. @override final int closedCaptionsVisibleCaptions; + + /// Configuration for how the SDK handles call moderation events. + /// + /// Defaults to [VideoModerationConfig.disabled]. + @override + final VideoModerationConfig videoModerationConfig; } diff --git a/packages/stream_video/lib/src/models/models.dart b/packages/stream_video/lib/src/models/models.dart index 7147adddd..baed81526 100644 --- a/packages/stream_video/lib/src/models/models.dart +++ b/packages/stream_video/lib/src/models/models.dart @@ -19,6 +19,7 @@ export 'call_status.dart'; export 'call_track_state.dart'; export 'disconnect_reason.dart'; export 'guest_created_data.dart'; +export 'moderation_blur_config.dart'; export 'push_device.dart'; export 'push_provider.dart'; export 'queried_calls.dart'; diff --git a/packages/stream_video/lib/src/models/moderation_blur_config.dart b/packages/stream_video/lib/src/models/moderation_blur_config.dart new file mode 100644 index 000000000..d41f8dda0 --- /dev/null +++ b/packages/stream_video/lib/src/models/moderation_blur_config.dart @@ -0,0 +1,71 @@ +import 'dart:ui'; + +/// Configures how the SDK handles `call.moderation` events. +/// +/// Convenience constructors [VideoModerationConfig.disabled], +/// [VideoModerationConfig.mute], and [VideoModerationConfig.blur] cover +/// the most common presets. +class VideoModerationConfig { + const VideoModerationConfig({ + this.muteAudio = false, + this.muteVideo = false, + this.applyBlur = false, + this.duration, + this.onApply, + this.onWarning, + this.onClear, + }); + + /// No automatic action. The event is still emitted on the call's event + /// stream for manual handling via `call.callEvents`. + const VideoModerationConfig.disabled() + : muteAudio = false, + muteVideo = false, + applyBlur = false, + duration = null, + onApply = null, + onWarning = null, + onClear = null; + + /// Mutes the local user's microphone and camera, and prevents re-enabling + /// them for the configured [duration]. If [duration] is null, the mute + /// persists until `call.clearModerationBlur()` is called. + const VideoModerationConfig.mute({this.duration}) + : muteAudio = true, + muteVideo = true, + applyBlur = false, + onApply = null, + onWarning = null, + onClear = null; + + /// Applies a full-frame native blur on the camera track via + /// `StreamVideoEffectsManager` from the `stream_video_filters` package. + /// The blur is visible to ALL participants because it modifies frames + /// before encoding. Requires the `stream_video_filters` package. + const VideoModerationConfig.blur({this.duration}) + : muteAudio = false, + muteVideo = false, + applyBlur = true, + onApply = null, + onWarning = null, + onClear = null; + + final bool muteAudio; + final bool muteVideo; + final bool applyBlur; + + final Duration? duration; + + final VoidCallback? onApply; + final void Function(String)? onWarning; + final VoidCallback? onClear; + + /// Whether this config takes no automatic action. + bool get isDisabled => + !muteAudio && + !muteVideo && + !applyBlur && + onApply == null && + onWarning == null && + onClear == null; +} diff --git a/packages/stream_video/lib/src/retry/retry_manager.dart b/packages/stream_video/lib/src/retry/retry_manager.dart index 1e269adfb..f903d53d3 100644 --- a/packages/stream_video/lib/src/retry/retry_manager.dart +++ b/packages/stream_video/lib/src/retry/retry_manager.dart @@ -1,3 +1,4 @@ +import '../../open_api/video/coordinator/api.dart'; import '../errors/video_error.dart'; import '../utils/result.dart'; import 'retry_policy.dart'; @@ -30,8 +31,30 @@ class RpcRetryManager { delegate, ); retryAttempt++; - } while (result.isFailure && retryAttempt < policy.config.rpcMaxRetries); + } while (result.isFailure && + retryAttempt < policy.config.rpcMaxRetries && + _isRetryable(result)); return result; } + + /// Returns false for permanent client errors (4xx except 408/429) + /// that should not be retried. + bool _isRetryable(Result result) { + if (result is! Failure) return true; + + final error = result.error; + if (error is! VideoErrorWithCause) return true; + + final cause = error.cause; + if (cause is! ApiException) return true; + + final statusCode = cause.code; + if (statusCode >= 400 && statusCode < 500) { + // 408 Request Timeout and 429 Too Many Requests are retryable + return statusCode == 408 || statusCode == 429; + } + + return true; + } } diff --git a/packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/StreamVideoFiltersPlugin.kt b/packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/StreamVideoFiltersPlugin.kt index d3bafa02f..d3e7f6ae6 100644 --- a/packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/StreamVideoFiltersPlugin.kt +++ b/packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/StreamVideoFiltersPlugin.kt @@ -11,6 +11,7 @@ import io.flutter.plugin.common.MethodChannel.Result import io.getstream.webrtc.flutter.videoEffects.ProcessorProvider import io.getstream.video.flutter.stream_video_filters.factories.BackgroundBlurFactory import io.getstream.video.flutter.stream_video_filters.factories.BlurIntensity +import io.getstream.video.flutter.stream_video_filters.factories.FullFrameBlurFactory import io.getstream.video.flutter.stream_video_filters.factories.VirtualBackgroundFactory /** @@ -63,6 +64,14 @@ class StreamVideoFiltersPlugin: FlutterPlugin, MethodCallHandler { ) } + result.success(null) + } + "registerFullFrameBlurEffectProcessor" -> { + ProcessorProvider.addProcessor( + "FullFrameBlur", + FullFrameBlurFactory() + ) + result.success(null) } else -> { diff --git a/packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/factories/FullFrameBlurFactory.kt b/packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/factories/FullFrameBlurFactory.kt new file mode 100644 index 000000000..2a75d7482 --- /dev/null +++ b/packages/stream_video_filters/android/src/main/kotlin/io/getstream/video/flutter/stream_video_filters/factories/FullFrameBlurFactory.kt @@ -0,0 +1,30 @@ +package io.getstream.video.flutter.stream_video_filters.factories + +import android.graphics.Bitmap +import android.graphics.Canvas +import com.google.android.renderscript.Toolkit +import io.getstream.webrtc.flutter.videoEffects.VideoFrameProcessor +import io.getstream.webrtc.flutter.videoEffects.VideoFrameProcessorFactoryInterface +import io.getstream.video.flutter.stream_video_filters.common.BitmapVideoFilter +import io.getstream.video.flutter.stream_video_filters.common.VideoFrameProcessorWithBitmapFilter + +class FullFrameBlurFactory : VideoFrameProcessorFactoryInterface { + override fun build(): VideoFrameProcessor { + return VideoFrameProcessorWithBitmapFilter { + FullFrameBlurFilter() + } + } +} + +private class FullFrameBlurFilter : BitmapVideoFilter() { + override fun applyFilter(videoFrameBitmap: Bitmap) { + val blurredBitmap = Toolkit.blur(videoFrameBitmap, BLUR_RADIUS) + val canvas = Canvas(videoFrameBitmap) + canvas.drawBitmap(blurredBitmap, 0f, 0f, null) + blurredBitmap.recycle() + } + + companion object { + private const val BLUR_RADIUS = 50 + } +} diff --git a/packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/FullFrameBlurVideoFrameProcessor.swift b/packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/FullFrameBlurVideoFrameProcessor.swift new file mode 100644 index 000000000..6692560da --- /dev/null +++ b/packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/FullFrameBlurVideoFrameProcessor.swift @@ -0,0 +1,25 @@ +import Foundation +import stream_webrtc_flutter + +@available(iOS 15.0, *) +final class FullFrameBlurVideoFrameProcessor: VideoFilter { + + @available(*, unavailable) + override public init( + filter: @escaping (Input) -> CIImage + ) { fatalError() } + + init() { + super.init( + filter: { input in input.originalImage } + ) + + self.filter = { input in + let blurred = input.originalImage.applyingFilter( + "CIGaussianBlur", + parameters: [kCIInputRadiusKey: 50.0] + ) + return blurred.cropped(to: input.originalImage.extent) + } + } +} diff --git a/packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/StreamVideoFiltersPlugin.swift b/packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/StreamVideoFiltersPlugin.swift index 1272a1b27..ccbfab7ed 100644 --- a/packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/StreamVideoFiltersPlugin.swift +++ b/packages/stream_video_filters/ios/stream_video_filters/Sources/stream_video_filters/StreamVideoFiltersPlugin.swift @@ -60,6 +60,15 @@ public class StreamVideoFiltersPlugin: NSObject, FlutterPlugin { forName: "VirtualBackground-\(backgroundImageUrl)" ) + result(nil) + case "registerFullFrameBlurEffectProcessor": + if #available(iOS 15.0, *) { + ProcessorProvider.addProcessor( + FullFrameBlurVideoFrameProcessor(), + forName: "FullFrameBlur") + } else { + print("Full frame blur effect is not supported on iOS versions earlier than 15.0") + } result(nil) default: result(FlutterMethodNotImplemented) diff --git a/packages/stream_video_filters/lib/stream_video_filters.dart b/packages/stream_video_filters/lib/stream_video_filters.dart index 710bdf501..ed2c14364 100644 --- a/packages/stream_video_filters/lib/stream_video_filters.dart +++ b/packages/stream_video_filters/lib/stream_video_filters.dart @@ -19,4 +19,9 @@ class StreamVideoFilters { backgroundImageUrl: backgroundImageUrl, ); } + + Future registerFullFrameBlurEffectProcessor() { + return StreamVideoFiltersPlatform.instance + .registerFullFrameBlurEffectProcessor(); + } } diff --git a/packages/stream_video_filters/lib/stream_video_filters_method_channel.dart b/packages/stream_video_filters/lib/stream_video_filters_method_channel.dart index 2f45ee5f9..574d38504 100644 --- a/packages/stream_video_filters/lib/stream_video_filters_method_channel.dart +++ b/packages/stream_video_filters/lib/stream_video_filters_method_channel.dart @@ -31,4 +31,11 @@ class MethodChannelStreamVideoFilters extends StreamVideoFiltersPlatform { 'backgroundImageUrl': backgroundImageUrl, }); } + + @override + Future registerFullFrameBlurEffectProcessor() { + return methodChannel.invokeMethod( + 'registerFullFrameBlurEffectProcessor', + ); + } } diff --git a/packages/stream_video_filters/lib/stream_video_filters_platform_interface.dart b/packages/stream_video_filters/lib/stream_video_filters_platform_interface.dart index 356e30a22..ca1880e5b 100644 --- a/packages/stream_video_filters/lib/stream_video_filters_platform_interface.dart +++ b/packages/stream_video_filters/lib/stream_video_filters_platform_interface.dart @@ -43,4 +43,10 @@ abstract class StreamVideoFiltersPlatform extends PlatformInterface { 'registerImageEffectProcessors has not been implemented.', ); } + + Future registerFullFrameBlurEffectProcessor() { + throw UnimplementedError( + 'registerFullFrameBlurEffectProcessor has not been implemented.', + ); + } } diff --git a/packages/stream_video_filters/lib/video_effects_manager.dart b/packages/stream_video_filters/lib/video_effects_manager.dart index a77279f08..f34cda25c 100644 --- a/packages/stream_video_filters/lib/video_effects_manager.dart +++ b/packages/stream_video_filters/lib/video_effects_manager.dart @@ -17,9 +17,18 @@ enum BlurIntensity { } class StreamVideoEffectsManager { - StreamVideoEffectsManager(this.call); + StreamVideoEffectsManager(this.call) { + if (call.state.value.preferences.videoModerationConfig.applyBlur) { + // ignore: invalid_use_of_internal_member + call.setModerationBlurEffectHandlers( + onApply: applyFullFrameBlurFilter, + onClear: disableAllFilters, + ); + } + } static bool isBlurRegistered = false; + static bool isFullFrameBlurRegistered = false; static Map isImageRegistered = {}; static Map isCustomEffectRegistered = {}; @@ -55,6 +64,18 @@ class StreamVideoEffectsManager { await applyVideoEffects([blurIntensity.name], track: track); } + /// Applies a full-frame blur filter to the local participant video stream. + Future applyFullFrameBlurFilter({ + RtcLocalTrack? track, + }) async { + if (!(await isSupported())) { + return; + } + + await ensureFullFrameBlurEffectRegistered(); + await applyVideoEffects(['FullFrameBlur'], track: track); + } + /// Applies a background image filter to the local participant video stream. /// The [imageUrl] parameter specifies the path to the image asset file or an URL to the image. Future applyBackgroundImageFilter( @@ -137,6 +158,14 @@ class StreamVideoEffectsManager { } } + /// Ensures that the full-frame blur effect processor is registered. + Future ensureFullFrameBlurEffectRegistered() async { + if (!isFullFrameBlurRegistered) { + await StreamVideoFilters().registerFullFrameBlurEffectProcessor(); + isFullFrameBlurRegistered = true; + } + } + /// Ensures that the image effect processor is registered. Future ensureImageEffectRegistered(String imageUrl) async { if (!isImageRegistered.containsKey(imageUrl)) {