From 1def3a88483e4e3996cdeeddfb74814a9554dfb0 Mon Sep 17 00:00:00 2001 From: Emmanuel Atawodi Date: Thu, 25 Sep 2025 10:18:57 +0100 Subject: [PATCH 1/2] feat: ingest backpressure + per-track stats --- .gitignore | 1 + example/index.js | 9 +- example/ios/Podfile.lock | 10 + example/ios/VisionRtcExample/Info.plist | 4 + example/package.json | 3 +- example/src/App.tsx | 182 ++++++++++++--- ios/VisionRTC+Spec.mm | 60 ++++- ios/VisionRTCModule.swift | 280 ++++++++++++++++++++++-- ios/VisionRTCViewManager.m | 6 + src/NativeVisionRtc.ts | 15 +- src/index.ts | 98 ++++++++- src/types.ts | 31 +++ src/vision-rtc-view.tsx | 30 ++- yarn.lock | 21 ++ 14 files changed, 688 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index 67f3212..54e7229 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ # VSCode .vscode/ +.cursor/ jsconfig.json # Xcode diff --git a/example/index.js b/example/index.js index ef707c2..c0b5867 100644 --- a/example/index.js +++ b/example/index.js @@ -1,5 +1,12 @@ import {AppRegistry} from 'react-native'; +import {SafeAreaProvider} from 'react-native-safe-area-context'; import App from './src/App'; import {name as appName} from './app.json'; -AppRegistry.registerComponent(appName, () => App); +const Root = () => ( + + + +); + +AppRegistry.registerComponent(appName, () => Root); diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index c2eaae7..d23365c 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -2341,6 +2341,12 @@ PODS: - React-utils (= 0.81.1) - SocketRocket - SocketRocket (0.7.1) + - VisionCamera (4.7.2): + - VisionCamera/Core (= 4.7.2) + - VisionCamera/React (= 4.7.2) + - VisionCamera/Core (4.7.2) + - VisionCamera/React (4.7.2): + - React-Core - VisionRtc (0.1.0): - boost - DoubleConversion @@ -2447,6 +2453,7 @@ DEPENDENCIES: - ReactCodegen (from `build/generated/ios`) - ReactCommon/turbomodule/core (from `../node_modules/react-native/ReactCommon`) - SocketRocket (~> 0.7.1) + - VisionCamera (from `../node_modules/react-native-vision-camera`) - VisionRtc (from `../..`) - WebRTC-SDK - Yoga (from `../node_modules/react-native/ReactCommon/yoga`) @@ -2600,6 +2607,8 @@ EXTERNAL SOURCES: :path: build/generated/ios ReactCommon: :path: "../node_modules/react-native/ReactCommon" + VisionCamera: + :path: "../node_modules/react-native-vision-camera" VisionRtc: :path: "../.." Yoga: @@ -2678,6 +2687,7 @@ SPEC CHECKSUMS: ReactCodegen: 4d203eddf6f977caa324640a20f92e70408d648b ReactCommon: ce5d4226dfaf9d5dacbef57b4528819e39d3a120 SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 + VisionCamera: 30b358b807324c692064f78385e9a732ce1bebfe VisionRtc: 56e770d48b49da73a130f18a3ca12148a12126e8 WebRTC-SDK: 69d4e56b0b4b27d788e87bab9b9a1326ed05b1e3 Yoga: 11c9686a21e2cd82a094a723649d9f4507200fb0 diff --git a/example/ios/VisionRtcExample/Info.plist b/example/ios/VisionRtcExample/Info.plist index ee0206d..410226d 100644 --- a/example/ios/VisionRtcExample/Info.plist +++ b/example/ios/VisionRtcExample/Info.plist @@ -49,5 +49,9 @@ UIViewControllerBasedStatusBarAppearance + NSCameraUsageDescription + This app requires camera access for live video. + NSMicrophoneUsageDescription + This app may use the microphone for audio capture. diff --git a/example/package.json b/example/package.json index f9dd9c7..cf0cb39 100644 --- a/example/package.json +++ b/example/package.json @@ -13,7 +13,8 @@ "@react-native/new-app-screen": "0.81.1", "react": "19.1.0", "react-native": "0.81.1", - "react-native-safe-area-context": "^5.6.1" + "react-native-safe-area-context": "^5.6.1", + "react-native-vision-camera": "^4.7.2" }, "devDependencies": { "@babel/core": "^7.25.2", diff --git a/example/src/App.tsx b/example/src/App.tsx index ad4571f..2306674 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -1,35 +1,64 @@ import * as React from 'react'; -import {Button, Text, StyleSheet, View} from 'react-native'; -import {SafeAreaView} from 'react-native-safe-area-context'; +import {Button, Text, StyleSheet, View, findNodeHandle} from 'react-native'; +import {SafeAreaView, useSafeAreaInsets} from 'react-native-safe-area-context'; +import {Camera, useCameraDevice} from 'react-native-vision-camera'; import { + createVisionCameraSource, createWebRTCTrack, + updateSource, + updateTrack, + disposeSource, disposeTrack, getStats, VisionRTCView, } from 'react-native-vision-rtc'; export default function App() { + const insets = useSafeAreaInsets(); + const cameraRef = React.useRef(null); + const device = useCameraDevice('back'); const [trackId, setTrackId] = React.useState(null); + const [sourceId, setSourceId] = React.useState(null); const [stats, setStats] = React.useState<{ - fps: number; + producedFps: number; + deliveredFps: number; droppedFrames: number; } | null>(null); const [creating, setCreating] = React.useState(false); + const [torch, setTorch] = React.useState(false); + const [facing, setFacing] = React.useState<'front' | 'back'>('back'); + const [backpressure, setBackpressure] = React.useState< + 'drop-late' | 'latest-wins' | 'throttle' + >('drop-late'); + + const ensurePermissions = React.useCallback(async () => { + const cam = await Camera.getCameraPermissionStatus(); + if (cam !== 'granted') { + const res = await Camera.requestCameraPermission(); + if (res !== 'granted') throw new Error('Camera permission not granted'); + } + }, []); const onStart = async () => { if (creating || trackId) return; setCreating(true); let newId: string | null = null; try { - const {trackId: id} = await createWebRTCTrack( - {__nativeSourceId: 'null'}, + await ensurePermissions(); + const node = findNodeHandle(cameraRef.current); + if (!node) throw new Error('Camera view not ready'); + const {__nativeSourceId} = await createVisionCameraSource(node); + setSourceId(__nativeSourceId); + const created = await createWebRTCTrack( + {__nativeSourceId}, { fps: 30, resolution: {width: 1280, height: 720}, + backpressure, } ); - newId = id; - setTrackId(id); + newId = created.trackId; + setTrackId(created.trackId); } catch (err) { if (newId) { try { @@ -43,13 +72,15 @@ export default function App() { }; const onStop = async () => { - if (!trackId) return; + if (!trackId && !sourceId) return; try { - await disposeTrack(trackId); + if (trackId) await disposeTrack(trackId); + if (sourceId) await disposeSource(sourceId); } catch (e) { console.warn('Failed to dispose track', e); } finally { setTrackId(null); + setSourceId(null); setStats(null); } }; @@ -62,26 +93,112 @@ export default function App() { }; }, [trackId]); - const onGetStats = async () => { - const s = await getStats(); - if (s) setStats({fps: s.fps, droppedFrames: s.droppedFrames}); + const onGetStats = React.useCallback(async () => { + const s = await getStats(trackId ?? undefined); + if (s) + setStats({ + producedFps: s.producedFps, + deliveredFps: s.deliveredFps, + droppedFrames: s.droppedFrames, + }); + }, [trackId]); + + React.useEffect(() => { + if (!trackId) return; + const t = setInterval(onGetStats, 1000); + return () => clearInterval(t); + }, [trackId, onGetStats]); + + const onFlip = async () => { + if (!sourceId) return; + const next = facing === 'back' ? 'front' : 'back'; + setFacing(next); + // Turning torch off when flipping avoids devices without torch (e.g., front) + setTorch(false); + await updateSource(sourceId, {position: next}); + }; + + const onTorch = async () => { + if (!sourceId) return; + const next = !torch; + setTorch(next); + await updateSource(sourceId, {torch: next}); + }; + + const onFps = async (fps: number) => { + if (!trackId) return; + console.log('onFps', fps); + await updateTrack(trackId, {fps}); + }; + + const onToggleBackpressure = async () => { + const order: Array<'drop-late' | 'latest-wins' | 'throttle'> = [ + 'drop-late', + 'latest-wins', + 'throttle', + ]; + const currentIndex = order.indexOf(backpressure); + const idx = currentIndex === -1 ? 0 : (currentIndex + 1) % order.length; + const next = order[idx] as 'drop-late' | 'latest-wins' | 'throttle'; + setBackpressure(next); + if (trackId) { + await updateTrack(trackId, {backpressure: next}); + } }; return ( - -