diff --git a/.gitignore b/.gitignore
index 67f3212..54e7229 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@
# VSCode
.vscode/
+.cursor/
jsconfig.json
# Xcode
diff --git a/example/index.js b/example/index.js
index ef707c2..c0b5867 100644
--- a/example/index.js
+++ b/example/index.js
@@ -1,5 +1,12 @@
import {AppRegistry} from 'react-native';
+import {SafeAreaProvider} from 'react-native-safe-area-context';
import App from './src/App';
import {name as appName} from './app.json';
-AppRegistry.registerComponent(appName, () => App);
+const Root = () => (
+
+
+
+);
+
+AppRegistry.registerComponent(appName, () => Root);
diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock
index c2eaae7..d23365c 100644
--- a/example/ios/Podfile.lock
+++ b/example/ios/Podfile.lock
@@ -2341,6 +2341,12 @@ PODS:
- React-utils (= 0.81.1)
- SocketRocket
- SocketRocket (0.7.1)
+ - VisionCamera (4.7.2):
+ - VisionCamera/Core (= 4.7.2)
+ - VisionCamera/React (= 4.7.2)
+ - VisionCamera/Core (4.7.2)
+ - VisionCamera/React (4.7.2):
+ - React-Core
- VisionRtc (0.1.0):
- boost
- DoubleConversion
@@ -2447,6 +2453,7 @@ DEPENDENCIES:
- ReactCodegen (from `build/generated/ios`)
- ReactCommon/turbomodule/core (from `../node_modules/react-native/ReactCommon`)
- SocketRocket (~> 0.7.1)
+ - VisionCamera (from `../node_modules/react-native-vision-camera`)
- VisionRtc (from `../..`)
- WebRTC-SDK
- Yoga (from `../node_modules/react-native/ReactCommon/yoga`)
@@ -2600,6 +2607,8 @@ EXTERNAL SOURCES:
:path: build/generated/ios
ReactCommon:
:path: "../node_modules/react-native/ReactCommon"
+ VisionCamera:
+ :path: "../node_modules/react-native-vision-camera"
VisionRtc:
:path: "../.."
Yoga:
@@ -2678,6 +2687,7 @@ SPEC CHECKSUMS:
ReactCodegen: 4d203eddf6f977caa324640a20f92e70408d648b
ReactCommon: ce5d4226dfaf9d5dacbef57b4528819e39d3a120
SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748
+ VisionCamera: 30b358b807324c692064f78385e9a732ce1bebfe
VisionRtc: 56e770d48b49da73a130f18a3ca12148a12126e8
WebRTC-SDK: 69d4e56b0b4b27d788e87bab9b9a1326ed05b1e3
Yoga: 11c9686a21e2cd82a094a723649d9f4507200fb0
diff --git a/example/ios/VisionRtcExample.xcodeproj/project.pbxproj b/example/ios/VisionRtcExample.xcodeproj/project.pbxproj
index 57c4f0a..7846993 100644
--- a/example/ios/VisionRtcExample.xcodeproj/project.pbxproj
+++ b/example/ios/VisionRtcExample.xcodeproj/project.pbxproj
@@ -257,18 +257,19 @@
isa = XCBuildConfiguration;
baseConfigurationReference = 3B4392A12AC88292D35C810B /* Pods-VisionRtcExample.debug.xcconfig */;
buildSettings = {
- ENABLE_PREVIEWS = NO;
- MACH_O_TYPE = mh_execute;
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CURRENT_PROJECT_VERSION = 1;
+ DEVELOPMENT_TEAM = 9W7U4QA43U;
ENABLE_BITCODE = NO;
+ ENABLE_PREVIEWS = NO;
INFOPLIST_FILE = VisionRtcExample/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
+ MACH_O_TYPE = mh_execute;
MARKETING_VERSION = 1.0;
OTHER_LDFLAGS = (
"$(inherited)",
@@ -287,17 +288,18 @@
isa = XCBuildConfiguration;
baseConfigurationReference = 5709B34CF0A7D63546082F79 /* Pods-VisionRtcExample.release.xcconfig */;
buildSettings = {
- ENABLE_PREVIEWS = NO;
- MACH_O_TYPE = mh_execute;
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CURRENT_PROJECT_VERSION = 1;
+ DEVELOPMENT_TEAM = 9W7U4QA43U;
+ ENABLE_PREVIEWS = NO;
INFOPLIST_FILE = VisionRtcExample/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
+ MACH_O_TYPE = mh_execute;
MARKETING_VERSION = 1.0;
OTHER_LDFLAGS = (
"$(inherited)",
diff --git a/example/ios/VisionRtcExample/Info.plist b/example/ios/VisionRtcExample/Info.plist
index ee0206d..410226d 100644
--- a/example/ios/VisionRtcExample/Info.plist
+++ b/example/ios/VisionRtcExample/Info.plist
@@ -49,5 +49,9 @@
UIViewControllerBasedStatusBarAppearance
+ NSCameraUsageDescription
+ This app requires camera access for live video.
+ NSMicrophoneUsageDescription
+ This app may use the microphone for audio capture.
diff --git a/example/package.json b/example/package.json
index f9dd9c7..cf0cb39 100644
--- a/example/package.json
+++ b/example/package.json
@@ -13,7 +13,8 @@
"@react-native/new-app-screen": "0.81.1",
"react": "19.1.0",
"react-native": "0.81.1",
- "react-native-safe-area-context": "^5.6.1"
+ "react-native-safe-area-context": "^5.6.1",
+ "react-native-vision-camera": "^4.7.2"
},
"devDependencies": {
"@babel/core": "^7.25.2",
diff --git a/example/src/App.tsx b/example/src/App.tsx
index ad4571f..b5d04e8 100644
--- a/example/src/App.tsx
+++ b/example/src/App.tsx
@@ -1,35 +1,66 @@
import * as React from 'react';
-import {Button, Text, StyleSheet, View} from 'react-native';
-import {SafeAreaView} from 'react-native-safe-area-context';
+import {Button, Text, StyleSheet, View, findNodeHandle} from 'react-native';
+import {SafeAreaView, useSafeAreaInsets} from 'react-native-safe-area-context';
+import {Camera, useCameraDevice} from 'react-native-vision-camera';
import {
+ createVisionCameraSource,
createWebRTCTrack,
+ updateSource,
+ updateTrack,
+ disposeSource,
disposeTrack,
getStats,
VisionRTCView,
} from 'react-native-vision-rtc';
+import TestFramework from './test-framework';
export default function App() {
+ const insets = useSafeAreaInsets();
+ const cameraRef = React.useRef(null);
+ const device = useCameraDevice('back');
const [trackId, setTrackId] = React.useState(null);
+ const [showTests, setShowTests] = React.useState(false);
+ const [sourceId, setSourceId] = React.useState(null);
const [stats, setStats] = React.useState<{
- fps: number;
+ producedFps: number;
+ deliveredFps: number;
droppedFrames: number;
} | null>(null);
const [creating, setCreating] = React.useState(false);
+ const [torch, setTorch] = React.useState(false);
+ const [facing, setFacing] = React.useState<'front' | 'back'>('back');
+ const [backpressure, setBackpressure] = React.useState<
+ 'drop-late' | 'latest-wins' | 'throttle'
+ >('drop-late');
+
+ const ensurePermissions = React.useCallback(async () => {
+ const cam = await Camera.getCameraPermissionStatus();
+ if (cam !== 'granted') {
+ const res = await Camera.requestCameraPermission();
+ if (res !== 'granted') throw new Error('Camera permission not granted');
+ }
+ }, []);
const onStart = async () => {
if (creating || trackId) return;
setCreating(true);
let newId: string | null = null;
try {
- const {trackId: id} = await createWebRTCTrack(
- {__nativeSourceId: 'null'},
+ await ensurePermissions();
+ const node = findNodeHandle(cameraRef.current);
+ if (!node) throw new Error('Camera view not ready');
+ const {__nativeSourceId} = await createVisionCameraSource(node);
+ setSourceId(__nativeSourceId);
+ const created = await createWebRTCTrack(
+ {__nativeSourceId},
{
fps: 30,
resolution: {width: 1280, height: 720},
+ backpressure,
}
);
- newId = id;
- setTrackId(id);
+ newId = created.trackId;
+ setTrackId(created.trackId);
} catch (err) {
if (newId) {
try {
@@ -43,13 +74,15 @@ export default function App() {
};
const onStop = async () => {
- if (!trackId) return;
+ if (!trackId && !sourceId) return;
try {
- await disposeTrack(trackId);
+ if (trackId) await disposeTrack(trackId);
+ if (sourceId) await disposeSource(sourceId);
} catch (e) {
console.warn('Failed to dispose track', e);
} finally {
setTrackId(null);
+ setSourceId(null);
setStats(null);
}
};
@@ -62,26 +95,119 @@ export default function App() {
};
}, [trackId]);
- const onGetStats = async () => {
- const s = await getStats();
- if (s) setStats({fps: s.fps, droppedFrames: s.droppedFrames});
+ const onGetStats = React.useCallback(async () => {
+ const s = await getStats(trackId ?? undefined);
+ if (s)
+ setStats({
+ producedFps: s.producedFps,
+ deliveredFps: s.deliveredFps,
+ droppedFrames: s.droppedFrames,
+ });
+ }, [trackId]);
+
+ React.useEffect(() => {
+ if (!trackId) return;
+ const t = setInterval(onGetStats, 1000);
+ return () => clearInterval(t);
+ }, [trackId, onGetStats]);
+
+ const onFlip = async () => {
+ if (!sourceId) return;
+ const next = facing === 'back' ? 'front' : 'back';
+ setFacing(next);
+ // Turning torch off when flipping avoids devices without torch (e.g., front)
+ setTorch(false);
+ await updateSource(sourceId, {position: next});
+ };
+
+ const onTorch = async () => {
+ if (!sourceId) return;
+ const next = !torch;
+ setTorch(next);
+ await updateSource(sourceId, {torch: next});
};
+ const onFps = async (fps: number) => {
+ if (!trackId) return;
+ console.log('onFps', fps);
+ await updateTrack(trackId, {fps});
+ };
+
+ const onToggleBackpressure = async () => {
+ const order: Array<'drop-late' | 'latest-wins' | 'throttle'> = [
+ 'drop-late',
+ 'latest-wins',
+ 'throttle',
+ ];
+ const currentIndex = order.indexOf(backpressure);
+ const idx = currentIndex === -1 ? 0 : (currentIndex + 1) % order.length;
+ const next = order[idx] as 'drop-late' | 'latest-wins' | 'throttle';
+ setBackpressure(next);
+ if (trackId) {
+ await updateTrack(trackId, {backpressure: next});
+ }
+ };
+
+ if (showTests) {
+ return setShowTests(false)} />;
+ }
+
return (
-
-
-
-
-
-
-
+ {device && (
+
+ )}
+
+
+ {trackId ?? 'No track id'}
+
+
+
- {`fps: ${stats?.fps ?? 0} drops: ${stats?.droppedFrames ?? 0}`}
+ {`prod: ${stats?.producedFps ?? 0} deliv: ${stats?.deliveredFps ?? 0} drops: ${stats?.droppedFrames ?? 0}`}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
);
}
@@ -90,23 +216,31 @@ const styles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: '#ffff',
+ justifyContent: 'center',
+ alignItems: 'center',
},
controls: {
padding: 12,
flexDirection: 'row',
justifyContent: 'space-around',
alignItems: 'center',
+ flexWrap: 'wrap',
},
+ btn: {minWidth: 120, margin: 6},
preview: {
flex: 1,
- alignItems: 'stretch',
- justifyContent: 'center',
+ position: 'absolute',
+ top: 0,
+ left: 0,
+ right: 0,
+ bottom: 0,
},
- fill: {flex: 1},
hud: {
position: 'absolute',
- bottom: 8,
- right: 8,
+ bottom: 30,
+ right: 0,
+ left: 0,
+ alignItems: 'center',
backgroundColor: 'rgba(0,0,0,0.4)',
paddingHorizontal: 8,
paddingVertical: 4,
@@ -114,6 +248,11 @@ const styles = StyleSheet.create({
},
hudText: {
color: '#fff',
- fontSize: 12,
+ fontSize: 14,
+ },
+ trackId: {
+ color: 'gray',
+ fontSize: 14,
+ textAlign: 'center',
},
});
diff --git a/example/src/test-framework.tsx b/example/src/test-framework.tsx
new file mode 100644
index 0000000..a85a905
--- /dev/null
+++ b/example/src/test-framework.tsx
@@ -0,0 +1,568 @@
+import React, {useState, useRef, useCallback, useEffect} from 'react';
+import {View, Text, Button, StyleSheet, ScrollView, Alert} from 'react-native';
+import {SafeAreaView, useSafeAreaInsets} from 'react-native-safe-area-context';
+import {
+ Camera,
+ useCameraDevice,
+ useFrameProcessor,
+} from 'react-native-vision-camera';
+import {
+ createVisionCameraSource,
+ createWebRTCTrack,
+ disposeTrack,
+ disposeSource,
+ getStats,
+} from 'react-native-vision-rtc';
+import {findNodeHandle} from 'react-native';
+import {
+ processFrame,
+ setSourceId,
+ clearSourceId,
+ getFrameCount,
+} from './vision-camera-frame-processor';
+
+type TestResult = {
+ name: string;
+ status: 'pending' | 'running' | 'passed' | 'failed';
+ message?: string;
+ duration?: number;
+};
+
+type TestSuite = {
+ name: string;
+ tests: TestResult[];
+};
+
+type TestFrameworkProps = {
+ onBack?: () => void;
+};
+
+export default function TestFramework({onBack}: TestFrameworkProps = {}) {
+ const [testSuites, setTestSuites] = useState([]);
+ const [isRunning, setIsRunning] = useState(false);
+ const cameraRef = useRef(null);
+ const device = useCameraDevice('back');
+
+ const insets = useSafeAreaInsets();
+
+ // Test state
+ const [currentSourceId, setCurrentSourceId] = useState(null);
+ const [currentTrackId, setCurrentTrackId] = useState(null);
+
+ // Frame processor for testing actual frame delivery
+ const frameProcessor = useFrameProcessor((frame) => {
+ processFrame(frame);
+ }, []);
+
+ const updateTestResult = useCallback(
+ (suiteName: string, testName: string, result: Partial) => {
+ setTestSuites((prev) =>
+ prev.map((suite) =>
+ suite.name === suiteName
+ ? {
+ ...suite,
+ tests: suite.tests.map((test) =>
+ test.name === testName ? {...test, ...result} : test
+ ),
+ }
+ : suite
+ )
+ );
+ },
+ []
+ );
+
+ const runTest = useCallback(
+ async (
+ suiteName: string,
+ testName: string,
+ testFn: () => Promise
+ ) => {
+ const startTime = Date.now();
+ updateTestResult(suiteName, testName, {status: 'running'});
+
+ try {
+ await testFn();
+ const duration = Date.now() - startTime;
+ updateTestResult(suiteName, testName, {
+ status: 'passed',
+ message: `โ
Passed in ${duration}ms`,
+ duration,
+ });
+ } catch (error) {
+ const duration = Date.now() - startTime;
+ updateTestResult(suiteName, testName, {
+ status: 'failed',
+ message: `โ ${error instanceof Error ? error.message : String(error)}`,
+ duration,
+ });
+ }
+ },
+ [updateTestResult]
+ );
+
+ // Initialize test suites
+ useEffect(() => {
+ setTestSuites([
+ {
+ name: 'Core Integration',
+ tests: [
+ {name: 'Create Vision Camera Source', status: 'pending'},
+ {name: 'Create WebRTC Track', status: 'pending'},
+ {name: 'Setup Frame Processor', status: 'pending'},
+ {name: 'Test Frame Delivery', status: 'pending'},
+ {name: 'Dispose Resources', status: 'pending'},
+ ],
+ },
+ {
+ name: 'Memory Management',
+ tests: [
+ {name: 'Multiple Track Creation/Disposal', status: 'pending'},
+ {name: 'Source Disposal Cleanup', status: 'pending'},
+ {name: 'Track Pause/Resume Cleanup', status: 'pending'},
+ ],
+ },
+ {
+ name: 'Performance',
+ tests: [
+ {name: 'Frame Delivery Latency', status: 'pending'},
+ {name: 'Statistics Accuracy', status: 'pending'},
+ {name: 'High FPS Handling', status: 'pending'},
+ ],
+ },
+ {
+ name: 'Error Handling',
+ tests: [
+ {name: 'Invalid Source ID', status: 'pending'},
+ {name: 'Invalid Pixel Buffer', status: 'pending'},
+ {name: 'Unsupported Pixel Format', status: 'pending'},
+ ],
+ },
+ ]);
+ }, []);
+
+ // Test implementations
+ const testCreateVisionCameraSource = useCallback(async () => {
+ const node = findNodeHandle(cameraRef.current);
+ if (!node) throw new Error('Camera ref not available');
+
+ const result = await createVisionCameraSource(node);
+ if (!result.__nativeSourceId) throw new Error('No source ID returned');
+
+ setCurrentSourceId(result.__nativeSourceId);
+ }, []);
+
+ const testCreateWebRTCTrack = useCallback(async () => {
+ if (!currentSourceId) throw new Error('No source ID available');
+
+ const result = await createWebRTCTrack(
+ {__nativeSourceId: currentSourceId},
+ {
+ fps: 30,
+ resolution: {width: 1280, height: 720},
+ backpressure: 'drop-late',
+ mode: 'external',
+ }
+ );
+
+ if (!result.trackId) throw new Error('No track ID returned');
+ setCurrentTrackId(result.trackId);
+ }, [currentSourceId]);
+
+ const testSetupFrameProcessor = useCallback(async () => {
+ if (!currentSourceId) throw new Error('No source ID available');
+
+ setSourceId(currentSourceId);
+
+ const NativeVisionRTC =
+ require('react-native-vision-rtc/src/NativeVisionRtc').default;
+ if (!NativeVisionRTC.deliverFrame) {
+ throw new Error('deliverFrame method not available');
+ }
+
+ console.log('โ
Frame processor setup complete');
+ }, [currentSourceId]);
+
+ const testFrameDelivery = useCallback(async () => {
+ if (!currentSourceId || !currentTrackId) {
+ throw new Error('Source ID and Track ID required');
+ }
+
+ // Wait for frames to be processed
+ const initialFrameCount = getFrameCount();
+
+ // Wait up to 3 seconds for frames to be delivered
+ let attempts = 0;
+ const maxAttempts = 30; // 3 seconds at 100ms intervals
+
+ while (attempts < maxAttempts) {
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ const currentFrameCount = getFrameCount();
+
+ if (currentFrameCount > initialFrameCount) {
+ // Get stats to verify frames are being delivered to WebRTC
+ const stats = await getStats(currentTrackId);
+ if (stats && (stats.producedFps > 0 || stats.deliveredFps > 0)) {
+ console.log(
+ `โ
Frame delivery working! Processed ${currentFrameCount} frames, Stats:`,
+ stats
+ );
+ return;
+ }
+ }
+
+ attempts++;
+ }
+
+ throw new Error(
+ `No frames delivered after ${maxAttempts * 100}ms. Frame count: ${getFrameCount()}`
+ );
+ }, [currentSourceId, currentTrackId]);
+
+ const testDisposeResources = useCallback(async () => {
+ // Clear frame processor first
+ clearSourceId();
+
+ if (currentTrackId) {
+ await disposeTrack(currentTrackId);
+ setCurrentTrackId(null);
+ }
+ if (currentSourceId) {
+ await disposeSource(currentSourceId);
+ setCurrentSourceId(null);
+ }
+ }, [currentTrackId, currentSourceId]);
+
+ const testMultipleTrackCreationDisposal = useCallback(async () => {
+ if (!currentSourceId) throw new Error('No source ID available');
+
+ const trackIds: string[] = [];
+
+ // Create multiple tracks
+ for (let i = 0; i < 3; i++) {
+ const result = await createWebRTCTrack(
+ {__nativeSourceId: currentSourceId},
+ {fps: 15, resolution: {width: 640, height: 480}}
+ );
+ trackIds.push(result.trackId);
+ }
+
+ // Dispose all tracks
+ for (const trackId of trackIds) {
+ await disposeTrack(trackId);
+ }
+ }, [currentSourceId]);
+
+ const testFrameDeliveryLatency = useCallback(async () => {
+ if (!currentTrackId) throw new Error('No track ID available');
+
+ const startTime = Date.now();
+ const stats = await getStats(currentTrackId);
+ const latency = Date.now() - startTime;
+
+ if (latency > 50) {
+ throw new Error(`Stats query too slow: ${latency}ms`);
+ }
+
+ console.log(`Stats query latency: ${latency}ms`, stats);
+ }, [currentTrackId]);
+
+ const testInvalidSourceId = useCallback(async () => {
+ const NativeVisionRTC =
+ require('react-native-vision-rtc/src/NativeVisionRtc').default;
+
+ try {
+ // This should fail with validation error
+ await NativeVisionRTC.deliverFrame('', null, 0);
+ throw new Error('Should have failed with invalid source ID');
+ } catch (error) {
+ if (
+ error instanceof Error &&
+ error.message.includes('INVALID_SOURCE_ID')
+ ) {
+ return; // Expected error
+ }
+ throw error;
+ }
+ }, []);
+
+ const runCoreIntegrationTests = useCallback(async () => {
+ await runTest(
+ 'Core Integration',
+ 'Create Vision Camera Source',
+ testCreateVisionCameraSource
+ );
+ await runTest(
+ 'Core Integration',
+ 'Create WebRTC Track',
+ testCreateWebRTCTrack
+ );
+ await runTest(
+ 'Core Integration',
+ 'Setup Frame Processor',
+ testSetupFrameProcessor
+ );
+ await runTest('Core Integration', 'Test Frame Delivery', testFrameDelivery);
+ await runTest(
+ 'Core Integration',
+ 'Dispose Resources',
+ testDisposeResources
+ );
+ }, [
+ runTest,
+ testCreateVisionCameraSource,
+ testCreateWebRTCTrack,
+ testSetupFrameProcessor,
+ testFrameDelivery,
+ testDisposeResources,
+ ]);
+
+ const runMemoryManagementTests = useCallback(async () => {
+ // Recreate source for memory tests
+ await runTest(
+ 'Core Integration',
+ 'Create Vision Camera Source',
+ testCreateVisionCameraSource
+ );
+
+ await runTest(
+ 'Memory Management',
+ 'Multiple Track Creation/Disposal',
+ testMultipleTrackCreationDisposal
+ );
+ await runTest(
+ 'Memory Management',
+ 'Source Disposal Cleanup',
+ testDisposeResources
+ );
+ }, [
+ runTest,
+ testCreateVisionCameraSource,
+ testMultipleTrackCreationDisposal,
+ testDisposeResources,
+ ]);
+
+ const runPerformanceTests = useCallback(async () => {
+ // Recreate resources for performance tests
+ await runTest(
+ 'Core Integration',
+ 'Create Vision Camera Source',
+ testCreateVisionCameraSource
+ );
+ await runTest(
+ 'Core Integration',
+ 'Create WebRTC Track',
+ testCreateWebRTCTrack
+ );
+
+ await runTest(
+ 'Performance',
+ 'Frame Delivery Latency',
+ testFrameDeliveryLatency
+ );
+
+ // Cleanup
+ await runTest(
+ 'Core Integration',
+ 'Dispose Resources',
+ testDisposeResources
+ );
+ }, [
+ runTest,
+ testCreateVisionCameraSource,
+ testCreateWebRTCTrack,
+ testFrameDeliveryLatency,
+ testDisposeResources,
+ ]);
+
+ const runErrorHandlingTests = useCallback(async () => {
+ await runTest('Error Handling', 'Invalid Source ID', testInvalidSourceId);
+ }, [runTest, testInvalidSourceId]);
+
+ const runAllTests = useCallback(async () => {
+ if (isRunning) return;
+
+ setIsRunning(true);
+
+ try {
+ await Camera.requestCameraPermission();
+
+ await runCoreIntegrationTests();
+ await runMemoryManagementTests();
+ await runPerformanceTests();
+ await runErrorHandlingTests();
+
+ Alert.alert('Tests Complete', 'All test suites have finished running');
+ } catch (error) {
+ Alert.alert('Test Error', `Failed to run tests: ${error}`);
+ } finally {
+ setIsRunning(false);
+ }
+ }, [
+ isRunning,
+ runCoreIntegrationTests,
+ runMemoryManagementTests,
+ runPerformanceTests,
+ runErrorHandlingTests,
+ ]);
+
+ const getOverallStatus = () => {
+ const allTests = testSuites.flatMap((suite) => suite.tests);
+ const passed = allTests.filter((t) => t.status === 'passed').length;
+ const failed = allTests.filter((t) => t.status === 'failed').length;
+ const total = allTests.length;
+
+ return {passed, failed, total};
+ };
+
+ const {passed, failed, total} = getOverallStatus();
+
+ return (
+
+
+
+ VisionRTC Test Framework
+
+ {passed}/{total} passed โข {failed} failed
+
+
+ {onBack && (
+
+ )}
+
+
+
+
+ {device && (
+
+ )}
+
+
+ {testSuites.map((suite) => (
+
+ {suite.name}
+ {suite.tests.map((test) => (
+
+
+ {test.name}
+
+ {test.status.toUpperCase()}
+
+
+ {test.message && (
+ {test.message}
+ )}
+
+ ))}
+
+ ))}
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ container: {
+ flex: 1,
+ backgroundColor: '#f5f5f5',
+ },
+ content: {
+ flex: 1,
+ },
+ header: {
+ padding: 20,
+ backgroundColor: '#fff',
+ borderBottomWidth: 1,
+ borderBottomColor: '#ddd',
+ },
+ title: {
+ fontSize: 24,
+ fontWeight: 'bold',
+ marginBottom: 5,
+ },
+ subtitle: {
+ fontSize: 16,
+ color: '#666',
+ marginBottom: 15,
+ },
+ camera: {
+ height: 100,
+ backgroundColor: '#000',
+ },
+ results: {
+ flex: 1,
+ padding: 15,
+ },
+ suite: {
+ backgroundColor: '#fff',
+ marginBottom: 15,
+ borderRadius: 8,
+ padding: 15,
+ },
+ suiteName: {
+ fontSize: 18,
+ fontWeight: 'bold',
+ marginBottom: 10,
+ color: '#333',
+ },
+ test: {
+ marginBottom: 10,
+ paddingBottom: 10,
+ borderBottomWidth: 1,
+ borderBottomColor: '#eee',
+ },
+ testHeader: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ },
+ testName: {
+ fontSize: 16,
+ flex: 1,
+ },
+ status: {
+ fontSize: 12,
+ fontWeight: 'bold',
+ paddingHorizontal: 8,
+ paddingVertical: 4,
+ borderRadius: 4,
+ },
+ pending: {
+ backgroundColor: '#f0f0f0',
+ color: '#666',
+ },
+ running: {
+ backgroundColor: '#fff3cd',
+ color: '#856404',
+ },
+ passed: {
+ backgroundColor: '#d4edda',
+ color: '#155724',
+ },
+ failed: {
+ backgroundColor: '#f8d7da',
+ color: '#721c24',
+ },
+ message: {
+ fontSize: 14,
+ color: '#666',
+ marginTop: 5,
+ fontFamily: 'monospace',
+ },
+ buttonRow: {
+ flexDirection: 'row',
+ gap: 10,
+ justifyContent: 'space-between',
+ },
+});
diff --git a/example/src/vision-camera-frame-processor.ts b/example/src/vision-camera-frame-processor.ts
new file mode 100644
index 0000000..d4c41e1
--- /dev/null
+++ b/example/src/vision-camera-frame-processor.ts
@@ -0,0 +1,51 @@
+import type {Frame} from 'react-native-vision-camera';
+import {deliverFrame} from 'react-native-vision-rtc';
+
+let sourceId: string | null = null;
+let frameCount = 0;
+
+export function setSourceId(id: string) {
+ sourceId = id;
+ frameCount = 0;
+}
+
+export function clearSourceId() {
+ sourceId = null;
+ frameCount = 0;
+}
+
+export function processFrame(frame: Frame): void {
+ 'worklet';
+
+ if (!sourceId) {
+ console.log('VisionCamera: No source ID set, skipping frame delivery');
+ return;
+ }
+
+ frameCount++;
+
+ // Only deliver every 2nd frame to avoid overwhelming the system during testing
+ if (frameCount % 2 !== 0) {
+ return;
+ }
+
+ try {
+ // Get the native pixel buffer reference
+ const pixelBufferRef = (frame as Frame).getNativeBuffer();
+ const timestampNs = frame.timestamp * 1000000; // Convert to nanoseconds
+
+ if (pixelBufferRef) {
+ deliverFrame(sourceId, pixelBufferRef, timestampNs).catch(
+ (error: unknown) => {
+ console.warn('VisionCamera: Failed to deliver frame:', error);
+ }
+ );
+ }
+ } catch (error) {
+ console.warn('VisionCamera: Error processing frame:', error);
+ }
+}
+
+export function getFrameCount(): number {
+ return frameCount;
+}
diff --git a/ios/VisionRTC+Spec.mm b/ios/VisionRTC+Spec.mm
index b7f7a73..453656e 100644
--- a/ios/VisionRTC+Spec.mm
+++ b/ios/VisionRTC+Spec.mm
@@ -8,6 +8,13 @@ @protocol VisionRTCExports
- (void)createVisionCameraSource:(NSNumber *)viewTag
resolver:(RCTPromiseResolveBlock)resolve
rejecter:(RCTPromiseRejectBlock)reject;
+- (void)updateSource:(NSString *)sourceId
+ opts:(NSDictionary *)opts
+ resolver:(RCTPromiseResolveBlock)resolve
+ rejecter:(RCTPromiseRejectBlock)reject;
+- (void)disposeSource:(NSString *)sourceId
+ resolver:(RCTPromiseResolveBlock)resolve
+ rejecter:(RCTPromiseRejectBlock)reject;
- (void)createTrack:(NSDictionary *)source
opts:(NSDictionary *)opts
resolver:(RCTPromiseResolveBlock)resolve
@@ -31,6 +38,14 @@ - (void)disposeTrack:(NSString *)trackId
rejecter:(RCTPromiseRejectBlock)reject;
- (void)getStats:(RCTPromiseResolveBlock)resolve
rejecter:(RCTPromiseRejectBlock)reject;
+ - (void)getStatsForTrack:(NSString *)trackId
+ resolver:(RCTPromiseResolveBlock)resolve
+ rejecter:(RCTPromiseRejectBlock)reject;
+- (void)deliverFrame:(NSString *)sourceId
+ pixelBuffer:(CVPixelBuffer *)pixelBuffer
+ timestampNs:(NSNumber *)timestampNs
+ resolver:(RCTPromiseResolveBlock)resolve
+ rejecter:(RCTPromiseRejectBlock)reject;
@end
@interface VisionRTCTurbo : NSObject
@@ -92,6 +107,10 @@ - (void)createTrack:(NSDictionary *)source
resDict[@"height"] = @((*res).height());
optsDict[@"resolution"] = resDict;
}
+ NSString *bp = opts.backpressure();
+ if (bp != nil) {
+ optsDict[@"backpressure"] = bp;
+ }
[self.swift createTrack:source opts:optsDict resolver:resolve rejecter:reject];
}
@@ -156,6 +175,10 @@ - (void)setTrackConstraints:(NSString *)trackId
resDict[@"height"] = @((*res).height());
optsDict[@"resolution"] = resDict;
}
+ NSString *bp = opts.backpressure();
+ if (bp != nil) {
+ optsDict[@"backpressure"] = bp;
+ }
[self.swift setTrackConstraints:trackId opts:optsDict resolver:resolve rejecter:reject];
}
@@ -184,4 +207,59 @@ - (void)getStats:(RCTPromiseResolveBlock)resolve
[self.swift getStats:resolve rejecter:reject];
}
-@end
\ No newline at end of file
+- (void)getStatsForTrack:(NSString *)trackId
+ resolve:(RCTPromiseResolveBlock)resolve
+ reject:(RCTPromiseRejectBlock)reject
+{
+ if (!self.swift) {
+ if (reject) reject(@"E_NO_SWIFT_IMPL",
+ @"VisionRTC Swift implementation not found. Ensure @objc(VisionRTC) exists in module 'VisionRtc'.",
+ nil);
+ return;
+ }
+ [self.swift getStatsForTrack:trackId resolver:resolve rejecter:reject];
+}
+
+- (void)updateSource:(NSString *)sourceId
+ opts:(NSDictionary *)opts
+ resolve:(RCTPromiseResolveBlock)resolve
+ reject:(RCTPromiseRejectBlock)reject
+{
+ if (!self.swift) {
+ if (reject) reject(@"E_NO_SWIFT_IMPL",
+ @"VisionRTC Swift implementation not found. Ensure @objc(VisionRTC) exists in module 'VisionRtc'.",
+ nil);
+ return;
+ }
+ [self.swift updateSource:sourceId opts:opts resolver:resolve rejecter:reject];
+}
+
+- (void)disposeSource:(NSString *)sourceId
+ resolve:(RCTPromiseResolveBlock)resolve
+ reject:(RCTPromiseRejectBlock)reject
+{
+ if (!self.swift) {
+ if (reject) reject(@"E_NO_SWIFT_IMPL",
+ @"VisionRTC Swift implementation not found. Ensure @objc(VisionRTC) exists in module 'VisionRtc'.",
+ nil);
+ return;
+ }
+ [self.swift disposeSource:sourceId resolver:resolve rejecter:reject];
+}
+
+- (void)deliverFrame:(NSString *)sourceId
+ pixelBuffer:(CVPixelBuffer *)pixelBuffer
+ timestampNs:(NSNumber *)timestampNs
+ resolve:(RCTPromiseResolveBlock)resolve
+ reject:(RCTPromiseRejectBlock)reject
+{
+ if (!self.swift) {
+ if (reject) reject(@"E_NO_SWIFT_IMPL",
+ @"VisionRTC Swift implementation not found. Ensure @objc(VisionRTC) exists in module 'VisionRtc'.",
+ nil);
+ return;
+ }
+ [self.swift deliverFrame:sourceId pixelBuffer:pixelBuffer timestampNs:timestampNs resolver:resolve rejecter:reject];
+}
+
+@end
diff --git a/ios/VisionRTCModule.swift b/ios/VisionRTCModule.swift
index be03fab..0b395ce 100644
--- a/ios/VisionRTCModule.swift
+++ b/ios/VisionRTCModule.swift
@@ -15,7 +15,8 @@ class VisionRTC: NSObject {
var width: Int32
var height: Int32
var fps: Int
- var mode: String
+ var mode: String // 'null-gpu' | 'null-cpu' | 'external'
+ var backpressure: String? // 'drop-late' | 'latest-wins' | 'throttle'
}
private var sources: [String: RTCVideoSource] = [:]
@@ -27,6 +28,26 @@ class VisionRTC: NSObject {
private let stateQueue = DispatchQueue(label: "com.visionrtc.state", attributes: .concurrent)
private var lastSent: [String: CFTimeInterval] = [:]
+ private struct SourceHandle {
+ let viewTag: Int
+ var position: String?
+ var torch: Bool?
+ var maxFps: Int?
+ }
+ private var cameraSources: [String: SourceHandle] = [:]
+ private var sourceToTrackIds: [String: Set] = [:]
+ private var trackToSourceId: [String: String] = [:]
+
+ // Backpressure and stats (per track)
+ private var latestBufferByTrack: [String: (pb: CVPixelBuffer, tsNs: Int64)] = [:]
+ private var producedThisSecond: [String: Int] = [:]
+ private var deliveredThisSecond: [String: Int] = [:]
+ private var lastSecondWallClock: [String: CFTimeInterval] = [:]
+ private var deliveredFpsByTrack: [String: Int] = [:]
+ private var producedFpsByTrack: [String: Int] = [:]
+ private var droppedFramesByTrack: [String: Int] = [:]
+ private var pausedForReconfig: Set = []
+
private var captureTimer: DispatchSourceTimer?
private let timerQueue = DispatchQueue(label: "com.visionrtc.capture", qos: .userInitiated)
private var gpuGenerators: [String: NullGPUGenerator] = [:]
@@ -38,9 +59,62 @@ class VisionRTC: NSObject {
resolver: RCTPromiseResolveBlock,
rejecter: RCTPromiseRejectBlock) {
let id = UUID().uuidString
+ let handle = SourceHandle(viewTag: viewTag.intValue, position: nil, torch: nil, maxFps: nil)
+ stateQueue.async(flags: .barrier) {
+ self.cameraSources[id] = handle
+ self.sourceToTrackIds[id] = Set()
+ }
resolver(["__nativeSourceId": id])
}
+ @objc(updateSource:opts:resolver:rejecter:)
+ func updateSource(sourceId: NSString, opts: NSDictionary,
+ resolver: RCTPromiseResolveBlock, rejecter: RCTPromiseRejectBlock) {
+ stateQueue.async(flags: .barrier) {
+ if var s = self.cameraSources[sourceId as String] {
+ if let pos = opts["position"] as? String { s.position = pos }
+ if let torch = opts["torch"] as? NSNumber { s.torch = torch.boolValue }
+ if let maxFps = opts["maxFps"] as? NSNumber { s.maxFps = maxFps.intValue }
+ self.cameraSources[sourceId as String] = s
+ }
+ let tracksForSource = Array(self.sourceToTrackIds[sourceId as String] ?? [])
+ for tid in tracksForSource {
+ self.pausedForReconfig.insert(tid)
+ self.latestBufferByTrack.removeValue(forKey: tid)
+ self.lastSent.removeValue(forKey: tid)
+ self.producedThisSecond[tid] = 0
+ self.deliveredThisSecond[tid] = 0
+ self.producedFpsByTrack[tid] = 0
+ self.deliveredFpsByTrack[tid] = 0
+ self.droppedFramesByTrack[tid] = 0
+ self.lastSecondWallClock[tid] = CACurrentMediaTime()
+ }
+ }
+ stateQueue.asyncAfter(deadline: .now() + 0.35, flags: .barrier) {
+ let tracksForSource = Array(self.sourceToTrackIds[sourceId as String] ?? [])
+ for tid in tracksForSource {
+ self.pausedForReconfig.remove(tid)
+ }
+ }
+ resolver(NSNull())
+ }
+
+ @objc(disposeSource:resolver:rejecter:)
+ func disposeSource(sourceId: NSString, resolver: RCTPromiseResolveBlock, rejecter: RCTPromiseRejectBlock) {
+ stateQueue.async(flags: .barrier) {
+ // Clean up pixel buffers for all tracks using this source
+ if let trackIds = self.sourceToTrackIds[sourceId as String] {
+ for trackId in trackIds {
+ self.latestBufferByTrack.removeValue(forKey: trackId)
+ self.trackToSourceId.removeValue(forKey: trackId)
+ }
+ }
+ self.cameraSources.removeValue(forKey: sourceId as String)
+ self.sourceToTrackIds.removeValue(forKey: sourceId as String)
+ }
+ resolver(NSNull())
+ }
+
@objc(createTrack:opts:resolver:rejecter:)
func createTrack(sourceDict: NSDictionary?, opts: NSDictionary?,
resolver: RCTPromiseResolveBlock, rejecter: RCTPromiseRejectBlock) {
@@ -48,7 +122,8 @@ class VisionRTC: NSObject {
var width: Int32 = 1280
var height: Int32 = 720
var fps: Int = 30
- let mode = (opts?["mode"] as? String) ?? "null-gpu"
+ var mode = (opts?["mode"] as? String)
+ var backpressure = opts?["backpressure"] as? String
if let res = opts?["resolution"] as? [String: Any],
let w = res["width"] as? NSNumber, let h = res["height"] as? NSNumber {
width = w.int32Value; height = h.int32Value
@@ -60,19 +135,33 @@ class VisionRTC: NSObject {
let trackId = UUID().uuidString
let track = factory.videoTrack(with: source, trackId: trackId)
+ var boundSourceId: String?
+ if let sd = sourceDict, let sid = sd["__nativeSourceId"] as? String {
+ boundSourceId = sid
+ if mode == nil { mode = "external" }
+ }
+ if mode == nil { mode = "null-gpu" }
+
stateQueue.async(flags: .barrier) {
self.sources[trackId] = source
self.tracks[trackId] = track
self.capturers[trackId] = RTCVideoCapturer(delegate: source)
- self.trackStates[trackId] = TrackState(width: width, height: height, fps: fps, mode: mode)
+ self.trackStates[trackId] = TrackState(width: width, height: height, fps: fps, mode: mode!, backpressure: backpressure)
self.activeTrackIds.insert(trackId)
self.lastSent[trackId] = 0
+ if let sid = boundSourceId {
+ self.trackToSourceId[trackId] = sid
+ var set = self.sourceToTrackIds[sid] ?? Set()
+ set.insert(trackId)
+ self.sourceToTrackIds[sid] = set
+ }
}
VisionRTCTrackRegistry.shared.register(trackId: trackId, track: track)
- if mode == "null-gpu" { startGpuGenerator(for: trackId) } else {
- // TODO(phase3): Support 'external' mode ingesting CVPixelBuffer from Vision Camera
+ if mode == "null-gpu" {
+ startGpuGenerator(for: trackId)
+ } else if mode == "null-cpu" {
startNullCapturer()
- }
+ } // 'external' emits frames from camera binding
resolver(["trackId": trackId])
}
@@ -87,13 +176,13 @@ class VisionRTC: NSObject {
func pauseTrack(trackId: NSString, resolver: RCTPromiseResolveBlock, rejecter: RCTPromiseRejectBlock) {
stateQueue.async(flags: .barrier) {
self.activeTrackIds.remove(trackId as String)
+ // Clean up retained pixel buffer to prevent memory leaks
+ self.latestBufferByTrack.removeValue(forKey: trackId as String)
}
stateQueue.sync {
if let st = self.trackStates[trackId as String], st.mode == "null-gpu" {
self.stopGpuGenerator(for: trackId as String)
}
- }
- stateQueue.sync {
if self.activeTrackIds.isEmpty { self.stopNullCapturer() }
}
resolver(NSNull())
@@ -105,10 +194,12 @@ class VisionRTC: NSObject {
self.activeTrackIds.insert(trackId as String)
}
stateQueue.sync {
- if let st = self.trackStates[trackId as String], st.mode == "null-gpu" {
- self.startGpuGenerator(for: trackId as String)
- } else {
- self.startNullCapturer()
+ if let st = self.trackStates[trackId as String] {
+ if st.mode == "null-gpu" {
+ self.startGpuGenerator(for: trackId as String)
+ } else if st.mode == "null-cpu" {
+ self.startNullCapturer()
+ }
}
}
resolver(NSNull())
@@ -120,18 +211,21 @@ class VisionRTC: NSObject {
var nextWidth: Int32?
var nextHeight: Int32?
var nextFps: Int?
+ var nextBackpressure: String?
if let res = opts["resolution"] as? [String: Any] {
if let w = res["width"] as? NSNumber { nextWidth = w.int32Value }
if let h = res["height"] as? NSNumber { nextHeight = h.int32Value }
}
if let f = opts["fps"] as? NSNumber { nextFps = f.intValue }
+ if let bp = opts["backpressure"] as? String { nextBackpressure = bp }
stateQueue.async(flags: .barrier) {
if var st = self.trackStates[trackId as String] {
if let w = nextWidth { st.width = w }
if let h = nextHeight { st.height = h }
if let f = nextFps { st.fps = f }
+ if let bp = nextBackpressure { st.backpressure = bp }
self.trackStates[trackId as String] = st
if let src = self.sources[trackId as String] {
src.adaptOutputFormat(toWidth: st.width, height: st.height, fps: Int32(st.fps))
@@ -157,6 +251,13 @@ class VisionRTC: NSObject {
self.sources.removeValue(forKey: trackId as String)
self.capturers.removeValue(forKey: trackId as String)
self.lastSent.removeValue(forKey: trackId as String)
+ self.latestBufferByTrack.removeValue(forKey: trackId as String)
+ self.producedThisSecond.removeValue(forKey: trackId as String)
+ self.deliveredThisSecond.removeValue(forKey: trackId as String)
+ self.lastSecondWallClock.removeValue(forKey: trackId as String)
+ self.deliveredFpsByTrack.removeValue(forKey: trackId as String)
+ self.producedFpsByTrack.removeValue(forKey: trackId as String)
+ self.droppedFramesByTrack.removeValue(forKey: trackId as String)
}
VisionRTCTrackRegistry.shared.unregister(trackId: trackId as String)
stopGpuGenerator(for: trackId as String)
@@ -168,7 +269,17 @@ class VisionRTC: NSObject {
@objc(getStats:rejecter:)
func getStats(resolver: RCTPromiseResolveBlock, rejecter: RCTPromiseRejectBlock) {
- resolver(["fps": producedFps, "droppedFrames": droppedFrames])
+ var deliveredSum = 0
+ var droppedSum = 0
+ stateQueue.sync {
+ deliveredSum = self.deliveredFpsByTrack.values.reduce(0, +)
+ droppedSum = self.droppedFramesByTrack.values.reduce(0, +)
+ // If not yet rolled in this second, fall back to current counters
+ if deliveredSum == 0 {
+ deliveredSum = self.deliveredThisSecond.values.reduce(0, +)
+ }
+ }
+ resolver(["fps": deliveredSum, "droppedFrames": droppedSum])
}
private func startNullCapturer() {
@@ -231,9 +342,15 @@ class VisionRTC: NSObject {
shouldEmit = true
}
}
- if !shouldEmit { continue }
+ if !shouldEmit {
+ // Only count drops for drop-late policy (null-cpu mode doesn't use backpressure policies)
+ stateQueue.async(flags: .barrier) {
+ self.droppedFramesByTrack[trackId] = (self.droppedFramesByTrack[trackId] ?? 0) + 1
+ }
+ continue
+ }
- if st.mode == "null-gpu" { continue }
+ if st.mode != "null-cpu" { continue }
let width = Int(st.width)
let height = Int(st.height)
let bytesPerPixel = 4
@@ -293,11 +410,179 @@ class VisionRTC: NSObject {
} else {
src.capturer(RTCVideoCapturer(delegate: src), didCapture: frame)
}
+ // Update per-track stats for null-cpu: produced and delivered together
+ self.stateQueue.sync(flags: .barrier) {
+ self.producedThisSecond[trackId] = (self.producedThisSecond[trackId] ?? 0) + 1
+ self.deliveredThisSecond[trackId] = (self.deliveredThisSecond[trackId] ?? 0) + 1
+ if self.lastSecondWallClock[trackId] == nil {
+ self.lastSecondWallClock[trackId] = nowSec
+ }
+ let lastSecond = self.lastSecondWallClock[trackId] ?? nowSec
+ if nowSec - lastSecond >= 1.0 {
+ self.producedFpsByTrack[trackId] = self.producedThisSecond[trackId] ?? 0
+ self.deliveredFpsByTrack[trackId] = self.deliveredThisSecond[trackId] ?? 0
+ self.producedThisSecond[trackId] = 0
+ self.deliveredThisSecond[trackId] = 0
+ self.lastSecondWallClock[trackId] = nowSec
+ }
+ }
}
}
}
extension VisionRTC {
+ fileprivate func deliverExternalFrame(sourceId: String, pixelBuffer: CVPixelBuffer, timestampNs: Int64) {
+ var trackIds: [String] = []
+ stateQueue.sync { trackIds = Array(self.sourceToTrackIds[sourceId] ?? []) }
+ if trackIds.isEmpty {
+ print("VisionRTC: Warning - No tracks found for source \(sourceId)")
+ return
+ }
+ for trackId in trackIds {
+ var isPaused = false
+ var stOpt: TrackState?
+ var srcOpt: RTCVideoSource?
+ var capOpt: RTCVideoCapturer?
+ stateQueue.sync {
+ isPaused = self.pausedForReconfig.contains(trackId)
+ stOpt = self.trackStates[trackId]
+ srcOpt = self.sources[trackId]
+ capOpt = self.capturers[trackId]
+ }
+ if isPaused { continue }
+ guard let st = stOpt, let src = srcOpt else { continue }
+ let nowSec = CACurrentMediaTime()
+ let intervalSec = 1.0 / Double(max(1, st.fps))
+
+ // Update produced counters and per-second rollups
+ stateQueue.async(flags: .barrier) {
+ self.producedThisSecond[trackId] = (self.producedThisSecond[trackId] ?? 0) + 1
+ if self.lastSecondWallClock[trackId] == nil {
+ self.lastSecondWallClock[trackId] = nowSec
+ }
+ let lastSecond = self.lastSecondWallClock[trackId] ?? nowSec
+ if nowSec - lastSecond >= 1.0 {
+ self.producedFpsByTrack[trackId] = self.producedThisSecond[trackId] ?? 0
+ self.deliveredFpsByTrack[trackId] = self.deliveredThisSecond[trackId] ?? 0
+ self.producedThisSecond[trackId] = 0
+ self.deliveredThisSecond[trackId] = 0
+ self.lastSecondWallClock[trackId] = nowSec
+ }
+ }
+
+ let policy = st.backpressure ?? "drop-late"
+ var shouldEmit = false
+ var bufferToSend: CVPixelBuffer = pixelBuffer
+
+ stateQueue.sync {
+ let last = self.lastSent[trackId] ?? 0
+ if policy == "latest-wins" {
+ // Always keep the latest; emit only on cadence. Count suppressed frames as drops.
+ self.latestBufferByTrack[trackId] = (pixelBuffer, timestampNs)
+ if nowSec - last >= intervalSec {
+ self.lastSent[trackId] = nowSec
+ if let latest = self.latestBufferByTrack[trackId] {
+ bufferToSend = latest.pb
+ }
+ shouldEmit = true
+ } else {
+ self.droppedFramesByTrack[trackId] = (self.droppedFramesByTrack[trackId] ?? 0) + 1
+ }
+ } else if policy == "throttle" {
+ self.latestBufferByTrack[trackId] = (pixelBuffer, timestampNs)
+ if nowSec - last >= intervalSec {
+ self.lastSent[trackId] = nowSec
+ if let latest = self.latestBufferByTrack[trackId] {
+ bufferToSend = latest.pb
+ }
+ shouldEmit = true
+ }
+ } else {
+ if nowSec - last >= intervalSec {
+ self.lastSent[trackId] = nowSec
+ shouldEmit = true
+ } else {
+ self.droppedFramesByTrack[trackId] = (self.droppedFramesByTrack[trackId] ?? 0) + 1
+ }
+ }
+ }
+
+ if !shouldEmit { continue }
+
+ let rtcBuf = RTCCVPixelBuffer(pixelBuffer: bufferToSend)
+ let ts: Int64 = (policy == "latest-wins" ? (stateQueue.sync { self.latestBufferByTrack[trackId]?.tsNs } ?? timestampNs) : timestampNs)
+ let frame = RTCVideoFrame(buffer: rtcBuf, rotation: ._0, timeStampNs: ts)
+
+ guard frame.buffer.width > 0 && frame.buffer.height > 0 else {
+ print("VisionRTC: Warning - Invalid frame dimensions for track \(trackId)")
+ continue
+ }
+
+ // Deliver frame to WebRTC
+ if let cap = capOpt {
+ src.capturer(cap, didCapture: frame)
+ } else {
+ src.capturer(RTCVideoCapturer(delegate: src), didCapture: frame)
+ }
+
+ stateQueue.async(flags: .barrier) {
+ self.deliveredThisSecond[trackId] = (self.deliveredThisSecond[trackId] ?? 0) + 1
+ let lastSecond = self.lastSecondWallClock[trackId] ?? nowSec
+ if nowSec - lastSecond >= 1.0 {
+ self.producedFpsByTrack[trackId] = self.producedThisSecond[trackId] ?? 0
+ self.deliveredFpsByTrack[trackId] = self.deliveredThisSecond[trackId] ?? 0
+ self.producedThisSecond[trackId] = 0
+ self.deliveredThisSecond[trackId] = 0
+ self.lastSecondWallClock[trackId] = nowSec
+ }
+ }
+ }
+ }
+
+ @objc(getStatsForTrack:resolver:rejecter:)
+ func getStatsForTrack(trackId: NSString, resolver: RCTPromiseResolveBlock, rejecter: RCTPromiseRejectBlock) {
+ var produced = 0
+ var delivered = 0
+ var dropped = 0
+ stateQueue.sync {
+ produced = self.producedFpsByTrack[trackId as String] ?? 0
+ delivered = self.deliveredFpsByTrack[trackId as String] ?? 0
+ dropped = self.droppedFramesByTrack[trackId as String] ?? 0
+ }
+ resolver(["producedFps": produced, "deliveredFps": delivered, "droppedFrames": dropped])
+ }
+
+ @objc(deliverFrame:pixelBuffer:timestampNs:resolver:rejecter:)
+ func deliverFrame(sourceId: NSString, pixelBuffer: CVPixelBuffer, timestampNs: NSNumber,
+ resolver: RCTPromiseResolveBlock, rejecter: RCTPromiseRejectBlock) {
+ guard !sourceId.isEqual(to: "") else {
+ rejecter("INVALID_SOURCE_ID", "Source ID cannot be empty", nil)
+ return
+ }
+
+ let width = CVPixelBufferGetWidth(pixelBuffer)
+ let height = CVPixelBufferGetHeight(pixelBuffer)
+ guard width > 0 && height > 0 else {
+ rejecter("INVALID_FRAME_SIZE", "Frame dimensions must be positive", nil)
+ return
+ }
+
+ let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
+ let supportedFormats: [OSType] = [
+ kCVPixelFormatType_32BGRA,
+ kCVPixelFormatType_32ARGB,
+ kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
+ kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
+ ]
+
+ guard supportedFormats.contains(pixelFormat) else {
+ rejecter("UNSUPPORTED_PIXEL_FORMAT", "Pixel format \(pixelFormat) not supported by WebRTC", nil)
+ return
+ }
+
+ deliverExternalFrame(sourceId: sourceId as String, pixelBuffer: pixelBuffer, timestampNs: timestampNs.int64Value)
+ resolver(NSNull())
+ }
fileprivate func startGpuGenerator(for trackId: String) {
var stOpt: TrackState?
var srcOpt: RTCVideoSource?
@@ -317,9 +602,27 @@ extension VisionRTC {
} else {
src.capturer(RTCVideoCapturer(delegate: src), didCapture: frame)
}
+ let nowSec = CACurrentMediaTime()
+ self.stateQueue.sync(flags: .barrier) {
+ self.producedThisSecond[trackId] = (self.producedThisSecond[trackId] ?? 0) + 1
+ self.deliveredThisSecond[trackId] = (self.deliveredThisSecond[trackId] ?? 0) + 1
+ if self.lastSecondWallClock[trackId] == nil {
+ self.lastSecondWallClock[trackId] = nowSec
+ }
+ let lastSecond = self.lastSecondWallClock[trackId] ?? nowSec
+ if nowSec - lastSecond >= 1.0 {
+ self.producedFpsByTrack[trackId] = self.producedThisSecond[trackId] ?? 0
+ self.deliveredFpsByTrack[trackId] = self.deliveredThisSecond[trackId] ?? 0
+ self.producedThisSecond[trackId] = 0
+ self.deliveredThisSecond[trackId] = 0
+ self.lastSecondWallClock[trackId] = nowSec
+ }
+ }
}, onFps: { [weak self] fpsNow, dropped in
- self?.producedFps = fpsNow
- self?.droppedFrames = dropped
+ guard let self = self else { return }
+ self.stateQueue.sync(flags: .barrier) {
+ self.droppedFramesByTrack[trackId] = dropped
+ }
})
gpuGenerators[trackId] = gen
gen.start()
diff --git a/ios/VisionRTCViewManager.m b/ios/VisionRTCViewManager.m
index 8513330..6bab544 100644
--- a/ios/VisionRTCViewManager.m
+++ b/ios/VisionRTCViewManager.m
@@ -8,6 +8,12 @@ @interface VisionRTCView : RTCMTLVideoView
@end
@implementation VisionRTCView
+- (void)dealloc {
+ if (self.attachedTrack) {
+ [self.attachedTrack removeRenderer:self];
+ self.attachedTrack = nil;
+ }
+}
- (void)setTrackId:(NSString *)trackId {
if ([_trackId isEqualToString:trackId]) { return; }
_trackId = [trackId copy];
diff --git a/lefthook.yml b/lefthook.yml
index 5f049a8..95c6753 100644
--- a/lefthook.yml
+++ b/lefthook.yml
@@ -3,10 +3,10 @@ pre-commit:
commands:
lint:
glob: "*.{js,ts,jsx,tsx}"
- run: npx eslint {staged_files}
+ run: yarn eslint --max-warnings=0 {staged_files}
types:
glob: "*.{js,ts,jsx,tsx}"
- run: npx tsc -p tsconfig.build.json --noEmit
+ run: yarn tsc -p tsconfig.build.json --noEmit
commit-msg:
parallel: true
commands:
diff --git a/src/NativeVisionRtc.ts b/src/NativeVisionRtc.ts
index cb18f9b..ab6006c 100644
--- a/src/NativeVisionRtc.ts
+++ b/src/NativeVisionRtc.ts
@@ -11,13 +11,19 @@ type NativePixelSourceShape =
type TrackOptionsShape = {
fps?: number;
resolution?: {width: number; height: number};
+ backpressure?: 'drop-late' | 'latest-wins' | 'throttle';
mode?: 'null-gpu' | 'null-cpu' | 'external';
};
-export interface Spec extends TurboModule {
+export type Spec = TurboModule & {
readonly createVisionCameraSource: (
viewTag: number
) => Promise;
+ readonly updateSource: (
+ sourceId: string,
+ opts: {position?: 'front' | 'back'; torch?: boolean; maxFps?: number}
+ ) => Promise;
+ readonly disposeSource: (sourceId: string) => Promise;
readonly createTrack: (
source: VisionCameraSourceShape | NativePixelSourceShape,
opts?: TrackOptionsShape
@@ -39,6 +45,16 @@ export interface Spec extends TurboModule {
droppedFrames: number;
encoderQueueDepth?: number;
}>;
-}
+ readonly getStatsForTrack?: (trackId: string) => Promise<{
+ producedFps: number;
+ deliveredFps: number;
+ droppedFrames: number;
+ }>;
+ readonly deliverFrame?: (
+ sourceId: string,
+ pixelBuffer: unknown,
+ timestampNs: number
+ ) => Promise;
+};
export default TurboModuleRegistry.getEnforcing('VisionRTC');
diff --git a/src/index.ts b/src/index.ts
index 1b42601..0f6a1fe 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -5,6 +5,9 @@ import type {
VisionCameraSource,
NativePixelSource,
Resolution,
+ Capabilities,
+ VisionRtcError,
+ TrackStats,
} from './types';
import VisionRTCView from './vision-rtc-view';
@@ -14,6 +17,9 @@ export type {
VisionCameraSource,
NativePixelSource,
Resolution,
+ Capabilities,
+ VisionRtcError,
+ TrackStats,
};
export {VisionRTCView};
@@ -24,6 +30,17 @@ export async function createVisionCameraSource(
return NativeVisionRTC.createVisionCameraSource(viewTag);
}
+export function updateSource(
+ sourceId: string,
+ opts: {position?: 'front' | 'back'; torch?: boolean; maxFps?: number}
+): Promise {
+ return NativeVisionRTC.updateSource(sourceId, opts);
+}
+
+export function disposeSource(sourceId: string): Promise {
+ return NativeVisionRTC.disposeSource(sourceId);
+}
+
export async function createWebRTCTrack(
source: VisionCameraSource | NativePixelSource,
opts?: TrackOptions
@@ -31,7 +48,7 @@ export async function createWebRTCTrack(
return NativeVisionRTC.createTrack(source, opts ?? {});
}
-export function replaceTrack(
+export function replaceSenderTrack(
senderId: string,
nextTrackId: string
): Promise {
@@ -46,21 +63,93 @@ export function resumeTrack(trackId: string): Promise {
return NativeVisionRTC.resumeTrack(trackId);
}
-export function setTrackConstraints(
+export function updateTrack(
trackId: string,
- opts: TrackOptions
+ constraints: TrackOptions
): Promise {
- return NativeVisionRTC.setTrackConstraints(trackId, opts);
+ return NativeVisionRTC.setTrackConstraints(trackId, constraints);
}
export function disposeTrack(trackId: string): Promise {
return NativeVisionRTC.disposeTrack(trackId);
}
-export async function getStats(): Promise<
- {fps: number; droppedFrames: number; encoderQueueDepth?: number} | undefined
-> {
- return NativeVisionRTC.getStats
- ? await NativeVisionRTC.getStats()
- : undefined;
+export async function deliverFrame(
+ sourceId: string,
+ pixelBuffer: unknown,
+ timestampNs: number
+): Promise {
+ if (!NativeVisionRTC.deliverFrame) {
+ throw new Error('VisionRTC native module missing deliverFrame support');
+ }
+ return NativeVisionRTC.deliverFrame(sourceId, pixelBuffer, timestampNs);
+}
+
+export async function getStats(
+ _trackId?: string
+): Promise {
+ if (_trackId && NativeVisionRTC.getStatsForTrack) {
+ const s = await NativeVisionRTC.getStatsForTrack(_trackId);
+ if (!s) return undefined;
+ return s;
+ }
+ if (NativeVisionRTC.getStats) {
+ const s = await NativeVisionRTC.getStats();
+ if (!s) return undefined;
+ return {
+ producedFps: s.fps ?? 0,
+ deliveredFps: s.fps ?? 0,
+ droppedFrames: s.droppedFrames ?? 0,
+ };
+ }
+ return undefined;
+}
+
+function detectExpoGo(): boolean {
+ try {
+ // Optional dependency; only if project uses Expo
+ const Constants = require('expo-constants').default;
+ return Constants?.appOwnership === 'expo';
+ } catch {
+ return false;
+ }
+}
+
+function hasVisionCamera(): boolean {
+ try {
+ const vc = require('react-native-vision-camera');
+ return !!vc;
+ } catch {
+ return false;
+ }
+}
+
+export function getCapabilities(): Capabilities {
+ const expoGo = detectExpoGo();
+ const webrtc = !!NativeVisionRTC;
+ const visionCamera = hasVisionCamera();
+ const arkit = false;
+ const hwEncoder = {
+ h264: true,
+ vp8: true,
+ };
+ return {webrtc, visionCamera, arkit, hwEncoder, expoGo};
+}
+
+export function assertSupportedOrThrow(): void {
+ const caps = getCapabilities();
+ if (caps.expoGo) {
+ const err: VisionRtcError = {
+ code: 'ERR_EXPO_GO',
+ message: 'Expo Go is not supported. Use Expo Dev Client.',
+ };
+ throw err;
+ }
+ if (!caps.webrtc) {
+ const err: VisionRtcError = {
+ code: 'ERR_NATIVE_MODULE_UNAVAILABLE',
+ message: 'VisionRTC native module not available.',
+ };
+ throw err;
+ }
}
diff --git a/src/types.ts b/src/types.ts
index 7218c57..0e7b442 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -3,6 +3,7 @@ export type Resolution = {width: number; height: number};
export type TrackOptions = {
fps?: number; // default 30
resolution?: Resolution;
+ backpressure?: Backpressure;
bitrate?: number;
colorSpace?: 'auto' | 'sRGB' | 'BT.709' | 'BT.2020';
orientationMode?: 'auto' | 'fixed-0' | 'fixed-90' | 'fixed-180' | 'fixed-270';
@@ -36,3 +37,33 @@ export type NativePixelSource =
export type VisionRTCTrack = {
trackId: string;
};
+
+export type Backpressure = 'drop-late' | 'latest-wins' | 'throttle';
+
+export type Capabilities = {
+ webrtc: boolean;
+ visionCamera: boolean;
+ arkit: boolean;
+ hwEncoder: {h264: boolean; vp8: boolean};
+ expoGo: boolean;
+};
+
+export type VisionRtcErrorCode =
+ | 'ERR_EXPO_GO'
+ | 'ERR_MISSING_VISION_CAMERA'
+ | 'ERR_UNSUPPORTED_PLATFORM'
+ | 'ERR_NATIVE_MODULE_UNAVAILABLE';
+
+export type VisionRtcError = {
+ code: VisionRtcErrorCode;
+ message: string;
+};
+
+export type TrackStats = {
+ producedFps: number;
+ deliveredFps: number;
+ droppedFrames: number;
+ bitrateKbps?: number;
+ avgEncodeMs?: number;
+ qp?: number;
+};
diff --git a/src/vision-rtc-view.tsx b/src/vision-rtc-view.tsx
index a63fc93..9fd3438 100644
--- a/src/vision-rtc-view.tsx
+++ b/src/vision-rtc-view.tsx
@@ -1,9 +1,31 @@
-import {View, Text, StyleSheet} from 'react-native';
+import type {ReactNode} from 'react';
+import {
+ requireNativeComponent,
+ StyleSheet,
+ View,
+ type ViewStyle,
+ type StyleProp,
+} from 'react-native';
-export default function VisionRTCView({trackId}: {trackId: string}) {
+type Props = {
+ trackId?: string | null;
+ style?: StyleProp;
+ children?: ReactNode;
+};
+
+const NativeView = requireNativeComponent<{
+ trackId: string;
+ style?: StyleProp;
+}>('VisionRTCView');
+
+export default function VisionRTCView({trackId, style, children}: Props) {
return (
-
- {trackId}
+
+
+ {children}
);
}
diff --git a/yarn.lock b/yarn.lock
index 9596a59..557713e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10224,6 +10224,26 @@ __metadata:
languageName: node
linkType: hard
+"react-native-vision-camera@npm:^4.7.2":
+ version: 4.7.2
+ resolution: "react-native-vision-camera@npm:4.7.2"
+ peerDependencies:
+ "@shopify/react-native-skia": "*"
+ react: "*"
+ react-native: "*"
+ react-native-reanimated: "*"
+ react-native-worklets-core: "*"
+ peerDependenciesMeta:
+ "@shopify/react-native-skia":
+ optional: true
+ react-native-reanimated:
+ optional: true
+ react-native-worklets-core:
+ optional: true
+ checksum: d0ed87b10f0be61c25b317cc599a7c4724ff6baccea211dd5b26bcf906c197e7445ef69a0ba4fa41a68221d8d90e941f2f55410c363b4a63209a41a3d0fa3f7d
+ languageName: node
+ linkType: hard
+
"react-native-vision-rtc-example@workspace:example":
version: 0.0.0-use.local
resolution: "react-native-vision-rtc-example@workspace:example"
@@ -10244,6 +10264,7 @@ __metadata:
react-native-builder-bob: ^0.40.13
react-native-monorepo-config: ^0.1.9
react-native-safe-area-context: ^5.6.1
+ react-native-vision-camera: ^4.7.2
languageName: unknown
linkType: soft