From 3a95e4dc7aeaf0d0b60828c66b50fd55fe1a044a Mon Sep 17 00:00:00 2001 From: Emmanuel Atawodi Date: Mon, 8 Sep 2025 11:29:51 +0100 Subject: [PATCH 1/2] chore: document APIs --- README.md | 31 + .../java/com/visionrtc/GlNullGenerator.kt | 97 ++ .../java/com/visionrtc/VisionRtcModule.kt | 63 +- .../java/com/visionrtc/VisionRtcPackage.kt | 5 + .../com/visionrtc/VisionRtcViewManager.kt | 42 + example/ios/Podfile.lock | 2 +- .../project.pbxproj | 4 + example/src/App.tsx | 48 +- ios/NullGPUGenerator.swift | 124 ++ ios/VisionRTCModule.swift | 74 +- ios/VisionRTCRegistryBridge.h | 8 + ios/VisionRTCRegistryBridge.mm | 21 + ios/VisionRTCTrackRegistry.swift | 30 + ios/VisionRTCViewManager.m | 45 + lefthook.yml | 4 +- src/NativeVisionRtc.ts | 1 + src/index.ts | 3 + src/types.ts | 1 + src/vision-rtc-view.tsx | 9 + yarn.lock | 1163 ++++++++--------- 20 files changed, 1131 insertions(+), 644 deletions(-) create mode 100644 android/src/main/java/com/visionrtc/GlNullGenerator.kt create mode 100644 android/src/main/java/com/visionrtc/VisionRtcViewManager.kt create mode 100644 ios/NullGPUGenerator.swift create mode 100644 ios/VisionRTCRegistryBridge.h create mode 100644 ios/VisionRTCRegistryBridge.mm create mode 100644 ios/VisionRTCTrackRegistry.swift create mode 100644 ios/VisionRTCViewManager.m create mode 100644 src/vision-rtc-view.tsx diff --git a/README.md b/README.md index 4f3f81f..c7e76ec 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,37 @@ async function demo(reactTag: number) { // demo(findNodeHandle(cameraRef)); ``` +### API + +- **Functions** + - `createVisionCameraSource(viewTag: number)`: Links a native camera-like view to the library and returns a source you can stream from. + - `createWebRTCTrack(source, opts?)`: Creates a WebRTC video track from a source. You can pass simple options like `fps` and `resolution`. + - `replaceTrack(senderId: string, nextTrackId: string)`: Swaps the video track used by an existing WebRTC sender. + - `pauseTrack(trackId: string)`: Temporarily stops sending frames for that track (does not destroy it). + - `resumeTrack(trackId: string)`: Restarts frame sending for a paused track. + - `setTrackConstraints(trackId: string, opts)`: Changes track settings on the fly (for example, fps or resolution). + - `disposeTrack(trackId: string)`: Frees native resources for that track. + - `getStats()`: Returns basic runtime stats like `fps` and `droppedFrames` (if supported on the platform). + +- **Component** + - `VisionRTCView`: A native view that can render a given `trackId`. + - Props: + - `trackId?: string | null` + - `style?: ViewStyle | ViewStyle[]` + +- **Types** + - `TrackOptions`: Options for tracks. Common fields: + - `fps?: number` (frames per second) + - `resolution?: { width: number; height: number }` + - `bitrate?: number` + - `colorSpace?: 'auto' | 'sRGB' | 'BT.709' | 'BT.2020'` + - `orientationMode?: 'auto' | 'fixed-0' | 'fixed-90' | 'fixed-180' | 'fixed-270'` + - `mode?: 'null-gpu' | 'null-cpu' | 'external'` + - `VisionRTCTrack`: `{ trackId: string }` returned when you create a track. + - `VisionCameraSource`: Source handle returned by `createVisionCameraSource`. + - `NativePixelSource`: Low-level source if you already have native pixels (platform-specific shapes). + - `Resolution`: `{ width: number; height: number }`. + ## Contributing diff --git a/android/src/main/java/com/visionrtc/GlNullGenerator.kt b/android/src/main/java/com/visionrtc/GlNullGenerator.kt new file mode 100644 index 0000000..3226c28 --- /dev/null +++ b/android/src/main/java/com/visionrtc/GlNullGenerator.kt @@ -0,0 +1,97 @@ +package com.visionrtc + +import android.opengl.GLES20 +import android.os.SystemClock +import org.webrtc.EglBase +import org.webrtc.JavaI420Buffer +import org.webrtc.VideoFrame +import java.nio.ByteBuffer +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import kotlin.math.roundToInt + +class GlNullGenerator( + private val eglBase: EglBase, + private val observer: org.webrtc.CapturerObserver, + width: Int, + height: Int, + fps: Int, + private val onFps: (Int) -> Unit, +) { + @Volatile var width: Int = width; private set + @Volatile var height: Int = height; private set + @Volatile var fps: Int = fps; private set + + private val executor: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor() + private var scheduled: ScheduledFuture<*>? = null + @Volatile private var running: Boolean = false + private var framesThisSecond = 0 + private var lastSecondTs = SystemClock.elapsedRealtime() + + fun start() { + if (running) return + running = true + val periodNs = (1_000_000_000L / fps.toLong()) + scheduled = executor.scheduleAtFixedRate({ tick() }, 0L, periodNs, TimeUnit.NANOSECONDS) + } + + fun pause() { running = false } + fun resume() { running = true } + fun stop() { + running = false + scheduled?.cancel(true) + executor.shutdownNow() + } + + fun setResolution(w: Int, h: Int) { width = w; height = h } + fun setFps(next: Int) { + if (next <= 0 || next == fps) return + fps = next + if (running) { + scheduled?.cancel(false) + val periodNs = (1_000_000_000L / fps.toLong()) + scheduled = executor.scheduleAtFixedRate({ tick() }, 0L, periodNs, TimeUnit.NANOSECONDS) + } + } + + private fun tick() { + if (!running) return + val now = SystemClock.elapsedRealtime() + if (now - lastSecondTs >= 1000) { + onFps(framesThisSecond) + framesThisSecond = 0 + lastSecondTs = now + } + + val w = width + val h = height + val buffer = JavaI420Buffer.allocate(w, h) + val yPlane: ByteBuffer = buffer.dataY + val uPlane: ByteBuffer = buffer.dataU + val vPlane: ByteBuffer = buffer.dataV + val ts = (now % 5000).toInt() + + for (y in 0 until h) { + for (x in 0 until w) { + val yVal = (((x + ts / 10) % w).toFloat() / w * 255f).roundToInt().coerceIn(0, 255) + yPlane.put(y * buffer.strideY + x, yVal.toByte()) + } + } + val chromaW = (w + 1) / 2 + val chromaH = (h + 1) / 2 + for (y in 0 until chromaH) { + for (x in 0 until chromaW) { + uPlane.put(y * buffer.strideU + x, 128.toByte()) + vPlane.put(y * buffer.strideV + x, 128.toByte()) + } + } + + val timestampNs = org.webrtc.TimestampAligner.getRtcTimeNanos() + val frame = VideoFrame(buffer, 0, timestampNs) + observer.onFrameCaptured(frame) + frame.release() + framesThisSecond += 1 + } +} \ No newline at end of file diff --git a/android/src/main/java/com/visionrtc/VisionRtcModule.kt b/android/src/main/java/com/visionrtc/VisionRtcModule.kt index e8c9901..dad168d 100644 --- a/android/src/main/java/com/visionrtc/VisionRtcModule.kt +++ b/android/src/main/java/com/visionrtc/VisionRtcModule.kt @@ -48,7 +48,8 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp private data class TrackHandle( val source: VideoSource, val track: VideoTrack, - val capturer: GradientCapturer + val cpuCapturer: GradientCapturer?, + val glCapturer: GlNullGenerator? ) private val tracks: MutableMap = ConcurrentHashMap() @@ -70,6 +71,7 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp var width: Int = 1280 var height: Int = 720 var fps: Int = 30 + val mode: String = if (opts != null && opts.hasKey("mode")) opts.getString("mode") ?: "null-gpu" else "null-gpu" if (opts != null) { if (opts.hasKey("resolution")) { @@ -98,36 +100,53 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp val trackId = UUID.randomUUID().toString() val track = factory.createVideoTrack(trackId, videoSource) - val capturer = GradientCapturer(videoSource.capturerObserver, width, height, fps) { deliveredFps -> - lastReportedFps = deliveredFps + val useGl = mode == "null-gpu" + var cpuCap: GradientCapturer? = null + var glCap: GlNullGenerator? = null + if (useGl) { + glCap = GlNullGenerator(eglBase, videoSource.capturerObserver, width, height, fps) { deliveredFps -> + lastReportedFps = deliveredFps + } + glCap.start() + } else { + cpuCap = GradientCapturer(videoSource.capturerObserver, width, height, fps) { deliveredFps -> + lastReportedFps = deliveredFps + } + cpuCap.start() } - capturer.start() - tracks[trackId] = TrackHandle(videoSource, track, capturer) + tracks[trackId] = TrackHandle(videoSource, track, cpuCap, glCap) val result = com.facebook.react.bridge.Arguments.createMap() result.putString("trackId", trackId) promise.resolve(result) } + fun eglContext(): EglBase.Context = eglBase.eglBaseContext + + fun findTrack(trackId: String?): TrackHandle? { + if (trackId == null) return null + return tracks[trackId] + } + override fun replaceSenderTrack(senderId: String?, newTrackId: String?, promise: Promise) { promise.resolve(null) } override fun pauseTrack(trackId: String, promise: Promise) { - val cap = tracks[trackId]?.capturer - if (cap == null) { - promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return - } - cap.pause(); promise.resolve(null) + val handle = tracks[trackId] + if (handle == null) { promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return } + handle.cpuCapturer?.pause() + handle.glCapturer?.pause() + promise.resolve(null) } override fun resumeTrack(trackId: String, promise: Promise) { - val cap = tracks[trackId]?.capturer - if (cap == null) { - promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return - } - cap.resume(); promise.resolve(null) + val handle = tracks[trackId] + if (handle == null) { promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return } + handle.cpuCapturer?.resume() + handle.glCapturer?.resume() + promise.resolve(null) } override fun setTrackConstraints(trackId: String, opts: ReadableMap, promise: Promise) { @@ -137,7 +156,8 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp return } - val cap = handle.capturer + val cpuCap = handle.cpuCapturer + val glCap = handle.glCapturer var nextWidth: Int? = null var nextHeight: Int? = null @@ -163,19 +183,22 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp } if (nextWidth != null && nextHeight != null) { - cap.setResolution(nextWidth, nextHeight) + cpuCap?.setResolution(nextWidth, nextHeight) + glCap?.setResolution(nextWidth, nextHeight) } if (nextFps != null) { - cap.setFps(nextFps) + cpuCap?.setFps(nextFps) + glCap?.setFps(nextFps) } - targetFps = cap.fps + targetFps = cpuCap?.fps ?: (glCap?.fps ?: targetFps) promise.resolve(null) } override fun disposeTrack(trackId: String, promise: Promise) { tracks.remove(trackId)?.let { handle -> - handle.capturer.stop() + handle.cpuCapturer?.stop() + handle.glCapturer?.stop() handle.track.setEnabled(false) handle.track.dispose() handle.source.dispose() diff --git a/android/src/main/java/com/visionrtc/VisionRtcPackage.kt b/android/src/main/java/com/visionrtc/VisionRtcPackage.kt index 9a67191..6088512 100644 --- a/android/src/main/java/com/visionrtc/VisionRtcPackage.kt +++ b/android/src/main/java/com/visionrtc/VisionRtcPackage.kt @@ -3,6 +3,7 @@ package com.visionrtc import com.facebook.react.BaseReactPackage import com.facebook.react.bridge.NativeModule import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.uimanager.ViewManager import com.facebook.react.module.model.ReactModuleInfo import com.facebook.react.module.model.ReactModuleInfoProvider import java.util.HashMap @@ -30,4 +31,8 @@ class VisionRTCPackage : BaseReactPackage() { moduleInfos } } + + override fun createViewManagers(reactContext: ReactApplicationContext): MutableList> { + return mutableListOf(VisionRtcViewManager(reactContext)) + } } diff --git a/android/src/main/java/com/visionrtc/VisionRtcViewManager.kt b/android/src/main/java/com/visionrtc/VisionRtcViewManager.kt new file mode 100644 index 0000000..29b8d9d --- /dev/null +++ b/android/src/main/java/com/visionrtc/VisionRtcViewManager.kt @@ -0,0 +1,42 @@ +package com.visionrtc + +import android.content.Context +import android.view.View +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.module.annotations.ReactModule +import com.facebook.react.uimanager.SimpleViewManager +import com.facebook.react.uimanager.ThemedReactContext +import com.facebook.react.uimanager.annotations.ReactProp +import org.webrtc.SurfaceViewRenderer + +@ReactModule(name = VisionRtcViewManager.NAME) +class VisionRtcViewManager(private val context: ReactApplicationContext) : SimpleViewManager() { + + companion object { const val NAME = "VisionRTCView" } + + override fun getName(): String = NAME + + override fun createViewInstance(reactContext: ThemedReactContext): SurfaceViewRenderer { + val view = SurfaceViewRenderer(reactContext) + val eglBase = (reactContext.getNativeModule(VisionRTCModule::class.java) as? VisionRTCModule)?.let { it }?.let { it } // will init later + // We use a lazy global egl in VisionRTCModule; init when attaching track + view.setEnableHardwareScaler(true) + view.setMirror(false) + return view + } + + @ReactProp(name = "trackId") + fun setTrackId(view: SurfaceViewRenderer, trackId: String?) { + val module = view.context.applicationContext.let { (it as? Context)?.let { _ -> null } } + val mod = (context.getNativeModule(VisionRTCModule::class.java)) ?: return + val handle = mod.findTrack(trackId) + view.release() + view.init(mod.eglContext(), null) + handle?.track?.addSink(view) + } + + override fun onDropViewInstance(view: SurfaceViewRenderer) { + super.onDropViewInstance(view) + view.release() + } +} \ No newline at end of file diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 3380452..c2eaae7 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -2684,4 +2684,4 @@ SPEC CHECKSUMS: PODFILE CHECKSUM: 99f11ef725f3d8e854a790a3639f6f813387d8e8 -COCOAPODS: 1.15.2 +COCOAPODS: 1.16.2 diff --git a/example/ios/VisionRtcExample.xcodeproj/project.pbxproj b/example/ios/VisionRtcExample.xcodeproj/project.pbxproj index b0b1eb3..57c4f0a 100644 --- a/example/ios/VisionRtcExample.xcodeproj/project.pbxproj +++ b/example/ios/VisionRtcExample.xcodeproj/project.pbxproj @@ -257,6 +257,8 @@ isa = XCBuildConfiguration; baseConfigurationReference = 3B4392A12AC88292D35C810B /* Pods-VisionRtcExample.debug.xcconfig */; buildSettings = { + ENABLE_PREVIEWS = NO; + MACH_O_TYPE = mh_execute; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = 1; @@ -285,6 +287,8 @@ isa = XCBuildConfiguration; baseConfigurationReference = 5709B34CF0A7D63546082F79 /* Pods-VisionRtcExample.release.xcconfig */; buildSettings = { + ENABLE_PREVIEWS = NO; + MACH_O_TYPE = mh_execute; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = 1; diff --git a/example/src/App.tsx b/example/src/App.tsx index f131a80..ad4571f 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -1,10 +1,11 @@ import * as React from 'react'; -import {Button, Text, StyleSheet} from 'react-native'; +import {Button, Text, StyleSheet, View} from 'react-native'; import {SafeAreaView} from 'react-native-safe-area-context'; import { createWebRTCTrack, disposeTrack, getStats, + VisionRTCView, } from 'react-native-vision-rtc'; export default function App() { @@ -68,12 +69,19 @@ export default function App() { return ( -