Feat: Implement GPU Generation Path

README.md

@@ -37,6 +37,37 @@ async function demo(reactTag: number) {
 // demo(findNodeHandle(cameraRef));
 ```
 
+### API
+
+- **Functions**
+  - `createVisionCameraSource(viewTag: number)`: Links a native camera-like view to the library and returns a source you can stream from.
+  - `createWebRTCTrack(source, opts?)`: Creates a WebRTC video track from a source. You can pass simple options like `fps` and `resolution`.
+  - `replaceTrack(senderId: string, nextTrackId: string)`: Swaps the video track used by an existing WebRTC sender.
+  - `pauseTrack(trackId: string)`: Temporarily stops sending frames for that track (does not destroy it).
+  - `resumeTrack(trackId: string)`: Restarts frame sending for a paused track.
+  - `setTrackConstraints(trackId: string, opts)`: Changes track settings on the fly (for example, fps or resolution).
+  - `disposeTrack(trackId: string)`: Frees native resources for that track.
+  - `getStats()`: Returns basic runtime stats like `fps` and `droppedFrames` (if supported on the platform).
+
+- **Component**
+  - `VisionRTCView`: A native view that can render a given `trackId`.
+    - Props:
+      - `trackId?: string | null`
+      - `style?: ViewStyle | ViewStyle[]`
+
+- **Types**
+  - `TrackOptions`: Options for tracks. Common fields:
+    - `fps?: number` (frames per second)
+    - `resolution?: { width: number; height: number }`
+    - `bitrate?: number`
+    - `colorSpace?: 'auto' | 'sRGB' | 'BT.709' | 'BT.2020'`
+    - `orientationMode?: 'auto' | 'fixed-0' | 'fixed-90' | 'fixed-180' | 'fixed-270'`
+    - `mode?: 'null-gpu' | 'null-cpu' | 'external'`
+  - `VisionRTCTrack`: `{ trackId: string }` returned when you create a track.
+  - `VisionCameraSource`: Source handle returned by `createVisionCameraSource`.
+  - `NativePixelSource`: Low-level source if you already have native pixels (platform-specific shapes).
+  - `Resolution`: `{ width: number; height: number }`.
+
 
 ## Contributing
 

android/src/main/java/com/visionrtc/GlNullGenerator.kt

@@ -0,0 +1,97 @@
+package com.visionrtc
+
+import android.opengl.GLES20
+import android.os.SystemClock
+import org.webrtc.EglBase
+import org.webrtc.JavaI420Buffer
+import org.webrtc.VideoFrame
+import java.nio.ByteBuffer
+import java.util.concurrent.Executors
+import java.util.concurrent.ScheduledExecutorService
+import java.util.concurrent.ScheduledFuture
+import java.util.concurrent.TimeUnit
+import kotlin.math.roundToInt
+
+class GlNullGenerator(
+  private val eglBase: EglBase,
+  private val observer: org.webrtc.CapturerObserver,
+  width: Int,
+  height: Int,
+  fps: Int,
+  private val onFps: (Int) -> Unit,
+) {
+  @Volatile var width: Int = width; private set
+  @Volatile var height: Int = height; private set
+  @Volatile var fps: Int = fps; private set
+
+  private val executor: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor()
+  private var scheduled: ScheduledFuture<*>? = null
+  @Volatile private var running: Boolean = false
+  private var framesThisSecond = 0
+  private var lastSecondTs = SystemClock.elapsedRealtime()
+
+  fun start() {
+    if (running) return
+    running = true
+    val periodNs = (1_000_000_000L / fps.toLong())
+    scheduled = executor.scheduleAtFixedRate({ tick() }, 0L, periodNs, TimeUnit.NANOSECONDS)
+  }
+
+  fun pause() { running = false }
+  fun resume() { running = true }
+  fun stop() {
+    running = false
+    scheduled?.cancel(true)
+    executor.shutdownNow()
+  }
+
+  fun setResolution(w: Int, h: Int) { width = w; height = h }
+  fun setFps(next: Int) {
+    if (next <= 0 || next == fps) return
+    fps = next
+    if (running) {
+      scheduled?.cancel(false)
+      val periodNs = (1_000_000_000L / fps.toLong())
+      scheduled = executor.scheduleAtFixedRate({ tick() }, 0L, periodNs, TimeUnit.NANOSECONDS)
+    }
+  }
+
+  private fun tick() {
+    if (!running) return
+    val now = SystemClock.elapsedRealtime()
+    if (now - lastSecondTs >= 1000) {
+      onFps(framesThisSecond)
+      framesThisSecond = 0
+      lastSecondTs = now
+    }
+
+    val w = width
+    val h = height
+    val buffer = JavaI420Buffer.allocate(w, h)
+    val yPlane: ByteBuffer = buffer.dataY
+    val uPlane: ByteBuffer = buffer.dataU
+    val vPlane: ByteBuffer = buffer.dataV
+    val ts = (now % 5000).toInt()
+
+    for (y in 0 until h) {
+      for (x in 0 until w) {
+        val yVal = (((x + ts / 10) % w).toFloat() / w * 255f).roundToInt().coerceIn(0, 255)
+        yPlane.put(y * buffer.strideY + x, yVal.toByte())
+      }
+    }
+    val chromaW = (w + 1) / 2
+    val chromaH = (h + 1) / 2
+    for (y in 0 until chromaH) {
+      for (x in 0 until chromaW) {
+        uPlane.put(y * buffer.strideU + x, 128.toByte())
+        vPlane.put(y * buffer.strideV + x, 128.toByte())
+      }
+    }
+
+    val timestampNs = org.webrtc.TimestampAligner.getRtcTimeNanos()
+    val frame = VideoFrame(buffer, 0, timestampNs)
+    observer.onFrameCaptured(frame)
+    frame.release()
+    framesThisSecond += 1
+  }
+}

android/src/main/java/com/visionrtc/VisionRtcModule.kt

@@ -48,7 +48,8 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp
   private data class TrackHandle(
     val source: VideoSource,
     val track: VideoTrack,
-    val capturer: GradientCapturer
+    val cpuCapturer: GradientCapturer?,
+    val glCapturer: GlNullGenerator?
   )
 
   private val tracks: MutableMap<String, TrackHandle> = ConcurrentHashMap()
@@ -70,6 +71,7 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp
     var width: Int = 1280
     var height: Int = 720
     var fps: Int = 30
+    val mode: String = if (opts != null && opts.hasKey("mode")) opts.getString("mode") ?: "null-gpu" else "null-gpu"
 
     if (opts != null) {
       if (opts.hasKey("resolution")) {
@@ -98,36 +100,53 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp
     val trackId = UUID.randomUUID().toString()
     val track = factory.createVideoTrack(trackId, videoSource)
 
-    val capturer = GradientCapturer(videoSource.capturerObserver, width, height, fps) { deliveredFps ->
-      lastReportedFps = deliveredFps
+    val useGl = mode == "null-gpu"
+    var cpuCap: GradientCapturer? = null
+    var glCap: GlNullGenerator? = null
+    if (useGl) {
+      glCap = GlNullGenerator(eglBase, videoSource.capturerObserver, width, height, fps) { deliveredFps ->
+        lastReportedFps = deliveredFps
+      }
+      glCap.start()
+    } else {
+      cpuCap = GradientCapturer(videoSource.capturerObserver, width, height, fps) { deliveredFps ->
+        lastReportedFps = deliveredFps
+      }
+      cpuCap.start()
     }
-    capturer.start()
 
-    tracks[trackId] = TrackHandle(videoSource, track, capturer)
+    tracks[trackId] = TrackHandle(videoSource, track, cpuCap, glCap)
 
     val result = com.facebook.react.bridge.Arguments.createMap()
     result.putString("trackId", trackId)
     promise.resolve(result)
   }
 
+  fun eglContext(): EglBase.Context = eglBase.eglBaseContext
+
+  fun findTrack(trackId: String?): TrackHandle? {
+    if (trackId == null) return null
+    return tracks[trackId]
+  }
+
   override fun replaceSenderTrack(senderId: String?, newTrackId: String?, promise: Promise) {
     promise.resolve(null)
   }
 
   override fun pauseTrack(trackId: String, promise: Promise) {
-    val cap = tracks[trackId]?.capturer
-    if (cap == null) {
-      promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return
-   }
-    cap.pause(); promise.resolve(null)
+    val handle = tracks[trackId]
+    if (handle == null) { promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return }
+    handle.cpuCapturer?.pause()
+    handle.glCapturer?.pause()
+    promise.resolve(null)
   }
 
   override fun resumeTrack(trackId: String, promise: Promise) {
-    val cap = tracks[trackId]?.capturer
-    if (cap == null) {
-      promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return
-    }
-    cap.resume(); promise.resolve(null)
+    val handle = tracks[trackId]
+    if (handle == null) { promise.reject("ERR_UNKNOWN_TRACK", "Unknown trackId: $trackId"); return }
+    handle.cpuCapturer?.resume()
+    handle.glCapturer?.resume()
+    promise.resolve(null)
   }
 
   override fun setTrackConstraints(trackId: String, opts: ReadableMap, promise: Promise) {
@@ -137,7 +156,8 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp
       return
     }
 
-    val cap = handle.capturer
+    val cpuCap = handle.cpuCapturer
+    val glCap = handle.glCapturer
 
     var nextWidth: Int? = null
     var nextHeight: Int? = null
@@ -163,19 +183,22 @@ class VisionRTCModule(reactContext: ReactApplicationContext) : NativeVisionRtcSp
     }
 
     if (nextWidth != null && nextHeight != null) {
-      cap.setResolution(nextWidth, nextHeight)
+      cpuCap?.setResolution(nextWidth, nextHeight)
+      glCap?.setResolution(nextWidth, nextHeight)
     }
     if (nextFps != null) {
-      cap.setFps(nextFps)
+      cpuCap?.setFps(nextFps)
+      glCap?.setFps(nextFps)
     }
 
-    targetFps = cap.fps
+    targetFps = cpuCap?.fps ?: (glCap?.fps ?: targetFps)
     promise.resolve(null)
   }
 
   override fun disposeTrack(trackId: String, promise: Promise) {
     tracks.remove(trackId)?.let { handle ->
-      handle.capturer.stop()
+      handle.cpuCapturer?.stop()
+      handle.glCapturer?.stop()
       handle.track.setEnabled(false)
       handle.track.dispose()
       handle.source.dispose()

android/src/main/java/com/visionrtc/VisionRtcPackage.kt

@@ -3,6 +3,7 @@ package com.visionrtc
 import com.facebook.react.BaseReactPackage
 import com.facebook.react.bridge.NativeModule
 import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.uimanager.ViewManager
 import com.facebook.react.module.model.ReactModuleInfo
 import com.facebook.react.module.model.ReactModuleInfoProvider
 import java.util.HashMap
@@ -30,4 +31,8 @@ class VisionRTCPackage : BaseReactPackage() {
       moduleInfos
     }
   }
+
+  override fun createViewManagers(reactContext: ReactApplicationContext): MutableList<ViewManager<*, *>> {
+    return mutableListOf(VisionRtcViewManager(reactContext))
+  }
 }

android/src/main/java/com/visionrtc/VisionRtcViewManager.kt

@@ -0,0 +1,42 @@
+package com.visionrtc
+
+import android.content.Context
+import android.view.View
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.module.annotations.ReactModule
+import com.facebook.react.uimanager.SimpleViewManager
+import com.facebook.react.uimanager.ThemedReactContext
+import com.facebook.react.uimanager.annotations.ReactProp
+import org.webrtc.SurfaceViewRenderer
+
+@ReactModule(name = VisionRtcViewManager.NAME)
+class VisionRtcViewManager(private val context: ReactApplicationContext) : SimpleViewManager<SurfaceViewRenderer>() {
+
+  companion object { const val NAME = "VisionRTCView" }
+
+  override fun getName(): String = NAME
+
+  override fun createViewInstance(reactContext: ThemedReactContext): SurfaceViewRenderer {
+    val view = SurfaceViewRenderer(reactContext)
+    val eglBase = (reactContext.getNativeModule(VisionRTCModule::class.java) as? VisionRTCModule)?.let { it }?.let { it } // will init later
+    // We use a lazy global egl in VisionRTCModule; init when attaching track
+    view.setEnableHardwareScaler(true)
+    view.setMirror(false)
+    return view
+  }
+
+  @ReactProp(name = "trackId")
+  fun setTrackId(view: SurfaceViewRenderer, trackId: String?) {
+    val module = view.context.applicationContext.let { (it as? Context)?.let { _ -> null } }
+    val mod = (context.getNativeModule(VisionRTCModule::class.java)) ?: return
+    val handle = mod.findTrack(trackId)
+    view.release()
+    view.init(mod.eglContext(), null)
+    handle?.track?.addSink(view)
+  }
+
+  override fun onDropViewInstance(view: SurfaceViewRenderer) {
+    super.onDropViewInstance(view)
+    view.release()
+  }
+}

example/ios/Podfile.lock

@@ -2684,4 +2684,4 @@ SPEC CHECKSUMS:
 
 PODFILE CHECKSUM: 99f11ef725f3d8e854a790a3639f6f813387d8e8
 
-COCOAPODS: 1.15.2
+COCOAPODS: 1.16.2

example/ios/VisionRtcExample.xcodeproj/project.pbxproj

@@ -257,6 +257,8 @@
 			isa = XCBuildConfiguration;
 			baseConfigurationReference = 3B4392A12AC88292D35C810B /* Pods-VisionRtcExample.debug.xcconfig */;
 			buildSettings = {
+				ENABLE_PREVIEWS = NO;
+				MACH_O_TYPE = mh_execute;
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_ENABLE_MODULES = YES;
 				CURRENT_PROJECT_VERSION = 1;
@@ -285,6 +287,8 @@
 			isa = XCBuildConfiguration;
 			baseConfigurationReference = 5709B34CF0A7D63546082F79 /* Pods-VisionRtcExample.release.xcconfig */;
 			buildSettings = {
+				ENABLE_PREVIEWS = NO;
+				MACH_O_TYPE = mh_execute;
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_ENABLE_MODULES = YES;
 				CURRENT_PROJECT_VERSION = 1;