@nk/vision camera

.cspell-wordlist.txt

@@ -95,6 +95,7 @@ Português
 codegen
 cstdint
 ocurred
+RNFS
 libfbjni
 libc
 gradlew

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
         run: yarn lint
 
       - name: Typecheck files
-        run: yarn typecheck
+        run: yarn workspaces foreach --all --topological-dev run prepare && yarn typecheck
 
   build-library:
     runs-on: ubuntu-latest
@@ -35,7 +35,5 @@ jobs:
       - name: Setup
         uses: ./.github/actions/setup
 
-      - name: Build package
-        run: |
-          cd packages/react-native-executorch
-          yarn prepare
+      - name: Build all packages
+        run: yarn workspaces foreach --all --topological-dev run prepare

.nvmrc

@@ -1 +1 @@
-v20
+v22

apps/computer-vision/app.json

@@ -18,7 +18,8 @@
       "bundleIdentifier": "com.anonymous.computervision",
       "infoPlist": {
         "NSCameraUsageDescription": "Process photo from camera"
-      }
+      },
+      "appleTeamId": "B357MU264T"
     },
     "android": {
       "adaptiveIcon": {
@@ -30,6 +31,17 @@
     "web": {
       "favicon": "./assets/icons/favicon.png"
     },
-    "plugins": ["expo-font", "expo-router"]
+    "plugins": [
+      "expo-font",
+      "expo-router",
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "minSdkVersion": 26
+          }
+        }
+      ]
+    ]
   }
 }

apps/computer-vision/app/_layout.tsx

@@ -1,4 +1,7 @@
 import { Drawer } from 'expo-router/drawer';
+import { initExecutorch } from 'react-native-executorch';
+import { ExpoResourceFetcher } from '@react-native-executorch/expo-resource-fetcher';
+
 import ColorPalette from '../colors';
 import React, { useState } from 'react';
 import { Text, StyleSheet, View } from 'react-native';
@@ -10,6 +13,10 @@ import {
 } from '@react-navigation/drawer';
 import { GeneratingContext } from '../context';
 
+initExecutorch({
+  resourceFetcher: ExpoResourceFetcher,
+});
+
 interface CustomDrawerProps extends DrawerContentComponentProps {
   isGenerating: boolean;
 }

apps/computer-vision/app/camera_object_detection/index.tsx

@@ -0,0 +1,265 @@
+import React, { useEffect, useMemo, useState } from 'react';
+import {
+  View,
+  StyleSheet,
+  Text,
+  ActivityIndicator,
+  TouchableOpacity,
+} from 'react-native';
+import {
+  Camera,
+  useCameraDevices,
+  getCameraFormat,
+  Templates,
+  useFrameProcessor,
+  useCameraPermission,
+  type Frame,
+} from 'react-native-vision-camera';
+import { useResizePlugin } from 'vision-camera-resize-plugin';
+import {
+  ObjectDetectionModule,
+  SSDLITE_320_MOBILENET_V3_LARGE,
+} from 'react-native-executorch';
+import ScreenWrapper from '../../ScreenWrapper';
+import ColorPalette from '../../colors';
+
+export default function CameraObjectDetectionScreen() {
+  // Model state
+  const detectionModel = useMemo(() => new ObjectDetectionModule(), []);
+  const [isModelReady, setIsModelReady] = useState(false);
+
+  // Screen dimensions
+
+  // Camera setup - V5 API
+  const devices = useCameraDevices();
+  const device = devices.find((d) => d.position === 'back') ?? devices[0];
+
+  const format = useMemo(() => {
+    if (device == null) return undefined;
+    return getCameraFormat(device, Templates.Video);
+  }, [device]);
+
+  const { hasPermission, requestPermission } = useCameraPermission();
+
+  // Resize plugin for efficient frame scaling
+  const { resize } = useResizePlugin();
+
+  // Load model
+  useEffect(() => {
+    (async () => {
+      try {
+        await detectionModel.load(SSDLITE_320_MOBILENET_V3_LARGE);
+        setIsModelReady(true);
+      } catch (error) {
+        console.error('Failed to load model:', error);
+      }
+    })();
+
+    return () => {
+      detectionModel.delete();
+    };
+  }, [detectionModel]);
+
+  // Frame processing with Vision Camera v4
+  const frameProcessor = useFrameProcessor(
+    (frame: Frame) => {
+      'worklet';
+
+      if (!isModelReady) {
+        return;
+      }
+
+      try {
+        // Use VisionCamera's resize plugin for better performance
+        const resized = resize(frame, {
+          scale: {
+            width: 640,
+            height: 640,
+          },
+          pixelFormat: 'rgb',
+          dataType: 'uint8',
+        });
+
+        // Prepare frame data for model
+        const frameData = {
+          data: resized.buffer,
+          width: 640,
+          height: 640,
+        };
+
+        // Run inference - generateFromFrame is JSI-bound for worklet compatibility
+        const result = detectionModel.generateFromFrame(frameData, 0.5);
+        console.log(result);
+      } catch (error: any) {
+        console.log(
+          'Frame processing error:',
+          error?.message || 'Unknown error'
+        );
+      }
+    },
+    [isModelReady, detectionModel, resize]
+  );
+
+  // Loading state
+  if (!isModelReady) {
+    return (
+      <ScreenWrapper>
+        <View style={styles.loadingContainer}>
+          <ActivityIndicator size="large" color={ColorPalette.strongPrimary} />
+          <Text style={styles.loadingText}>Loading model...</Text>
+        </View>
+      </ScreenWrapper>
+    );
+  }
+
+  // Permission request
+  if (!hasPermission) {
+    return (
+      <ScreenWrapper>
+        <View style={styles.permissionContainer}>
+          <Text style={styles.permissionText}>
+            Camera permission is required
+          </Text>
+          <TouchableOpacity
+            style={styles.permissionButton}
+            onPress={requestPermission}
+          >
+            <Text style={styles.permissionButtonText}>Grant Permission</Text>
+          </TouchableOpacity>
+        </View>
+      </ScreenWrapper>
+    );
+  }
+
+  // No camera device
+  if (device == null) {
+    return (
+      <ScreenWrapper>
+        <View style={styles.errorContainer}>
+          <Text style={styles.errorText}>No camera device found</Text>
+        </View>
+      </ScreenWrapper>
+    );
+  }
+
+  return (
+    <ScreenWrapper>
+      <View style={styles.container}>
+        {/* Camera View */}
+        <Camera
+          style={StyleSheet.absoluteFill}
+          device={device}
+          isActive={true}
+          frameProcessor={frameProcessor}
+          format={format}
+          pixelFormat="yuv"
+        />
+      </View>
+    </ScreenWrapper>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+  },
+  statsContainer: {
+    position: 'absolute',
+    top: 20,
+    right: 20,
+    flexDirection: 'row',
+    gap: 12,
+  },
+  statBox: {
+    backgroundColor: 'rgba(0, 0, 0, 0.7)',
+    paddingHorizontal: 16,
+    paddingVertical: 8,
+    borderRadius: 12,
+    alignItems: 'center',
+    minWidth: 70,
+  },
+  statLabel: {
+    color: '#888',
+    fontSize: 11,
+    fontWeight: '600',
+    textTransform: 'uppercase',
+  },
+  statValue: {
+    color: '#fff',
+    fontSize: 24,
+    fontWeight: 'bold',
+    marginTop: 2,
+  },
+  detectionList: {
+    position: 'absolute',
+    bottom: 20,
+    left: 20,
+    right: 20,
+    gap: 8,
+  },
+  detectionItem: {
+    backgroundColor: 'rgba(0, 0, 0, 0.8)',
+    paddingHorizontal: 16,
+    paddingVertical: 12,
+    borderRadius: 12,
+    flexDirection: 'row',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+    borderLeftWidth: 4,
+  },
+  detectionLabel: {
+    color: 'white',
+    fontSize: 16,
+    fontWeight: '600',
+    textTransform: 'capitalize',
+  },
+  detectionScore: {
+    color: '#4ECDC4',
+    fontSize: 16,
+    fontWeight: 'bold',
+  },
+  loadingContainer: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+  loadingText: {
+    marginTop: 16,
+    fontSize: 16,
+    color: ColorPalette.strongPrimary,
+  },
+  errorContainer: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+    padding: 20,
+  },
+  errorText: {
+    fontSize: 16,
+    color: '#d32f2f',
+    textAlign: 'center',
+  },
+  permissionContainer: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+    padding: 20,
+  },
+  permissionText: {
+    fontSize: 18,
+    color: ColorPalette.strongPrimary,
+    marginBottom: 20,
+    textAlign: 'center',
+  },
+  permissionButton: {
+    backgroundColor: ColorPalette.strongPrimary,
+    paddingHorizontal: 24,
+    paddingVertical: 12,
+    borderRadius: 8,
+  },
+  permissionButtonText: {
+    color: 'white',
+    fontSize: 16,
+    fontWeight: 'bold',
+  },
+});

apps/computer-vision/app/index.tsx

@@ -53,6 +53,12 @@ export default function Home() {
         >
           <Text style={styles.buttonText}>Image Generation</Text>
         </TouchableOpacity>
+        <TouchableOpacity
+          style={[styles.button, styles.cameraButton]}
+          onPress={() => router.navigate('camera_object_detection/')}
+        >
+          <Text style={styles.buttonText}>🎥 Camera Object Detection</Text>
+        </TouchableOpacity>
       </View>
     </View>
   );
@@ -92,6 +98,12 @@ const styles = StyleSheet.create({
     alignItems: 'center',
     marginBottom: 10,
   },
+  cameraButton: {
+    backgroundColor: '#2563eb',
+  },
+  testButton: {
+    backgroundColor: '#10b981',
+  },
   buttonText: {
     color: 'white',
     fontSize: fontSizes.md,