Apriltag ml experimental

photon-client/src/components/dashboard/tabs/AprilTagTab.vue

@@ -5,19 +5,50 @@ import PvSlider from "@/components/common/pv-slider.vue";
 import PvSwitch from "@/components/common/pv-switch.vue";
 import { computed } from "vue";
 import { useStateStore } from "@/stores/StateStore";
-import type { ActivePipelineSettings } from "@/types/PipelineTypes";
+import type { AprilTagPipelineSettings } from "@/types/PipelineTypes";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
+import type { ObjectDetectionModelProperties } from "@/types/SettingTypes";
 import { useDisplay } from "vuetify";
 
 // TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
 // Defer reference to store access method
-const currentPipelineSettings = computed<ActivePipelineSettings>(
-  () => useCameraSettingsStore().currentPipelineSettings
+const currentPipelineSettings = computed<AprilTagPipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings as AprilTagPipelineSettings
 );
 const { mdAndDown } = useDisplay();
 const interactiveCols = computed(() =>
   mdAndDown.value && (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode) ? 8 : 7
 );
+
+// Check if ML detection is available on this platform
+const mlDetectionAvailable = computed(() => useSettingsStore().general.supportedBackends.length > 0);
+
+// Filter models to only those supported by available backends
+const supportedModels = computed<ObjectDetectionModelProperties[]>(() => {
+  const { availableModels, supportedBackends } = useSettingsStore().general;
+  const isSupported = (model: ObjectDetectionModelProperties) => {
+    return supportedBackends.some((backend: string) => backend.toLowerCase() === model.family.toLowerCase());
+  };
+  return availableModels.filter(isSupported);
+});
+
+const selectedModel = computed({
+  get: () => {
+    const currentModelName = currentPipelineSettings.value.mlModelName;
+    if (!currentModelName) return undefined;
+
+    const index = supportedModels.value.findIndex((model) => model.nickname === currentModelName);
+    return index === -1 ? undefined : index;
+  },
+
+  set: (v) => {
+    if (v !== undefined && v >= 0 && v < supportedModels.value.length) {
+      const newModel = supportedModels.value[v];
+      useCameraSettingsStore().changeCurrentPipelineSetting({ mlModelName: newModel.nickname }, true);
+    }
+  }
+});
 </script>
 
 <template>
@@ -88,5 +119,83 @@ const interactiveCols = computed(() =>
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ refineEdges: value }, false)
       "
     />
+
+    <!-- ML-Assisted Detection Section -->
+    <v-divider class="mt-3 mb-2" v-if="mlDetectionAvailable" />
+    <div v-if="mlDetectionAvailable" class="ml-settings-section">
+      <p class="text-subtitle-2 mb-2">AI-Assisted Detection (NPU)</p>
+      <pv-switch
+        v-model="currentPipelineSettings.useMLDetection"
+        :switch-cols="interactiveCols"
+        label="Enable AI Detection"
+        tooltip="Use NPU-accelerated ML model for faster AprilTag detection. Requires compatible hardware (RK3588 or QCS6490)"
+        @update:modelValue="
+          (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ useMLDetection: value }, false)
+        "
+      />
+      <div v-if="currentPipelineSettings.useMLDetection">
+        <pv-select
+          v-model="selectedModel"
+          label="Model"
+          tooltip="The model used for AI-assisted AprilTag detection"
+          :select-cols="interactiveCols"
+          :items="supportedModels.map((model) => model.nickname)"
+        />
+        <pv-slider
+          v-model="currentPipelineSettings.mlConfidenceThreshold"
+          :slider-cols="interactiveCols"
+          label="Confidence Threshold"
+          tooltip="Minimum confidence score for ML detection (0-1). Higher values reduce false positives"
+          :min="0.1"
+          :max="1.0"
+          :step="0.05"
+          @update:modelValue="
+            (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ mlConfidenceThreshold: value }, false)
+          "
+        />
+        <pv-slider
+          v-model="currentPipelineSettings.mlNmsThreshold"
+          :slider-cols="interactiveCols"
+          label="NMS Threshold"
+          tooltip="Non-maximum suppression threshold for overlapping detections (0-1)"
+          :min="0.1"
+          :max="1.0"
+          :step="0.05"
+          @update:modelValue="
+            (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ mlNmsThreshold: value }, false)
+          "
+        />
+        <pv-slider
+          v-model="currentPipelineSettings.mlRoiPaddingPixels"
+          :slider-cols="interactiveCols"
+          label="ROI Padding (px)"
+          tooltip="Pixels of padding added around each detected region. Naturally adapts: small/far tags get more relative expansion, large/close tags get less"
+          :min="10"
+          :max="150"
+          :step="5"
+          @update:modelValue="
+            (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ mlRoiPaddingPixels: value }, false)
+          "
+        />
+        <pv-switch
+          v-model="currentPipelineSettings.mlFallbackToTraditional"
+          :switch-cols="interactiveCols"
+          label="Fallback to Traditional"
+          tooltip="If ML detection finds no tags, fall back to traditional full-frame detection"
+          @update:modelValue="
+            (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ mlFallbackToTraditional: value }, false)
+          "
+        />
+        <pv-switch
+          v-model="currentPipelineSettings.showDetectionBoxes"
+          :switch-cols="interactiveCols"
+          label="Show ROI Boxes"
+          tooltip="Draw the ML model's detected bounding boxes on the processed image for tuning visualization"
+          @update:modelValue="
+            (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ showDetectionBoxes: value }, false)
+          "
+        />
+      </div>
+    </div>
   </div>
 </template>

photon-client/src/components/dashboard/tabs/OutputTab.vue

@@ -3,6 +3,7 @@ import PvSelect from "@/components/common/pv-select.vue";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { type ActivePipelineSettings, PipelineType, RobotOffsetPointMode } from "@/types/PipelineTypes";
 import PvSwitch from "@/components/common/pv-switch.vue";
+import PvSlider from "@/components/common/pv-slider.vue";
 import { computed } from "vue";
 import { RobotOffsetType } from "@/types/SettingTypes";
 import { useStateStore } from "@/stores/StateStore";
@@ -58,14 +59,17 @@ const interactiveCols = computed(() =>
 
 <template>
   <div>
-    <pv-switch
-      v-model="useCameraSettingsStore().currentPipelineSettings.outputShowMultipleTargets"
-      label="Show Multiple Targets"
-      tooltip="If enabled, up to five targets will be displayed and sent via PhotonLib, instead of just one"
-      :disabled="isTagPipeline"
+    <pv-slider
+      v-model="useCameraSettingsStore().currentPipelineSettings.outputMaximumTargets"
+      label="Maximum Targets"
+      tooltip="The maximum number of targets to display and send."
+      :hidden="isTagPipeline"
+      :min="1"
+      :max="127"
+      :step="1"
       :switch-cols="interactiveCols"
       @update:modelValue="
-        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ outputShowMultipleTargets: value }, false)
+        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ outputMaximumTargets: value }, false)
       "
     />
     <pv-switch

photon-client/src/types/PipelineTypes.ts

@@ -66,7 +66,7 @@ export interface PipelineSettings {
   hsvHue: WebsocketNumberPair | [number, number];
   ledMode: boolean;
   hueInverted: boolean;
-  outputShowMultipleTargets: boolean;
+  outputMaximumTargets: number;
   contourSortMode: number;
   cameraExposureRaw: number;
   cameraMinExposureRaw: number;
@@ -108,7 +108,7 @@ export type ConfigurablePipelineSettings = Partial<
 // Omitted settings are changed for all pipeline types
 export const DefaultPipelineSettings: Omit<
   PipelineSettings,
-  "cameraGain" | "targetModel" | "ledMode" | "outputShowMultipleTargets" | "cameraExposureRaw" | "pipelineType"
+  "cameraGain" | "targetModel" | "ledMode" | "cameraExposureRaw" | "pipelineType"
 > = {
   offsetRobotOffsetMode: RobotOffsetPointMode.None,
   streamingFrameDivisor: 0,
@@ -137,6 +137,7 @@ export const DefaultPipelineSettings: Omit<
   offsetDualPointB: { x: 0, y: 0 },
   hsvHue: { first: 50, second: 180 },
   hueInverted: false,
+  outputMaximumTargets: 20,
   contourSortMode: 0,
   offsetSinglePoint: { x: 0, y: 0 },
   cameraBrightness: 50,
@@ -166,7 +167,7 @@ export const DefaultReflectivePipelineSettings: ReflectivePipelineSettings = {
   cameraGain: 20,
   targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
   ledMode: true,
-  outputShowMultipleTargets: false,
+  outputMaximumTargets: 20,
   cameraExposureRaw: 6,
   pipelineType: PipelineType.Reflective,
 
@@ -197,7 +198,7 @@ export const DefaultColoredShapePipelineSettings: ColoredShapePipelineSettings =
   cameraGain: 75,
   targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
   ledMode: true,
-  outputShowMultipleTargets: false,
+  outputMaximumTargets: 20,
   cameraExposureRaw: 20,
   pipelineType: PipelineType.ColoredShape,
 
@@ -227,6 +228,14 @@ export interface AprilTagPipelineSettings extends PipelineSettings {
   tagFamily: AprilTagFamily;
   doMultiTarget: boolean;
   doSingleTargetAlways: boolean;
+  // ML-assisted detection settings
+  useMLDetection: boolean;
+  mlConfidenceThreshold: number;
+  mlNmsThreshold: number;
+  mlRoiPaddingPixels: number;
+  mlFallbackToTraditional: boolean;
+  mlModelName: string | null;
+  showDetectionBoxes: boolean;
 }
 export type ConfigurableAprilTagPipelineSettings = Partial<
   Omit<AprilTagPipelineSettings, "pipelineType" | "hammingDist" | "debug">
@@ -237,10 +246,9 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = {
   cameraGain: 75,
   targetModel: TargetModel.AprilTag6p5in_36h11,
   ledMode: false,
-  outputShowMultipleTargets: true,
+  outputMaximumTargets: 127,
   cameraExposureRaw: 20,
   pipelineType: PipelineType.AprilTag,
-
   hammingDist: 0,
   numIterations: 40,
   decimate: 1,
@@ -251,7 +259,15 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = {
   threads: 4,
   tagFamily: AprilTagFamily.Family36h11,
   doMultiTarget: false,
-  doSingleTargetAlways: false
+  doSingleTargetAlways: false,
+  // ML-assisted detection defaults
+  useMLDetection: false,
+  mlConfidenceThreshold: 0.5,
+  mlNmsThreshold: 0.45,
+  mlRoiPaddingPixels: 40,
+  mlFallbackToTraditional: true,
+  mlModelName: null,
+  showDetectionBoxes: true
 };
 
 export interface ArucoPipelineSettings extends PipelineSettings {
@@ -278,13 +294,12 @@ export type ConfigurableArucoPipelineSettings = Partial<Omit<ArucoPipelineSettin
 export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
   ...DefaultPipelineSettings,
   cameraGain: 75,
-  outputShowMultipleTargets: true,
+  outputMaximumTargets: 127,
   targetModel: TargetModel.AprilTag6p5in_36h11,
   cameraExposureRaw: -1,
   cameraAutoExposure: true,
   ledMode: false,
   pipelineType: PipelineType.Aruco,
-
   tagFamily: AprilTagFamily.Family36h11,
   threshWinSizes: { first: 11, second: 91 },
   threshStepSize: 40,
@@ -316,7 +331,7 @@ export const DefaultObjectDetectionPipelineSettings: ObjectDetectionPipelineSett
   cameraGain: 20,
   targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
   ledMode: true,
-  outputShowMultipleTargets: false,
+  outputMaximumTargets: 20,
   cameraExposureRaw: 6,
   confidence: 0.9,
   nms: 0.45,
@@ -335,7 +350,7 @@ export const DefaultCalibration3dPipelineSettings: Calibration3dPipelineSettings
   cameraGain: 20,
   targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
   ledMode: true,
-  outputShowMultipleTargets: false,
+  outputMaximumTargets: 1,
   cameraExposureRaw: 6,
   drawAllSnapshots: false
 };

photon-client/src/views/CameraMatchingView.vue

@@ -24,24 +24,28 @@ const activatingModule = ref(false);
 const activateModule = (moduleUniqueName: string) => {
   if (activatingModule.value) return;
   activatingModule.value = true;
-
   axiosPost("/utils/activateMatchedCamera", "activate a matched camera", {
     cameraUniqueName: moduleUniqueName
-  }).finally(() => (activatingModule.value = false));
+  })
+    .then(() => {
+      // Reload page to ensure UI shows correct camera controls
+      window.location.reload();
+    })
+    .finally(() => (activatingModule.value = false));
 };
 
 const assigningCamera = ref(false);
 const assignCamera = (cameraInfo: PVCameraInfo) => {
   if (assigningCamera.value) return;
   assigningCamera.value = true;
-
-  const payload = {
+  axiosPost("/utils/assignUnmatchedCamera", "assign an unmatched camera", {
     cameraInfo: cameraInfo
-  };
-
-  axiosPost("/utils/assignUnmatchedCamera", "assign an unmatched camera", payload).finally(
-    () => (assigningCamera.value = false)
-  );
+  })
+    .then(() => {
+      // Reload page to ensure UI shows correct camera controls
+      window.location.reload();
+    })
+    .finally(() => (assigningCamera.value = false));
 };
 
 const deactivatingModule = ref(false);

photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java

@@ -311,6 +311,56 @@ public Optional<Model> getDefaultModel() {
         return models.get(supportedBackends.get(0)).stream().findFirst();
     }
 
+    /**
+     * Gets the default AprilTag detection model. Searches for a model with "apriltag" in its
+     * nickname (case-insensitive).
+     *
+     * @return Optional containing the AprilTag model if found
+     */
+    public Optional<Model> getDefaultAprilTagModel() {
+        if (models == null || supportedBackends.isEmpty()) {
+            return Optional.empty();
+        }
+
+        for (Family backend : supportedBackends) {
+            if (models.containsKey(backend)) {
+                Optional<Model> model =
+                        models.get(backend).stream()
+                                .filter(m -> m.getNickname().toLowerCase().contains("apriltag"))
+                                .findFirst();
+                if (model.isPresent()) {
+                    return model;
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
+    /**
+     * Gets a model by its exact nickname.
+     *
+     * @param name The nickname of the model to retrieve
+     * @return Optional containing the model if found
+     */
+    public Optional<Model> getModelByName(String name) {
+        if (models == null || supportedBackends.isEmpty() || name == null) {
+            return Optional.empty();
+        }
+
+        for (Family backend : supportedBackends) {
+            if (models.containsKey(backend)) {
+                Optional<Model> model =
+                        models.get(backend).stream()
+                                .filter(m -> m.getNickname().equals(name))
+                                .findFirst();
+                if (model.isPresent()) {
+                    return model;
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
     // Do checking later on, when we create the model object
     private void loadModel(Path path) {
         if (models == null) {