fix: replace fal Florence-2 face detection with Gemini vision

Florence-2 object detection had false negatives on AI-generated face
images. Replaces it with a Gemini 2.5 Flash vision call via the Recoup
Chat API, which can reliably determine if an image is a portrait/headshot.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: sweetmantech

src/content/__tests__/detectFace.test.ts

@@ -4,108 +4,71 @@ vi.mock("../../sandboxes/logStep", () => ({
   logStep: vi.fn(),
 }));
 
-const mockFalSubscribe = vi.fn();
-vi.mock("../falSubscribe", () => ({
-  falSubscribe: (...args: unknown[]) => mockFalSubscribe(...args),
-}));
+const mockFetch = vi.fn();
+vi.stubGlobal("fetch", mockFetch);
 
 import { detectFace } from "../detectFace";
 
+function mockChatResponse(text: string) {
+  return {
+    ok: true,
+    json: () => Promise.resolve({ text }),
+  };
+}
+
 describe("detectFace", () => {
   beforeEach(() => {
     vi.clearAllMocks();
+    process.env.RECOUP_API_KEY = "test-key";
   });
 
-  it("returns true when a person label is detected", async () => {
-    mockFalSubscribe.mockResolvedValue({
-      data: {
-        results: {
-          bboxes: [[10, 20, 100, 200]],
-          labels: ["person"],
-        },
-      },
-    });
+  it("returns true when the model says the image contains a face", async () => {
+    mockFetch.mockResolvedValue(mockChatResponse("true"));
 
     const result = await detectFace("https://example.com/headshot.png");
 
-    expect(result).toBe(true);
-    expect(mockFalSubscribe).toHaveBeenCalledWith(
-      "fal-ai/florence-2-large/object-detection",
-      { image_url: "https://example.com/headshot.png" },
-    );
-  });
-
-  it("returns true when a face label is detected among other objects", async () => {
-    mockFalSubscribe.mockResolvedValue({
-      data: {
-        results: {
-          bboxes: [[0, 0, 50, 50], [10, 20, 100, 200]],
-          labels: ["chair", "human face"],
-        },
-      },
-    });
-
-    const result = await detectFace("https://example.com/photo.png");
-
     expect(result).toBe(true);
   });
 
-  it("returns false when no person or face labels are detected", async () => {
-    mockFalSubscribe.mockResolvedValue({
-      data: {
-        results: {
-          bboxes: [[0, 0, 300, 300]],
-          labels: ["album cover"],
-        },
-      },
-    });
+  it("returns false when the model says no face is present", async () => {
+    mockFetch.mockResolvedValue(mockChatResponse("false"));
 
     const result = await detectFace("https://example.com/album-cover.png");
 
     expect(result).toBe(false);
   });
 
-  it("returns false when results are empty", async () => {
-    mockFalSubscribe.mockResolvedValue({
-      data: {
-        results: {
-          bboxes: [],
-          labels: [],
-        },
-      },
-    });
+  it("sends the image URL in the prompt to the chat API", async () => {
+    mockFetch.mockResolvedValue(mockChatResponse("true"));
 
-    const result = await detectFace("https://example.com/blank.png");
+    await detectFace("https://example.com/photo.png");
 
-    expect(result).toBe(false);
+    expect(mockFetch).toHaveBeenCalledOnce();
+    const [url, options] = mockFetch.mock.calls[0];
+    expect(url).toContain("/api/chat/generate");
+    const body = JSON.parse(options.body);
+    expect(body.prompt).toContain("https://example.com/photo.png");
   });
 
-  it("returns false when detection fails", async () => {
-    mockFalSubscribe.mockRejectedValue(new Error("Detection failed"));
+  it("returns false when API call fails", async () => {
+    mockFetch.mockRejectedValue(new Error("Network error"));
 
     const result = await detectFace("https://example.com/broken.png");
 
     expect(result).toBe(false);
   });
 
-  it("does not false-positive on labels containing face words as substrings", async () => {
-    mockFalSubscribe.mockResolvedValue({
-      data: {
-        results: {
-          bboxes: [[0, 0, 200, 200]],
-          labels: ["ottoman", "mannequin", "womanizer"],
-        },
-      },
-    });
+  it("returns false when API returns non-ok response", async () => {
+    mockFetch.mockResolvedValue({ ok: false, status: 500 });
 
-    const result = await detectFace("https://example.com/furniture.png");
+    const result = await detectFace("https://example.com/broken.png");
 
     expect(result).toBe(false);
   });
 
   it("logs the error when detection fails", async () => {
     const { logStep } = await import("../../sandboxes/logStep");
-    mockFalSubscribe.mockRejectedValue(new Error("Rate limit exceeded"));
+    mockFetch.mockRejectedValue(new Error("Rate limit exceeded"));
 
     await detectFace("https://example.com/broken.png");
 
@@ -115,4 +78,12 @@ describe("detectFace", () => {
       expect.objectContaining({ error: "Rate limit exceeded" }),
     );
   });
+
+  it("handles whitespace and casing in model response", async () => {
+    mockFetch.mockResolvedValue(mockChatResponse("  True  "));
+
+    const result = await detectFace("https://example.com/headshot.png");
+
+    expect(result).toBe(true);
+  });
 });

src/content/detectFace.ts

@@ -1,34 +1,49 @@
 import { logStep } from "../sandboxes/logStep";
-import { falSubscribe } from "./falSubscribe";
 
-const DETECTION_MODEL = "fal-ai/florence-2-large/object-detection";
+const PROMPT_PREFIX = `Look at the image at this URL and determine if it contains a human face or person portrait (headshot, selfie, press photo, etc).
 
-/** Labels that indicate a human face or person is present in the image. */
-const FACE_LABELS = ["person", "face", "human face", "man", "woman", "boy", "girl"];
+Respond with ONLY "true" or "false". Nothing else.
+
+Image URL: `;
 
 /**
- * Detects whether an image contains a human face using Florence-2 object detection.
+ * Detects whether an image contains a human face using a vision-capable text model.
  *
  * @param imageUrl - URL of the image to analyze
- * @returns true if at least one face/person is detected, false otherwise
+ * @returns true if the image contains a face/portrait, false otherwise
  */
 export async function detectFace(imageUrl: string): Promise<boolean> {
   try {
-    const result = await falSubscribe(DETECTION_MODEL, {
-      image_url: imageUrl,
+    const recoupApiKey = process.env.RECOUP_API_KEY;
+    if (!recoupApiKey) {
+      logStep("Face detection skipped — RECOUP_API_KEY not set", false);
+      return false;
+    }
+
+    const recoupApiUrl = process.env.RECOUP_API_URL ?? "https://recoup-api.vercel.app";
+    const response = await fetch(`${recoupApiUrl}/api/chat/generate`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": recoupApiKey,
+      },
+      body: JSON.stringify({
+        prompt: `${PROMPT_PREFIX}${imageUrl}`,
+        model: "google/gemini-2.5-flash",
+        excludeTools: ["create_task"],
+      }),
     });
 
-    const data = result.data as Record<string, unknown>;
-    const results = data.results as { labels?: string[] } | undefined;
-    const labels = results?.labels ?? [];
+    if (!response.ok) {
+      logStep("Face detection API returned error", false, { status: response.status });
+      return false;
+    }
 
-    const hasFace = labels.some((label) => {
-      const lower = label.toLowerCase();
-      return FACE_LABELS.some(
-        (faceLabel) => lower === faceLabel || lower.split(" ").includes(faceLabel),
-      );
-    });
-    logStep("Face detection result", false, { imageUrl: imageUrl.slice(0, 80), hasFace, labels });
+    const json = (await response.json()) as { text?: string };
+    const answer = (json.text ?? "").trim().toLowerCase();
+    const hasFace = answer === "true";
+
+    logStep("Face detection result", false, { imageUrl: imageUrl.slice(0, 80), hasFace, answer });
     return hasFace;
   } catch (err) {
     logStep("Face detection failed, assuming no face", false, {