api: Improve experimental image search

Author: bperel

.dockerignore

@@ -8,4 +8,4 @@
 **/node_modules
 **/Pods
 docker-compose*
-!**/napi-v3/**
+!**/napi-v6/**

.gitattributes

@@ -0,0 +1 @@
+packages/api/services/story-search/model/comic_embedding_model.onnx filter=lfs diff=lfs merge=lfs -text

.github/workflows/deploy.yml

@@ -23,7 +23,26 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          lfs: false # Don't fetch LFS during checkout
           fetch-depth: 0
+        id: checkout
+
+      - name: Fetch required LFS objects
+        run: |
+          echo "Fetching required LFS objects..."
+          git lfs install
+
+          # Try to fetch the LFS objects
+          echo "Attempting to fetch LFS objects..."
+          git lfs pull origin ${{ github.sha }} --include="packages/api/services/story-search/model/*.onnx" || {
+            echo "LFS pull failed, trying alternative approach..."
+            # If LFS pull fails, try to get the file from git directly
+            git checkout ${{ github.sha }} -- packages/api/services/story-search/model/comic_embedding_model.onnx || true
+          }
+
+          # Verify the model file is available and has content
+          ls -lh packages/api/services/story-search/model/comic_embedding_model.onnx || echo "File still missing"
+          file packages/api/services/story-search/model/comic_embedding_model.onnx
 
       - name: Log in to the Container registry
         uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1

.vscode/launch.json

@@ -114,6 +114,15 @@
             "stopOnEntry": false,
             "watchMode": true
         },
+        {
+            "type": "bun",
+            "request": "launch",
+            "name": "Debug import-entryurl-vectors",
+            "program": "scripts/import-entryurl-vectors.ts",
+            "cwd": "${workspaceFolder}/packages/api",
+            "stopOnEntry": false,
+            "watchMode": true
+        },
     ],
     "compounds": [
         {

apps/web/src/pages/image-search-test.vue

@@ -36,7 +36,30 @@ meta:
       small
       caption-top
       responsive
-    />
+    >
+      <template #cell(results)="row">
+        <template v-if="row.item.results">
+          <div v-if="'error' in row.item.results">
+            {{ row.item.results.error }}
+          </div>
+          <div v-else>
+            <b-table
+              class="results"
+              :items="row.item.results"
+              :fields="['score', 'fullUrl']"
+              small
+              responsive
+            >
+              <template #cell(fullUrl)="result">
+                <img
+                  :src="CLOUDINARY_URL + result.item.fullUrl"
+                  class="img-fluid"
+                />
+              </template>
+            </b-table></div
+        ></template>
+      </template>
+    </b-table>
   </div>
 </template>
 
@@ -54,25 +77,37 @@ type Example = {
   isCover: boolean;
 };
 
+type Results =
+  | {
+      error: string;
+    }
+  | {
+      score: number;
+      fullUrl: string;
+    }[];
+
+const CLOUDINARY_URL =
+  "https://res.cloudinary.com/dl7hskxab/image/upload/inducks-covers/";
+
 const examples = [
   {
-    url: "https://res.cloudinary.com/dl7hskxab/image/upload/inducks-covers/webusers/webusers/2018/07/fr_aljm_011a_001.jpg",
+    url: `${CLOUDINARY_URL}webusers/webusers/2018/07/fr_aljm_011a_001.jpg`,
     isCover: true,
   },
   {
-    url: "https://res.cloudinary.com/dl7hskxab/image/upload/inducks-covers/webusers/webusers/2014/02/fr_tp_0024a_001.jpg",
+    url: `${CLOUDINARY_URL}webusers/webusers/2014/02/fr_tp_0024a_001.jpg`,
     isCover: true,
   },
   {
-    url: "https://res.cloudinary.com/dl7hskxab/image/upload/inducks-covers/webusers/webusers/2011/01/fr_tp_0013a_001.jpg",
+    url: `${CLOUDINARY_URL}webusers/webusers/2011/01/fr_tp_0013a_001.jpg`,
     isCover: true,
   },
   {
-    url: "https://res.cloudinary.com/dl7hskxab/image/upload/inducks-covers/webusers/webusers/2024/05/eg_mg_0149p053_001.jpg",
+    url: `${CLOUDINARY_URL}webusers/webusers/2024/05/eg_mg_0149p053_001.jpg`,
     isCover: false,
   },
   {
-    url: "https://res.cloudinary.com/dl7hskxab/image/upload/inducks-covers/webusers/webusers/2021/06/yu_mzc_983c_001.jpg",
+    url: `${CLOUDINARY_URL}webusers/webusers/2021/06/yu_mzc_983c_001.jpg`,
     isCover: false,
   },
 ] as const;
@@ -82,9 +117,9 @@ const models = ref<
     modelData: "covers" | "story first pages";
     indexSize?: number | string;
     getIndexSize: () => Promise<number | string>;
-    run: (base64: string) => Promise<string>;
+    run: (base64: string) => Promise<Results>;
     time?: string;
-    results?: string;
+    results?: Results;
   }[]
 >([
   {
@@ -101,17 +136,12 @@ const models = ref<
         const searchResults = await coverIdEvents.searchFromCover(base64);
 
         return "error" in searchResults
-          ? searchResults.errorDetails || "Error"
-          : JSON.stringify(
-              searchResults.covers.map(({ issuecode, score }) => ({
-                issuecode,
-                score,
-              })),
-            );
+          ? { error: searchResults.errorDetails || "Error" }
+          : searchResults.covers;
       } catch (error) {
         return typeof error === "object" && "errorDetails" in error!
-          ? (error.errorDetails as string) || "Error"
-          : "Error";
+          ? { error: (error.errorDetails as string) || "Error" }
+          : { error: "Error" };
       }
     },
   },
@@ -125,14 +155,8 @@ const models = ref<
         true,
       );
       if ("error" in searchResults) {
-        return searchResults.error!;
-      } else
-        return JSON.stringify(
-          searchResults.results.map(({ entrycode, score }) => ({
-            entrycode,
-            score,
-          })),
-        );
+        return { error: searchResults.error! };
+      } else return searchResults.results;
     },
   },
   {
@@ -145,14 +169,8 @@ const models = ref<
         false,
       );
       if ("error" in searchResults) {
-        return searchResults.error!;
-      } else
-        return JSON.stringify(
-          searchResults.results.map(({ entrycode, score }) => ({
-            entrycode,
-            score,
-          })),
-        );
+        return { error: searchResults.error! };
+      } else return searchResults.results;
     },
   },
 ]);
@@ -214,6 +232,12 @@ onMounted(async () => {
 </script>
 
 <style scoped lang="scss">
+:deep(.results) {
+  td {
+    width: 50%;
+  }
+}
+
 img {
   cursor: pointer;
   height: 100px;

apps/whattheduck/src/views/IssueList.vue

@@ -220,7 +220,7 @@ const getSortedItemsWithCovers = async () => {
     key,
     item: {
       ...item,
-      cover: coverUrls[item.issuecode]!.fullUrl,
+      cover: coverUrls[item.issuecode]?.fullUrl || null,
     },
   }));
 };

packages/api/Dockerfile

@@ -9,7 +9,9 @@ RUN apt-get update && apt-get install -y openssl \
 
 RUN npm install -g pm2
 
-COPY ./node_modules/.pnpm/onnxruntime-node@*/node_modules/onnxruntime-node/bin/napi-v3/linux/x64 /bin/napi-v3/linux/x64
+COPY ./node_modules/.pnpm/onnxruntime-node@*/node_modules/onnxruntime-node/bin/napi-v6/linux/x64 /bin/napi-v6/linux/x64
+
+COPY ./packages/api/services/story-search/model/comic_embedding_model.onnx /app/services/story-search/model/comic_embedding_model.onnx
 
 RUN npm install --os=linux --cpu=x64 sharp
 

packages/api/index-story-search.ts

@@ -2,7 +2,7 @@ import { SocketIoInstrumentation } from "@opentelemetry/instrumentation-socket.i
 import * as Sentry from "@sentry/node";
 import dotenv from "dotenv";
 
-import { loadModel, server as storySearch } from "./services/story-search";
+import { getSession, server as storySearch } from "./services/story-search";
 import createSocketServer from "./socket";
 
 dotenv.config({
@@ -20,4 +20,4 @@ if (process.env.SENTRY_DSN) {
 const storySearchIo = createSocketServer(3011);
 storySearch(storySearchIo);
 
-loadModel();
+getSession();

packages/api/package.json

@@ -19,7 +19,6 @@
     "sentry:sourcemaps": "sentry-cli sourcemaps inject --org bruno-perel --project dm-api ./dist && sentry-cli sourcemaps upload --org bruno-perel --project dm-api ./dist"
   },
   "dependencies": {
-    "@huggingface/transformers": "~3.5.2",
     "@opentelemetry/instrumentation-socket.io": "^0.46.1",
     "@pusher/push-notifications-server": "^1.2.7",
     "@sentry/node": "^9.46.0",
@@ -34,6 +33,7 @@
     "ejs": "^3.1.10",
     "i18n": "^0.15.1",
     "jsonwebtoken": "^9.0.2",
+    "onnxruntime-node": "^1.22.0-rev",
     "nodemailer": "^7.0.5",
     "sharp": "0.33.5",
     "sharp-0-34": "npm:sharp@^0.34.3",