stats-updater: Fix timeout, remove deprecated fields

stats-updater,api,web,whattheduck: Make notation nullable instead of setting -1

Author: bperel

apps/stats-updater/docker-compose-dev.yml

@@ -12,7 +12,7 @@ import {
   Prisma,
 } from "~prisma-schemas/schemas/coa";
 import { prismaClient as prismaCoa } from "~prisma-schemas/schemas/coa/client";
-import type { authorUser } from "~prisma-schemas/schemas/dm";
+import type { authorUser, issue as dmIssue } from "~prisma-schemas/schemas/dm";
 import { prismaClient as prismaDm } from "~prisma-schemas/schemas/dm/client";
 import type { authorStory } from "~prisma-schemas/schemas/dm_stats";
 import { prismaClient as prismaDmStats } from "~prisma-schemas/schemas/dm_stats/client";
@@ -109,6 +109,17 @@ const runQuery = async (sql: string) => {
 const runQueryFile = async (dbName: string, sqlFile: string) =>
   runQuery(`USE ${dbName};` + readFileSync(sqlFile).toString());
 
+// Direct connection for heavy SQL (avoids Prisma adapter issues). Use ANALYZE after bulk insert — OPTIMIZE rebuilds can run long and close the socket.
+const runQueryOnDirectConnection = async (sql: string, dbName?: string) => {
+  if (dbName) {
+    await connection.query(`USE ${dbName}`);
+  }
+  await connection.query(
+    "SET SESSION net_read_timeout = 28800, net_write_timeout = 28800",
+  );
+  await connection.query(sql);
+};
+
 connect().then(async () => {
   await runQuery(`DROP DATABASE IF EXISTS ${DATABASE_NAME_DM_STATS_NEW}`);
   await runQuery(`CREATE DATABASE ${DATABASE_NAME_DM_STATS_NEW}`);
@@ -132,21 +143,23 @@ connect().then(async () => {
   });
 
   await prismaDmStats.issueSimple.createMany({
-    data: await prismaDm.issue.findMany({
-      distinct: ["issuecode", "userId"],
-      select: {
-        issuecode: true,
-        userId: true,
-      },
-      where: {
-        issuecode: {
-          not: null,
+    data: (
+      await prismaDm.issue.findMany({
+        distinct: ["issuecode", "userId"],
+        select: {
+          issuecode: true,
+          userId: true,
         },
-        userId: {
-          in: userIdsWithPersoncodes,
+        where: {
+          issuecode: {
+            not: null,
+          },
+          userId: {
+            in: userIdsWithPersoncodes,
+          },
         },
-      },
-    }),
+      })
+    ).map((issue) => issue as dmIssue & { issuecode: string }),
   });
 
   console.log("Creating storyIssue entries");
@@ -208,27 +221,39 @@ connect().then(async () => {
   await prismaDmStats.$executeRaw`OPTIMIZE TABLE utilisateurs_histoires_manquantes`;
 
   console.log("Creating missingIssueForUser entries");
-  await prismaDmStats.$executeRaw`
-    insert into utilisateurs_publications_manquantes(ID_User, personcode, storycode, issuecode, Notation)
+  await runQueryOnDirectConnection(
+    `insert into utilisateurs_publications_manquantes(ID_User, personcode, storycode, issuecode, Notation)
     select distinct u_h_m.ID_User AS userId,
       u_h_m.personcode,
       u_h_m.storycode,
       h_p.issuecode,
       a_p.Notation AS notation
     from utilisateurs_histoires_manquantes u_h_m
       inner join histoires_publications h_p using (storycode)
-      inner join auteurs_pseudos a_p on a_p.ID_User = u_h_m.ID_User and u_h_m.personcode = a_p.NomAuteurAbrege`;
+      inner join auteurs_pseudos a_p on a_p.ID_User = u_h_m.ID_User and u_h_m.personcode = a_p.NomAuteurAbrege`,
+    DATABASE_NAME_DM_STATS_NEW,
+  );
 
-  await prismaDmStats.$executeRaw`OPTIMIZE TABLE utilisateurs_publications_manquantes`;
+  console.log("ANALYZING TABLE utilisateurs_publications_manquantes");
+  await runQueryOnDirectConnection(
+    "ANALYZE TABLE utilisateurs_publications_manquantes",
+    DATABASE_NAME_DM_STATS_NEW,
+  );
 
   console.log("Creating suggestedIssueForUser entries");
-  await prismaDmStats.$executeRaw`
-      insert into utilisateurs_publications_suggerees(ID_User, issuecode, oldestdate, Score)
+  await runQueryOnDirectConnection(
+    `insert into utilisateurs_publications_suggerees(ID_User, issuecode, oldestdate, Score)
       select ID_User AS userId, issuecode, '0000-00-00', sum(Notation) AS score
       from utilisateurs_publications_manquantes
-      group by ID_User, issuecode`;
+      group by ID_User, issuecode`,
+    DATABASE_NAME_DM_STATS_NEW,
+  );
 
-  await prismaDmStats.$executeRaw`OPTIMIZE TABLE utilisateurs_publications_suggerees`;
+  console.log("ANALYZING TABLE utilisateurs_publications_suggerees");
+  await runQueryOnDirectConnection(
+    "ANALYZE TABLE utilisateurs_publications_suggerees",
+    DATABASE_NAME_DM_STATS_NEW,
+  );
 
   await runQuery(`DROP DATABASE IF EXISTS ${DATABASE_NAME_DM_STATS}_old`);
   await runQuery(`CREATE DATABASE ${DATABASE_NAME_DM_STATS}_old`);

apps/stats-updater/index.ts

@@ -42,7 +42,7 @@
           </b-col>
           <b-col lg="2">
             <StarRating
-              v-model:rating="author.notation"
+              :rating="author.notation ?? 5"
               :readonly="false"
               :max-rating="10"
               @update:rating="updateRating(author)"

apps/web/src/components/AuthorList.vue

@@ -94,7 +94,7 @@ const countryNamesWithAllCountriesOption = $computed(
     ],
 );
 const watchedAuthorsWithNotation = $computed(() =>
-  ratings.value?.filter(({ notation }) => notation > 0),
+  ratings.value?.filter(({ notation }) => notation !== null),
 );
 
 watch(

apps/web/src/pages/expand/suggestions.vue

@@ -31,7 +31,7 @@
           </ion-col>
           <ion-col size="5">
             <StarRating
-              v-model:rating="author.notation"
+              :rating="author.notation ?? 5"
               :readonly="isOffline"
               :max-rating="10"
               @update:rating="updateRating(author)"

apps/whattheduck/src/views/Authors.vue

@@ -29,7 +29,7 @@ export default ({ _socket }: UserServices) => ({
 
   addWatchedAuthor: async (personcode: string) => {
     try {
-      await upsertAuthorUser(personcode, _socket.data.user.id);
+      await upsertAuthorUser(personcode, _socket.data.user.id, null);
     } catch (e) {
       console.log(e);
       return { error: "Error", errorDetails: (e as Error).message };
@@ -59,7 +59,7 @@ export default ({ _socket }: UserServices) => ({
 const upsertAuthorUser = async (
   personcode: string,
   userId: number,
-  notation?: number,
+  notation: number | null,
 ) => {
   const criteria = {
     personcode,
@@ -84,7 +84,7 @@ const upsertAuthorUser = async (
     await prismaDm.authorUser.create({
       data: {
         ...criteria,
-        notation: 5,
+        notation,
       },
     });
   }

packages/api/services/collection/watched-authors/index.ts

@@ -43,9 +43,10 @@ model authorUser {
   id         Int    @id @default(autoincrement()) @map("ID")
   personcode String @map("NomAuteurAbrege") @db.VarChar(79)
   userId     Int    @map("ID_user")
-  notation   Int    @default(-1) @map("Notation")
+  notation   Int?   @map("Notation") @db.UnsignedTinyInt
 
   @@unique([userId, personcode], map: "auteurs_pseudos_uindex")
+  @@index([userId], map: "auteurs_pseudos_ID_user_index")
   @@map("auteurs_pseudos")
 }
 

packages/prisma-schemas/schemas/dm/schema.prisma

@@ -11,32 +11,32 @@ CREATE TABLE `auteurs_histoires` (
 
 -- CreateTable
 CREATE TABLE `auteurs_pseudos` (
+    `ID` INTEGER NOT NULL AUTO_INCREMENT,
     `NomAuteurAbrege` VARCHAR(79) NOT NULL,
-    `ID_user` INTEGER NOT NULL,
-    `Notation` INTEGER NOT NULL DEFAULT -1,
+    `ID_User` INTEGER NOT NULL,
+    `Notation` TINYINT UNSIGNED NULL,
 
-    INDEX `auteurs_pseudos_ID_user_index`(`ID_user`),
-    UNIQUE INDEX `auteurs_pseudos_uindex`(`ID_user`, `NomAuteurAbrege`)
+    INDEX `auteurs_pseudos_ID_user_index`(`ID_User`),
+    UNIQUE INDEX `auteurs_pseudos_uindex`(`ID_User`, `NomAuteurAbrege`),
+    PRIMARY KEY (`ID`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `numeros_simple` (
     `ID` INTEGER NOT NULL AUTO_INCREMENT,
     `ID_Utilisateur` INTEGER NOT NULL,
-    `issuecode` VARCHAR(25) NULL DEFAULT (concat(convert(`Pays` using utf8mb3),'/',convert(`Magazine` using utf8mb3),' ',`Numero`)),
+    `issuecode` VARCHAR(25) NOT NULL,
 
-    INDEX `Utilisateur`(`ID_Utilisateur`),
-    INDEX `issuecode`(`issuecode`),
-    INDEX `issuecode_user`(`issuecode`, `ID_Utilisateur`),
+    INDEX `ID_Utilisateur`(`ID_Utilisateur`),
+    INDEX `user_issue`(`issuecode`),
+    UNIQUE INDEX `unique_index`(`ID_Utilisateur`, `issuecode`),
     PRIMARY KEY (`ID`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `histoires_publications` (
     `ID` INTEGER NOT NULL AUTO_INCREMENT,
     `storycode` VARCHAR(19) NOT NULL,
-    `publicationcode` VARCHAR(12) NULL,
-    `issuenumber` VARCHAR(12) NULL,
     `issuecode` VARCHAR(25) NOT NULL,
     `oldestdate` VARCHAR(10) NULL,
 
@@ -64,35 +64,27 @@ CREATE TABLE `utilisateurs_publications_manquantes` (
     `ID_User` INTEGER NOT NULL,
     `personcode` VARCHAR(79) NOT NULL,
     `storycode` VARCHAR(19) NOT NULL,
-    `publicationcode` VARCHAR(12) NULL,
-    `issuenumber` VARCHAR(12) NULL,
     `issuecode` VARCHAR(25) NOT NULL,
     `oldestdate` VARCHAR(10) NULL,
-    `Notation` TINYINT UNSIGNED NOT NULL,
+    `Notation` TINYINT UNSIGNED NULL,
 
-    INDEX `missing_user_issue`(`ID_User`, `publicationcode`, `issuenumber`),
-    INDEX `suggested`(`ID_User`, `publicationcode`, `issuenumber`, `oldestdate`),
+    INDEX `missing_user_issue`(`ID_User`, `issuecode`),
+    INDEX `suggested`(`ID_User`, `issuecode`, `oldestdate`),
     INDEX `user_stories`(`ID_User`, `personcode`, `storycode`),
-    INDEX `missing_user_issue_issuecode`(`ID_User`, `issuecode`),
-    INDEX `suggested_issuecode`(`ID_User`, `issuecode`, `oldestdate`),
-    UNIQUE INDEX `unique_index`(`ID_User`, `personcode`, `storycode`, `publicationcode`, `issuenumber`),
-    UNIQUE INDEX `unique_index_issuecode`(`ID_User`, `personcode`, `storycode`, `issuecode`),
+    UNIQUE INDEX `unique_index`(`ID_User`, `personcode`, `storycode`, `issuecode`),
     PRIMARY KEY (`ID`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `utilisateurs_publications_suggerees` (
     `ID` INTEGER NOT NULL AUTO_INCREMENT,
     `ID_User` INTEGER NOT NULL,
-    `publicationcode` VARCHAR(12) NULL,
-    `issuenumber` VARCHAR(12) NULL,
     `issuecode` VARCHAR(25) NOT NULL,
     `oldestdate` VARCHAR(10) NULL,
     `Score` INTEGER NOT NULL,
 
     INDEX `suggested_issue_user`(`ID_User`),
-    UNIQUE INDEX `suggested_issue_for_user`(`ID_User`, `publicationcode`, `issuenumber`),
-    UNIQUE INDEX `suggested_issue_for_user_issuecode`(`ID_User`, `issuecode`),
+    UNIQUE INDEX `suggested_issue_for_user`(`ID_User`, `issuecode`),
     PRIMARY KEY (`ID`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 

packages/prisma-schemas/schemas/dm_stats/migrations/0_init/migration.sql

@@ -18,22 +18,24 @@ model authorStory {
 }
 
 model authorUser {
+  id         Int    @id @default(autoincrement()) @map("ID")
   userId     Int    @map("ID_User")
   personcode String @map("NomAuteurAbrege") @db.VarChar(79)
-  notation   Int?   @map("Notation") @db.TinyInt
+  notation   Int?   @map("Notation") @db.UnsignedTinyInt
 
-  @@id([userId, personcode])
+  @@unique([userId, personcode], map: "auteurs_pseudos_uindex")
+  @@index([userId], map: "auteurs_pseudos_ID_user_index")
   @@map("auteurs_pseudos")
 }
 
 model issueSimple {
-  userId          Int    @map("ID_Utilisateur")
-  Publicationcode String @db.VarChar(12)
-  Numero          String @db.VarChar(12)
+  id        Int    @id @default(autoincrement()) @map("ID")
+  userId    Int    @map("ID_Utilisateur")
+  issuecode String @db.VarChar(25)
 
-  @@id([userId, Publicationcode, Numero])
+  @@unique([userId, issuecode], map: "unique_index")
   @@index([userId], map: "ID_Utilisateur")
-  @@index([Publicationcode, Numero], map: "user_issue")
+  @@index([issuecode], map: "user_issue")
   @@map("numeros_simple")
 }
 
@@ -61,17 +63,13 @@ model missingStoryForUser {
 }
 
 model missingIssueForUser {
-  ID              Int     @id @default(autoincrement())
-  userId          Int     @map("ID_User")
-  personcode      String  @db.VarChar(79)
-  storycode       String  @db.VarChar(19)
-  /// @deprecated
-  publicationcode String? @db.VarChar(12)
-  /// @deprecated
-  issuenumber     String? @db.VarChar(12)
-  issuecode       String  @db.VarChar(25)
-  oldestdate      String? @db.VarChar(10)
-  notation        Int     @map("Notation") @db.UnsignedTinyInt
+  id         Int     @id @default(autoincrement()) @map("ID")
+  userId     Int     @map("ID_User")
+  personcode String  @db.VarChar(79)
+  storycode  String  @db.VarChar(19)
+  issuecode  String  @db.VarChar(25)
+  oldestdate String? @db.VarChar(10)
+  notation   Int?    @map("Notation") @db.UnsignedTinyInt
 
   @@unique([userId, personcode, storycode, issuecode], map: "unique_index")
   @@index([userId, issuecode], map: "missing_user_issue")
@@ -81,15 +79,11 @@ model missingIssueForUser {
 }
 
 model suggestedIssueForUser {
-  ID              Int     @id @default(autoincrement())
-  userId          Int     @map("ID_User")
-  /// @deprecated
-  publicationcode String? @db.VarChar(12)
-  /// @deprecated
-  issuenumber     String? @db.VarChar(12)
-  issuecode       String  @db.VarChar(25)
-  oldestdate      String? @db.VarChar(10)
-  score           Int     @map("Score")
+  id         Int     @id @default(autoincrement()) @map("ID")
+  userId     Int     @map("ID_User")
+  issuecode  String  @db.VarChar(25)
+  oldestdate String? @db.VarChar(10)
+  score      Int     @map("Score")
 
   @@unique([userId, issuecode], map: "suggested_issue_for_user")
   @@index([userId], map: "suggested_issue_user")