feat: Smoother Data Loading with Web Workers & Better Validation

packages/phoenix-event-display/CHANGELOG.md

@@ -7,114 +7,59 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 ## 3.0.4 (2024-11-29)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 ## [3.0.3](https://github.com/HSF/phoenix/compare/v3.0.2...v3.0.3) (2024-11-22)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 ## [3.0.2](https://github.com/HSF/phoenix/compare/v3.0.1...v3.0.2) (2024-11-22)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 ## [3.0.1](https://github.com/HSF/phoenix/compare/v2.17.0...v3.0.1) (2024-11-22)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 # 3.0.0 (2024-11-21)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 # 2.17.0 (2024-11-21)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 # 2.16.0 (2024-03-24)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 ## 2.15.1 (2024-03-22)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 # [2.15.0](https://github.com/HSF/phoenix/compare/v2.14.1...v2.15.0) (2024-03-21)
 
-
 ### Reverts
 
-* Revert "chore(release): v2.15.0" ([8f018e7](https://github.com/HSF/phoenix/commit/8f018e7c11284d3dfd4290f84e87949b8ac2e9d7))
-
-
-
-
+- Revert "chore(release): v2.15.0" ([8f018e7](https://github.com/HSF/phoenix/commit/8f018e7c11284d3dfd4290f84e87949b8ac2e9d7))
 
 ## 2.14.1 (2023-05-14)
 
-
 ### Bug Fixes
 
-* configure git when trying to release ([fc4ae96](https://github.com/HSF/phoenix/commit/fc4ae96efdb5ce47db09c0e026fce4561a42ec24))
-
-
-
-
+- configure git when trying to release ([fc4ae96](https://github.com/HSF/phoenix/commit/fc4ae96efdb5ce47db09c0e026fce4561a42ec24))
 
 # [2.14.0](https://github.com/HSF/phoenix/compare/v2.13.0...v2.14.0) (2023-03-22)
 
 **Note:** Version bump only for package phoenix-event-display
 
-
-
-
-
 # [2.13.0](https://github.com/HSF/phoenix/compare/v2.12.0...v2.13.0) (2023-03-02)
 
-
 ### Reverts
 
-* Revert "Updated versions to try to reduce yarn warnings" ([477845f](https://github.com/HSF/phoenix/commit/477845f916b9da939a8fbe8c3eb7278684255813))
-
-
-
-
+- Revert "Updated versions to try to reduce yarn warnings" ([477845f](https://github.com/HSF/phoenix/commit/477845f916b9da939a8fbe8c3eb7278684255813))
 
 # [2.12.0](https://github.com/HSF/phoenix/compare/v2.11.4...v2.12.0) (2023-01-13)
 

packages/phoenix-event-display/configs/jest.conf.js

@@ -5,6 +5,7 @@ module.exports = {
   rootDir: '../',
   roots: ['<rootDir>/src/tests'],
   preset: 'ts-jest/presets/js-with-ts-legacy',
+  setupFilesAfterEnv: ['<rootDir>/configs/jest.setup.js'],
   moduleNameMapper: {
     '^(\\.\\.?\\/.+)\\.js$': '$1',
   },
@@ -15,6 +16,10 @@ module.exports = {
         tsconfig: {
           rootDir: null,
           allowJs: true,
+          module: 'esnext',
+        },
+        diagnostics: {
+          ignoreCodes: [1343],
         },
         astTransformers: {
           before: [

packages/phoenix-event-display/configs/jest.setup.js

@@ -0,0 +1,10 @@
+/* eslint-disable no-undef */
+/**
+ * Jest setup file to add polyfills for Node.js test environment.
+ * Required for libraries like jsroot that use TextEncoder/TextDecoder.
+ */
+
+const { TextEncoder, TextDecoder } = require('util');
+
+global.TextEncoder = TextEncoder;
+global.TextDecoder = TextDecoder;

packages/phoenix-event-display/package.json

@@ -46,7 +46,8 @@
     "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "stats-js": "^1.0.1",
-    "three": "~0.178.0"
+    "three": "~0.178.0",
+    "zod": "^4.3.5"
   },
   "devDependencies": {
     "@babel/helper-string-parser": "^7.27.1",

packages/phoenix-event-display/src/helpers/validation-schemas.ts

@@ -0,0 +1,50 @@
+import { z } from 'zod';
+
+/**
+ * Validates the structure of the 'Types' object in CMS event data.
+ * The keys are collection names, and the values are arrays of attribute definitions.
+ * Each attribute definition is an array where the first element is the attribute name.
+ */
+export const CMSTypesSchema = z.record(
+  z.string(),
+  z.array(z.tuple([z.string()]).rest(z.union([z.string(), z.number()]))),
+);
+
+/**
+ * Validates the structure of the 'Collections' object in CMS event data.
+ * The keys are collection names, and the values are arrays of objects (rows).
+ * Each row is an array of values corresponding to the attributes defined in 'Types'.
+ */
+export const CMSCollectionsSchema = z.record(
+  z.string(),
+  z.array(z.array(z.union([z.number(), z.string()]))),
+);
+
+/**
+ * Validates a single item in an Association.
+ * Typically a tuple of [unknown, index].
+ */
+export const CMSAssociationItemSchema = z.array(
+  z.tuple([z.number(), z.number()]),
+);
+
+/**
+ * Validates the structure of the 'Associations' object in CMS event data.
+ */
+export const CMSAssociationsSchema = z.record(
+  z.string(),
+  z.array(CMSAssociationItemSchema),
+);
+
+/**
+ * Main schema for a CMS Event.
+ */
+export const CMSEventSchema = z
+  .object({
+    Types: CMSTypesSchema,
+    Collections: CMSCollectionsSchema,
+    Associations: CMSAssociationsSchema.optional(),
+  })
+  .passthrough(); // Allow other properties like 'run number' etc if they exist at top level (though usually they are extracted)
+
+export type CMSEvent = z.infer<typeof CMSEventSchema>;

packages/phoenix-event-display/src/loaders/cms-loader.ts

@@ -50,44 +50,50 @@ export class CMSLoader extends PhoenixLoader {
     this.loadingManager.addLoadableItem('ig_archive');
     const igArchive = new JSZip();
     const eventsDataInIg: any[] = [];
+
     const readArchive = (res: File | ArrayBuffer) => {
-      igArchive.loadAsync(res).then(() => {
+      igArchive.loadAsync(res).then(async () => {
         let allFilesPath = Object.keys(igArchive.files);
         // If the event path or name is given then filter all data to get the required events
         if (eventPathName) {
           allFilesPath = allFilesPath.filter((filePath) =>
             filePath.includes(eventPathName),
           );
         }
-        let i = 1;
+
+        // We'll process files sequentially or in parallel?
+        // Parallel could spawn too many workers messages. But `async` loop is fine.
+        let processedCount = 0;
+        const totalFiles = allFilesPath.length;
+
+        if (totalFiles === 0) {
+          this.loadingManager.itemLoaded('ig_archive');
+          return;
+        }
+
         for (const filePathInIg of allFilesPath) {
           // If the files are in the "Events" folder then process them.
           if (filePathInIg.toLowerCase().startsWith('events')) {
-            igArchive
-              ?.file(filePathInIg)!
-              .async('string')
-              .then((singleEvent: string) => {
-                // The data has some inconsistencies which need to be removed to properly parse JSON
-                singleEvent = singleEvent
-                  .replace(/'/g, '"')
-                  .replace(/\(/g, '[')
-                  .replace(/\)/g, ']')
-                  .replace(/nan/g, '0');
-                const eventJSON = JSON.parse(singleEvent);
-                eventJSON.eventPath = filePathInIg;
-                eventsDataInIg.push(eventJSON);
-                if (i === allFilesPath.length) {
-                  onFileRead(eventsDataInIg);
-                  this.loadingManager.itemLoaded('ig_archive');
-                }
-                i++;
-              });
-          } else {
-            if (i === allFilesPath.length) {
-              onFileRead(eventsDataInIg);
-              this.loadingManager.itemLoaded('ig_archive');
+            try {
+              const singleEvent = await igArchive
+                .file(filePathInIg)!
+                .async('string');
+              // Use Web Worker to parse
+              const eventJSON = await this.parseWithWorker(
+                singleEvent,
+                filePathInIg,
+              );
+              eventJSON.eventPath = filePathInIg;
+              eventsDataInIg.push(eventJSON);
+            } catch (error) {
+              console.error(`Error parsing event ${filePathInIg}:`, error);
             }
-            i++;
+          }
+
+          processedCount++;
+          if (processedCount === totalFiles) {
+            onFileRead(eventsDataInIg);
+            this.loadingManager.itemLoaded('ig_archive');
           }
         }
       });
@@ -104,6 +110,36 @@ export class CMSLoader extends PhoenixLoader {
     }
   }
 
+  /**
+   * Parse event data. Currently uses main-thread parsing.
+   * @param data Raw string data to parse
+   * @param id Identifier for the event (unused, kept for signature compatibility)
+   */
+  private parseWithWorker(data: string, id: string): Promise<any> {
+    return this.parseOnMainThread(data);
+  }
+
+  /**
+   * Fallback parser for environments where Web Worker is not available.
+   * Parses the event data on the main thread.
+   */
+  private parseOnMainThread(data: string): Promise<any> {
+    return new Promise((resolve, reject) => {
+      try {
+        // Clean the data (same logic as worker)
+        const cleanedData = data
+          .replace(/'/g, '"')
+          .replace(/\(/g, '[')
+          .replace(/\)/g, ']')
+          .replace(/nan/gi, '0');
+        const eventJSON = JSON.parse(cleanedData);
+        resolve(eventJSON);
+      } catch (error) {
+        reject(error instanceof Error ? error.message : String(error));
+      }
+    });
+  }
+
   /**
    * Load event data from an ".ig" archive.
    * @param filePath Path to the ".ig" archive file.
@@ -118,7 +154,11 @@ export class CMSLoader extends PhoenixLoader {
     this.readIgArchive(
       filePath,
       (allEvents: any[]) => {
-        onEventRead(allEvents[0]);
+        if (allEvents.length > 0) {
+          onEventRead(allEvents[0]);
+        } else {
+          console.warn('No events found in archive');
+        }
       },
       eventPathName,
     );