Auto purging old files if the cache limit had reached

package.json

@@ -1,7 +1,7 @@
 {
   "name": "cod-dicomweb-server",
   "title": "COD Dicomweb server",
-  "version": "1.3.11",
+  "version": "1.3.12",
   "private": false,
   "description": "A wadors server proxy that get data from a Cloud Optimized Dicom format.",
   "main": "dist/umd/main.js",

src/classes/CodDicomWebServer.ts

@@ -21,31 +21,31 @@ import { download, getDirectoryHandle } from '../fileAccessSystemUtils';
 class CodDicomWebServer {
   private filePromises: Record<string, Promise<void>> = {};
   private options: CodDicomWebServerOptions = {
-    maxWorkerFetchSize: Infinity,
+    maxCacheSize: 4 * 1024 * 1024 * 1024, // 4GB
     domain: constants.url.DOMAIN,
     enableLocalCache: false
   };
   private fileManager;
   private metadataManager;
   private seriesUidFileUrls: Record<string, Set<{ type: Enums.URLType; url: string }>> = {};
 
-  constructor(args: { maxWorkerFetchSize?: number; domain?: string; disableWorker?: boolean; enableLocalCache?: boolean } = {}) {
-    const { maxWorkerFetchSize, domain, disableWorker, enableLocalCache } = args;
+  constructor(args: { maxCacheSize?: number; domain?: string; disableWorker?: boolean; enableLocalCache?: boolean } = {}) {
+    const { maxCacheSize, domain, disableWorker, enableLocalCache } = args;
 
-    this.options.maxWorkerFetchSize = maxWorkerFetchSize || this.options.maxWorkerFetchSize;
+    this.options.maxCacheSize = maxCacheSize || this.options.maxCacheSize;
     this.options.domain = domain || this.options.domain;
     this.options.enableLocalCache = !!enableLocalCache;
     const fileStreamingScriptName = constants.dataRetrieval.FILE_STREAMING_WORKER_NAME;
     const filePartialScriptName = constants.dataRetrieval.FILE_PARTIAL_WORKER_NAME;
-    this.fileManager = new FileManager({ fileStreamingScriptName });
+    this.fileManager = new FileManager();
     this.metadataManager = new MetadataManager();
 
     if (disableWorker) {
       const dataRetrievalManager = getDataRetrievalManager();
       dataRetrievalManager.setDataRetrieverMode(Enums.DataRetrieveMode.REQUEST);
     }
 
-    register({ fileStreamingScriptName, filePartialScriptName }, this.options.maxWorkerFetchSize);
+    register({ fileStreamingScriptName, filePartialScriptName });
   }
 
   public setOptions = (newOptions: Partial<CodDicomWebServerOptions>): void => {
@@ -213,17 +213,12 @@ class CodDicomWebServer {
     }
 
     const directoryHandle = this.options.enableLocalCache && (await getDirectoryHandle());
-    const { maxWorkerFetchSize } = this.getOptions();
     const dataRetrievalManager = getDataRetrievalManager();
-    const { FILE_STREAMING_WORKER_NAME, FILE_PARTIAL_WORKER_NAME, THRESHOLD } = constants.dataRetrieval;
+    const { FILE_STREAMING_WORKER_NAME, FILE_PARTIAL_WORKER_NAME } = constants.dataRetrieval;
     let tarPromise: Promise<void>;
 
     if (!this.filePromises[fileUrl]) {
       tarPromise = new Promise<void>((resolveFile, rejectFile) => {
-        if (this.fileManager.getTotalSize() + THRESHOLD > maxWorkerFetchSize) {
-          throw new CustomError(`CodDicomWebServer.ts: Maximum size(${maxWorkerFetchSize}) for fetching files reached`);
-        }
-
         const FetchTypeEnum = constants.Enums.FetchType;
 
         if (fetchType === FetchTypeEnum.API_OPTIMIZED) {
@@ -315,14 +310,21 @@ class CodDicomWebServer {
           throw evt.error;
         }
 
-        const { url, position, chunk, isAppending } = evt.data;
+        const { url, position, chunk, totalLength, isAppending } = evt.data;
 
         if (isAppending) {
           if (chunk) {
             this.fileManager.append(url, chunk, position);
           } else {
             this.fileManager.setPosition(url, position);
           }
+        } else {
+          // The full empty file including with first chunk have been stored to fileManager
+          // by the worker listener in the file promise.
+          // So, we check whether the cache exceeded the limit here.
+          if (this.fileManager.getTotalSize() > this.options.maxCacheSize) {
+            this.fileManager.decacheNecessaryBytes(url, totalLength);
+          }
         }
 
         if (!requestResolved && url === fileUrl && offsets && position > offsets.endByte) {

src/classes/customClasses.ts

@@ -17,5 +17,6 @@ export class CustomMessageEvent extends MessageEvent<{
   chunk?: Uint8Array;
   isAppending?: boolean;
   fileArraybuffer?: Uint8Array;
+  totalLength: number;
   offsets?: { startByte: number; endByte: number };
 }> {}

src/dataRetrieval/register.ts

@@ -3,13 +3,7 @@ import { getDataRetrievalManager } from './dataRetrievalManager';
 import filePartial from './scripts/filePartial';
 import fileStreaming from './scripts/fileStreaming';
 
-export function register(
-  workerNames: {
-    fileStreamingScriptName: string;
-    filePartialScriptName: string;
-  },
-  maxFetchSize: number
-): void {
+export function register(workerNames: { fileStreamingScriptName: string; filePartialScriptName: string }): void {
   const { fileStreamingScriptName, filePartialScriptName } = workerNames;
   const dataRetrievalManager = getDataRetrievalManager();
 
@@ -33,6 +27,4 @@ export function register(
 
     dataRetrievalManager.register(filePartialScriptName, partialWorkerFn);
   }
-
-  dataRetrievalManager.executeTask(fileStreamingScriptName, 'setMaxFetchSize', maxFetchSize);
 }

src/dataRetrieval/scripts/fileStreaming.ts

@@ -2,21 +2,6 @@ import { CustomError } from '../../classes/customClasses';
 import { createStreamingFileName, readFile, writeFile } from '../../fileAccessSystemUtils';
 
 const fileStreaming = {
-  maxFetchSize: 4 * 1024 * 1024 * 1024, // 4GB
-  fetchedSize: 0,
-
-  setMaxFetchSize(size: number): void {
-    if (size > 0) {
-      this.maxFetchSize = size;
-    }
-  },
-
-  decreaseFetchedSize(size: number): void {
-    if (size > 0 && size <= this.fetchedSize) {
-      this.fetchedSize -= size;
-    }
-  },
-
   async stream(
     args: {
       url: string;
@@ -30,6 +15,7 @@ const fileStreaming = {
       isAppending?: boolean;
       fileArraybuffer?: Uint8Array;
       chunk?: Uint8Array;
+      totalLength: number;
     }) => void
   ): Promise<Uint8Array | void> {
     const { url, headers, useSharedArrayBuffer, directoryHandle } = args;
@@ -42,7 +28,8 @@ const fileStreaming = {
       if (directoryHandle) {
         const file = (await readFile(directoryHandle, fileName, { isJson: false })) as ArrayBuffer;
         if (file) {
-          callBack({ url, position: file.byteLength, fileArraybuffer: new Uint8Array(file) });
+          const totalLength = file.byteLength;
+          callBack({ url, position: totalLength, fileArraybuffer: new Uint8Array(file), totalLength });
           return;
         }
       }
@@ -74,21 +61,14 @@ const fileStreaming = {
       if (!completed) {
         let position = firstChunk.value.length;
 
-        if (this.fetchedSize + position > this.maxFetchSize) {
-          controller.abort();
-          throw new CustomError(`Maximum size(${this.maxFetchSize}) for fetching files reached`);
-        }
-
-        this.fetchedSize += position;
-
         if (useSharedArrayBuffer) {
           sharedArraybuffer = new SharedArrayBuffer(totalLength);
           fileArraybuffer = new Uint8Array(sharedArraybuffer);
         } else {
           fileArraybuffer = new Uint8Array(totalLength);
         }
         fileArraybuffer.set(firstChunk.value);
-        callBack({ url, position, fileArraybuffer });
+        callBack({ url, position, fileArraybuffer, totalLength });
 
         while (!completed) {
           result = await reader.read();
@@ -100,22 +80,15 @@ const fileStreaming = {
 
           const chunk = result.value;
 
-          if (this.fetchedSize + chunk.length > this.maxFetchSize) {
-            sharedArraybuffer = null;
-            fileArraybuffer = null;
-            controller.abort();
-            throw new CustomError(`Maximum size(${this.maxFetchSize}) for fetching files reached`);
-          }
-
-          this.fetchedSize += chunk.length;
           fileArraybuffer.set(chunk, position);
           position += chunk.length;
 
           callBack({
             isAppending: true,
             url,
             position: position,
-            chunk: !useSharedArrayBuffer ? chunk : undefined
+            chunk: !useSharedArrayBuffer ? chunk : undefined,
+            totalLength
           });
         }
 

src/fileManager.ts

@@ -1,16 +1,10 @@
-import type { FileManagerOptions } from './types';
-import { getDataRetrievalManager } from './dataRetrieval/dataRetrievalManager';
+import type { FileManagerFile } from './types';
 
 class FileManager {
-  private files: Record<string, { data: Uint8Array; position: number }> = {};
-  private fileStreamingScriptName: string;
+  private files: Record<string, FileManagerFile> = {};
 
-  constructor({ fileStreamingScriptName }: FileManagerOptions) {
-    this.fileStreamingScriptName = fileStreamingScriptName;
-  }
-
-  set(url: string, file: { data: Uint8Array; position: number }): void {
-    this.files[url] = file;
+  set(url: string, file: Omit<FileManagerFile, 'lastModified'>): void {
+    this.files[url] = { ...file, lastModified: Date.now() };
   }
 
   get(url: string, offsets?: { startByte: number; endByte: number }): Uint8Array | null {
@@ -24,6 +18,7 @@ class FileManager {
   setPosition(url: string, position: number): void {
     if (this.files[url]) {
       this.files[url].position = position;
+      this.files[url].lastModified = Date.now();
     }
   }
 
@@ -39,29 +34,46 @@ class FileManager {
   }
 
   getTotalSize(): number {
-    return Object.entries(this.files).reduce((total, [url, { position }]) => {
-      return url.includes('?bytes=') ? total : total + position;
+    return Object.values(this.files).reduce((total, { data }) => {
+      return total + data.byteLength;
     }, 0);
   }
 
   remove(url: string): void {
-    const removedSize = this.getPosition(url);
-    delete this.files[url];
-
-    if (url.includes('?bytes=')) {
-      return;
+    try {
+      delete this.files[url];
+      console.log(`Removed ${url} from CodDicomwebServer cache`);
+    } catch (error) {
+      console.warn(`Error removing ${url} from CodDicomwebServer cache:`, error);
     }
-
-    const retrievalManager = getDataRetrievalManager();
-    retrievalManager.executeTask(this.fileStreamingScriptName, 'decreaseFetchedSize', removedSize);
   }
 
   purge(): void {
+    const fileURLs = Object.keys(this.files);
+    const totalSize = this.getTotalSize();
+    fileURLs.forEach((url) => this.remove(url));
+
+    console.log(`Purged ${totalSize - this.getTotalSize()} bytes from CodDicomwebServer cache`);
+  }
+
+  decacheNecessaryBytes(url: string, bytesNeeded: number): number {
     const totalSize = this.getTotalSize();
-    this.files = {};
+    const filesToDelete: string[] = [];
+    let collectiveSize = 0;
+
+    Object.entries(this.files)
+      .sort(([, a], [, b]) => a.lastModified - b.lastModified)
+      .forEach(([key, file]) => {
+        if (collectiveSize < bytesNeeded && key !== url) {
+          filesToDelete.push(key);
+          collectiveSize += file.data.byteLength;
+        }
+      });
+
+    filesToDelete.forEach((key) => this.remove(key));
 
-    const retrievalManager = getDataRetrievalManager();
-    retrievalManager.executeTask(this.fileStreamingScriptName, 'decreaseFetchedSize', totalSize);
+    console.log(`Decached ${totalSize - this.getTotalSize()} bytes`);
+    return collectiveSize;
   }
 }
 

src/tests/classes/CodDicomWebServer.test.ts

@@ -7,7 +7,6 @@ describe('CodDicomWebServer', () => {
   const getDataRetrievalManagerMock = jest.spyOn(require('../../dataRetrieval/dataRetrievalManager'), 'getDataRetrievalManager');
   const fileManagerMock = jest.spyOn(require('../../fileManager'), 'default');
   const metadataManagerMock = jest.spyOn(require('../../metadataManager'), 'default');
-  const getDirectoryHandleMock = jest.spyOn(require('../../fileAccessSystemUtils'), 'getDirectoryHandle').mockReturnThis();
 
   const workerAddEventListener = jest.fn();
   const fileManagerSet = jest.fn();
@@ -68,7 +67,7 @@ describe('CodDicomWebServer', () => {
     });
 
     it('should create a new instance with custom options', () => {
-      const options = { maxWorkerFetchSize: 10000, domain: 'example.com' };
+      const options = { maxCacheSize: 10000, domain: 'example.com' };
       const serverWithCustomOptions = new CodDicomWebServer(options);
       expect(serverWithCustomOptions).toBeInstanceOf(CodDicomWebServer);
     });
@@ -77,23 +76,23 @@ describe('CodDicomWebServer', () => {
   describe('getOptions', () => {
     it('should return the default options if not set', () => {
       const options = server.getOptions();
-      expect(options).toEqual({ maxWorkerFetchSize: Infinity, domain: url.DOMAIN, enableLocalCache: false });
+      expect(options).toEqual({ maxCacheSize: Infinity, domain: url.DOMAIN, enableLocalCache: false });
     });
   });
 
   describe('setOptions', () => {
     it('should set new options', () => {
-      const newOptions = { maxWorkerFetchSize: 2000 };
+      const newOptions = { maxCacheSize: 2000 };
       server.setOptions(newOptions);
       expect(server.getOptions()).toEqual({ domain: url.DOMAIN, enableLocalCache: false, ...newOptions });
     });
 
     it('should not set new options if the value is undefined', () => {
-      const newOptions = { maxWorkerFetchSize: 2000, domain: undefined };
+      const newOptions = { maxCacheSize: 2000, domain: undefined };
       server.setOptions(newOptions);
       expect(server.getOptions()).toEqual({
         domain: url.DOMAIN,
-        maxWorkerFetchSize: newOptions.maxWorkerFetchSize,
+        maxCacheSize: newOptions.maxCacheSize,
         enableLocalCache: false
       });
     });
@@ -202,21 +201,19 @@ describe('CodDicomWebServer', () => {
       );
     });
 
-    it('should throw an error if the maxFetchSize has been exceeded', async () => {
+    it('should handle even if there is a cache limit', async () => {
       const fileUrl = 'fileUrl';
       const headers = { 'Content-Type': 'application/octet-stream' };
       const options = {
         offsets: { startByte: 20, endByte: 100 },
         useSharedArrayBuffer: true,
         fetchType: Enums.FetchType.BYTES_OPTIMIZED
       };
-      server.setOptions({ maxWorkerFetchSize: 20 });
+      server.setOptions({ maxCacheSize: 20 });
       fileManagerGetTotalSize.mockReturnValueOnce(23);
 
       // @ts-ignore
-      await expect(server.fetchFile(fileUrl, headers, options)).rejects.toThrow(
-        'CodDicomWebServer.ts: Maximum size(20) for fetching files reached'
-      );
+      await expect(server.fetchFile(fileUrl, headers, options)).resolves;
     });
 
     it('should return the file if cached in the fileManager', async () => {
@@ -242,7 +239,7 @@ describe('CodDicomWebServer', () => {
     it('should delete a series instance', () => {
       const seriesInstanceUID = '1.2.3.4';
 
-      server.addFileUrl(seriesInstanceUID, 'fileUrl');
+      server.addFileUrl(seriesInstanceUID, Enums.URLType.FILE, 'fileUrl');
       server.delete(seriesInstanceUID);
 
       expect(fileManagerRemove).toHaveBeenCalledTimes(1);
@@ -258,10 +255,10 @@ describe('CodDicomWebServer', () => {
 
   describe('deleteAll', () => {
     it('should delete all series instances', () => {
-      server.addFileUrl('1.2.3.4', 'fileUrl1');
-      server.addFileUrl('1.2.4', 'fileUrl2');
-      server.addFileUrl('1.2.4', 'fileUrl3');
-      server.addFileUrl('1.2.3.4', 'fileUrl3');
+      server.addFileUrl('1.2.3.4', Enums.URLType.FILE, 'fileUrl1');
+      server.addFileUrl('1.2.4', Enums.URLType.FILE, 'fileUrl2');
+      server.addFileUrl('1.2.4', Enums.URLType.FILE, 'fileUrl3');
+      server.addFileUrl('1.2.3.4', Enums.URLType.FILE, 'fileUrl3');
       server.deleteAll();
 
       expect(fileManagerRemove).toHaveBeenCalledTimes(4);