feat: deduplication on get many (#366)

* Adding unique utility

* Get many keys deduplication

* Adding some tests

* Fixing minor lint issue

* Lint fixes

* Release prepare

* Doc update

Author: CarlosGamero

README.md

@@ -40,7 +40,7 @@ Since there are a few cache solutions, here is a table comparing them:
 | Single Entity Fetch                              |                               ✓                               |                                  ✓                                   |                          ✓                          |                                     ✓                                     |
 | Bulk Entity Fetch                                |                               ✓                               |                                                                      |                          ✓                          |                                     ✓                                     |
 | Single Entity Fetch Deduplication (Read-Through) |                               ✓                               |                                  ✓                                   |                          ✓                          |                                                                           |
-| Bulk Entity Fetch Deduplication                  |                                                               |                                                                      |                          ✓                          |                                                                           |
+| Bulk Entity Fetch Deduplication                  |                               ✓                               |                                                                      |                          ✓                          |                                                                           |
 | Preemptive Cache Refresh (Refresh-Ahead)         |                               ✓                               |                                                                      |                                                     |                                                                           |
 | Tiered Caches                                    |                               ✓                               |                                                                      |                                                     |                                     ✓                                     |
 | Group Support                                    |                               ✓                               |                partially, references for invalidation                |                                                     |                                                                           |
@@ -216,7 +216,7 @@ Loader provides following methods:
 - `invalidateCacheForMany(keys: string[]): Promise<void>` - expunge all entries for given keys from all caches of this Loader;
 - `invalidateCache(): Promise<void>` - expunge all entries from all caches of this Loader;
 - `get(loadParams: LoadParams = string): Promise<T>` - sequentially attempt to retrieve data for specified key from all caches and loaders, in an order in which those data sources passed to the Loader constructor.
-- `getMany(keys: string[], loadParams?: P): Promise<T>` - sequentially attempt to retrieve data for specified keys from all caches and data sources, in an order in which those data sources were passed to the Loader constructor. Note that this retrieval mode doesn't support neither fetch deduplication nor the preemptive background refresh. Note that you need to manually resolve all keys upfront for this retrieval method (e. g. by using cacheKeyFromLoadParamsResolver from the Loader).
+- `getMany(keys: string[], loadParams?: P): Promise<T>` - sequentially attempt to retrieve data for specified keys from all caches and data sources, in an order in which those data sources were passed to the Loader constructor. Duplicate keys in the input array are automatically deduplicated to optimize performance and prevent redundant data source calls. Note that this retrieval mode doesn't support preemptive background refresh. Note that you need to manually resolve all keys upfront for this retrieval method (e. g. by using cacheKeyFromLoadParamsResolver from the Loader).
 
 ## Parametrized loading
 

lib/AbstractFlatCache.ts

@@ -1,8 +1,9 @@
-import {AbstractCache, CommonCacheConfig} from './AbstractCache'
+import {AbstractCache} from './AbstractCache'
 import type { Cache } from './types/DataSources'
 import type { GetManyResult, SynchronousCache } from './types/SyncDataSources'
 import {InMemoryCacheConfiguration} from "./memory/InMemoryCache";
 import {NotificationPublisher} from "./notifications/NotificationPublisher";
+import {unique} from "./util/unique";
 
 export abstract class AbstractFlatCache<LoadedValue, LoadParams = string> extends AbstractCache<
   LoadedValue,
@@ -86,7 +87,8 @@ export abstract class AbstractFlatCache<LoadedValue, LoadParams = string> extend
   }
 
   public getMany(keys: string[], loadParams?: LoadParams): Promise<LoadedValue[]> {
-    const inMemoryValues = this.getManyInMemoryOnly(keys)
+    const uniqueKeys = unique(keys)
+    const inMemoryValues = this.getManyInMemoryOnly(uniqueKeys)
     // everything is in memory, hurray
     if (inMemoryValues.unresolvedKeys.length === 0) {
       return Promise.resolve(inMemoryValues.resolvedValues)

lib/AbstractGroupCache.ts

@@ -3,6 +3,7 @@ import type { InMemoryGroupCacheConfiguration } from './memory/InMemoryGroupCach
 import type { GroupNotificationPublisher } from './notifications/GroupNotificationPublisher'
 import type { GroupCache } from './types/DataSources'
 import type { GetManyResult, SynchronousGroupCache } from './types/SyncDataSources'
+import {unique} from "./util/unique";
 
 export abstract class AbstractGroupCache<LoadedValue, LoadParams = string> extends AbstractCache<
   LoadedValue,
@@ -84,7 +85,7 @@ export abstract class AbstractGroupCache<LoadedValue, LoadParams = string> exten
     group: string,
     loadParams?: LoadParams,
   ): Promise<GetManyResult<LoadedValue>> {
-    // This doesn't support deduplication, and never might, as that would affect perf strongly. Maybe as an opt-in option in the future?
+    // Deduplication is handled at the getMany level for optimal performance
     return this.resolveManyGroupValues(keys, group, loadParams).then((result) => {
       for (let i = 0; i < result.resolvedValues.length; i++) {
         const resolvedValue = result.resolvedValues[i]
@@ -110,7 +111,8 @@ export abstract class AbstractGroupCache<LoadedValue, LoadParams = string> exten
     group: string,
     loadParams?: LoadParams,
   ): Promise<LoadedValue[]> {
-    const inMemoryValues = this.getManyInMemoryOnly(keys, group)
+    const uniqueKeys = unique(keys)
+    const inMemoryValues = this.getManyInMemoryOnly(uniqueKeys, group)
     // everything is in memory, hurray
     if (inMemoryValues.unresolvedKeys.length === 0) {
       return Promise.resolve(inMemoryValues.resolvedValues)

lib/util/unique.spec.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it } from 'vitest'
+import { unique } from './unique'
+
+describe('unique', () => {
+  it('returns a new array of mixed primitive value without duplicates', () => {
+    const objectA = {}
+    const objectB = {}
+    const duplicateValues = [
+      1,
+      1,
+      'a',
+      'a',
+      Number.NaN,
+      Number.NaN,
+      true,
+      true,
+      false,
+      false,
+      null,
+      null,
+      undefined,
+      undefined,
+      objectA,
+      objectA,
+      objectB,
+      objectB,
+    ]
+
+    expect(unique(duplicateValues)).toEqual([
+      1,
+      'a',
+      Number.NaN,
+      true,
+      false,
+      null,
+      undefined,
+      objectA,
+      objectB,
+    ])
+  })
+})

lib/util/unique.ts

@@ -0,0 +1 @@
+export const unique = <T>(arr: T[]): T[] => Array.from(new Set(arr))

package.json

@@ -1,6 +1,6 @@
 {
   "name": "layered-loader",
-  "version": "14.0.1",
+  "version": "14.1.0",
   "description": "Data loader with support for caching and fallback data sources ",
   "license": "MIT",
   "maintainers": [

test/GroupLoader-main.spec.ts

@@ -776,6 +776,20 @@ describe('GroupLoader Main', () => {
       expect(value2).toEqual([user1])
       expect(loader.counter).toBe(2)
     })
+
+    it('deduplicates keys in getMany with mixed cache layers', async () => {
+      const loader = new GroupLoader({
+        inMemoryCache: IN_MEMORY_CACHE_CONFIG,
+        asyncCache: new DummyGroupedCache(userValues),
+        dataSources: [new CountingGroupedLoader(userValues)],
+        cacheKeyFromValueResolver: (user: User) => user.userId,
+      })
+
+      const duplicatedKeys = ['1', '1', '2', '2', '1']
+
+      const result = await loader.getMany(duplicatedKeys, '1')
+      expect(result).toEqual([user1, user2])
+    })
   })
 
   describe('invalidateCacheFor', () => {

test/Loader-main.spec.ts

@@ -957,6 +957,18 @@ describe('Loader Main', () => {
       expect(value3).toBe('value')
       expect(loader.counter).toBe(3)
     })
+
+    it('deduplicates keys in getMany with mixed cache layers', async () => {
+      const loader = new Loader({
+        inMemoryCache: IN_MEMORY_CACHE_CONFIG,
+        asyncCache: new DummyCache('value'),
+        dataSources: [new CountingDataSource('value')],
+        cacheKeyFromValueResolver: idResolver,
+      })
+
+      const result = await loader.getMany(['key1', 'key1', 'key2', 'key2', 'key1'])
+      expect(result).toEqual(['value', 'value'])
+    })
   })
 
   describe('forceRefresh', () => {

test/fakes/CountingGroupedLoader.ts

@@ -17,7 +17,11 @@ export class CountingGroupedLoader implements GroupDataSource<User> {
     return Promise.resolve(this.groupValues?.[group]?.[key])
   }
 
-  getManyFromGroup(keys: string[], group: string, _loadParams: undefined): Promise<User[]> {
+  getManyFromGroup(
+    keys: string[],
+    group: string,
+    _loadParams: string | undefined,
+  ): Promise<User[]> {
     this.counter++
 
     const groupValues = this.groupValues?.[group] ?? {}