[rush-resolver-cache] Fix subpackage cache optimization

common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-56.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Dedupe shrinkwrap parsing by content hash.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-57.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "[resolver-cache] Use shrinkwrap hash to skip resolver cache regeneration.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

libraries/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts

@@ -291,10 +291,9 @@ export function normalizePnpmVersionSpecifier(versionSpecifier: IPnpmVersionSpec
   }
 }
 
-export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
-  // TODO: Implement cache eviction when a lockfile is copied back
-  private static _cacheByLockfilePath: Map<string, PnpmShrinkwrapFile | undefined> = new Map();
+const cacheByLockfileHash: Map<string, PnpmShrinkwrapFile | undefined> = new Map();
 
+export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
   public readonly shrinkwrapFileMajorVersion: number;
   public readonly isWorkspaceCompatible: boolean;
   public readonly registry: string;
@@ -304,14 +303,17 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
   public readonly packages: ReadonlyMap<string, IPnpmShrinkwrapDependencyYaml>;
   public readonly overrides: ReadonlyMap<string, string>;
   public readonly packageExtensionsChecksum: undefined | string;
+  public readonly hash: string;
 
   private readonly _shrinkwrapJson: IPnpmShrinkwrapYaml;
   private readonly _integrities: Map<string, Map<string, string>>;
   private _pnpmfileConfiguration: PnpmfileConfiguration | undefined;
 
-  private constructor(shrinkwrapJson: IPnpmShrinkwrapYaml) {
+  private constructor(shrinkwrapJson: IPnpmShrinkwrapYaml, hash: string) {
     super();
+    this.hash = hash;
     this._shrinkwrapJson = shrinkwrapJson;
+    cacheByLockfileHash.set(hash, this);
 
     // Normalize the data
     const lockfileVersion: string | number | undefined = shrinkwrapJson.lockfileVersion;
@@ -362,33 +364,35 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
     return dependencyPath.removeSuffix(version).includes('@', 1) ? version : `${name}@${version}`;
   }
 
+  /**
+   * Clears the cache of PnpmShrinkwrapFile instances to free up memory.
+   */
+  public static clearCache(): void {
+    cacheByLockfileHash.clear();
+  }
+
   public static loadFromFile(
     shrinkwrapYamlFilePath: string,
-    { withCaching }: ILoadFromFileOptions = {}
+    options: ILoadFromFileOptions = {}
   ): PnpmShrinkwrapFile | undefined {
-    let loaded: PnpmShrinkwrapFile | undefined;
-    if (withCaching) {
-      loaded = PnpmShrinkwrapFile._cacheByLockfilePath.get(shrinkwrapYamlFilePath);
-    }
-
-    // TODO: Promisify this
-    loaded ??= (() => {
-      try {
-        const shrinkwrapContent: string = FileSystem.readFile(shrinkwrapYamlFilePath);
-        return PnpmShrinkwrapFile.loadFromString(shrinkwrapContent);
-      } catch (error) {
-        if (FileSystem.isNotExistError(error as Error)) {
-          return undefined; // file does not exist
-        }
-        throw new Error(`Error reading "${shrinkwrapYamlFilePath}":\n  ${(error as Error).message}`);
+    try {
+      const shrinkwrapContent: string = FileSystem.readFile(shrinkwrapYamlFilePath);
+      return PnpmShrinkwrapFile.loadFromString(shrinkwrapContent);
+    } catch (error) {
+      if (FileSystem.isNotExistError(error as Error)) {
+        return undefined; // file does not exist
       }
-    })();
-
-    PnpmShrinkwrapFile._cacheByLockfilePath.set(shrinkwrapYamlFilePath, loaded);
-    return loaded;
+      throw new Error(`Error reading "${shrinkwrapYamlFilePath}":\n  ${(error as Error).message}`);
+    }
   }
 
   public static loadFromString(shrinkwrapContent: string): PnpmShrinkwrapFile {
+    const hash: string = crypto.createHash('sha-256').update(shrinkwrapContent, 'utf8').digest('hex');
+    const cached: PnpmShrinkwrapFile | undefined = cacheByLockfileHash.get(hash);
+    if (cached) {
+      return cached;
+    }
+
     const shrinkwrapJson: IPnpmShrinkwrapYaml = yamlModule.safeLoad(shrinkwrapContent);
     if ((shrinkwrapJson as LockfileFileV9).snapshots) {
       const lockfile: IPnpmShrinkwrapYaml | null = convertLockfileV9ToLockfileObject(
@@ -418,10 +422,10 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
           lockfile.dependencies[name] = PnpmShrinkwrapFile.getLockfileV9PackageId(name, versionSpecifier);
         }
       }
-      return new PnpmShrinkwrapFile(lockfile);
+      return new PnpmShrinkwrapFile(lockfile, hash);
     }
 
-    return new PnpmShrinkwrapFile(shrinkwrapJson);
+    return new PnpmShrinkwrapFile(shrinkwrapJson, hash);
   }
 
   public getShrinkwrapHash(experimentsConfig?: IExperimentsJson): string {

rush-plugins/rush-resolver-cache-plugin/src/afterInstallAsync.ts

@@ -41,7 +41,18 @@ function getPlatformInfo(): IPlatformInfo {
 }
 
 const END_TOKEN: string = '/package.json":';
-const SUBPACKAGE_CACHE_FILE_VERSION: 1 = 1;
+const RESOLVER_CACHE_FILE_VERSION: 1 = 1;
+
+interface IExtendedResolverCacheFile extends IResolverCacheFile {
+  /**
+   * The hash of the shrinkwrap file this cache file was generated from.
+   */
+  shrinkwrapHash: string;
+  /**
+   * The version of the resolver cache file.
+   */
+  version: number;
+}
 
 interface INestedPackageJsonCache {
   subPackagesByIntegrity: [string, string[] | boolean][];
@@ -73,19 +84,30 @@ export async function afterInstallAsync(
   terminal.writeLine(`Using pnpm-lock from: ${lockFilePath}`);
   terminal.writeLine(`Using pnpm store folder: ${pnpmStoreDir}`);
 
+  const workspaceRoot: string = subspace.getSubspaceTempFolderPath();
+  const cacheFilePath: string = `${workspaceRoot}/resolver-cache.json`;
+
   const lockFile: PnpmShrinkwrapFile | undefined = PnpmShrinkwrapFile.loadFromFile(lockFilePath, {
     withCaching: true
   });
   if (!lockFile) {
     throw new Error(`Failed to load shrinkwrap file: ${lockFilePath}`);
   }
 
-  const workspaceRoot: string = subspace.getSubspaceTempFolderPath();
+  try {
+    const oldCacheFileContent: string = await FileSystem.readFileAsync(cacheFilePath);
+    const oldCache: IExtendedResolverCacheFile = JSON.parse(oldCacheFileContent);
+    if (oldCache.version === RESOLVER_CACHE_FILE_VERSION && oldCache.shrinkwrapHash === lockFile.hash) {
+      // Cache is valid, use it
+      return;
+    }
+  } catch (err) {
+    // Ignore
+  }
 
   const projectByImporterPath: LookupByPath<RushConfigurationProject> =
     rushConfiguration.getProjectLookupForRoot(workspaceRoot);
 
-  const cacheFilePath: string = `${workspaceRoot}/resolver-cache.json`;
   const subPackageCacheFilePath: string = `${workspaceRoot}/subpackage-entry-cache.json`;
 
   terminal.writeLine(`Resolver cache will be written at ${cacheFilePath}`);
@@ -95,9 +117,9 @@ export async function afterInstallAsync(
   try {
     const cacheContent: string = await FileSystem.readFileAsync(subPackageCacheFilePath);
     const cacheJson: INestedPackageJsonCache = JSON.parse(cacheContent);
-    if (cacheJson.version !== SUBPACKAGE_CACHE_FILE_VERSION) {
+    if (cacheJson.version !== RESOLVER_CACHE_FILE_VERSION) {
       terminal.writeLine(
-        `Expected subpackage cache version ${SUBPACKAGE_CACHE_FILE_VERSION}, got ${cacheJson.version}`
+        `Expected subpackage cache version ${RESOLVER_CACHE_FILE_VERSION}, got ${cacheJson.version}`
       );
     } else {
       oldSubPackagesByIntegrity = new Map(cacheJson.subPackagesByIntegrity);
@@ -204,9 +226,8 @@ export async function afterInstallAsync(
         terminal.writeDebugLine(
           `Nested "package.json" files found for package at ${descriptionFileRoot}: ${result.join(', ')}`
         );
-        // Clone this array to ensure that mutations don't affect the subpackage cache.
         // eslint-disable-next-line require-atomic-updates
-        context.nestedPackageDirs = [...result];
+        context.nestedPackageDirs = result;
       }
     }
 
@@ -218,22 +239,7 @@ export async function afterInstallAsync(
     });
   }
 
-  // Serialize this before `computeResolverCacheFromLockfileAsync` because bundledDependencies get removed
-  // from the `nestedPackageDirs` array. We clone above for safety, but this is making doubly sure.
-  const newSubPackageCache: INestedPackageJsonCache = {
-    version: SUBPACKAGE_CACHE_FILE_VERSION,
-    subPackagesByIntegrity: Array.from(subPackagesByIntegrity)
-  };
-  const serializedSubpackageCache: string = JSON.stringify(newSubPackageCache);
-  const writeSubPackageCachePromise: Promise<void> = FileSystem.writeFileAsync(
-    subPackageCacheFilePath,
-    serializedSubpackageCache,
-    {
-      ensureFolderExists: true
-    }
-  );
-
-  const cacheFile: IResolverCacheFile = await computeResolverCacheFromLockfileAsync({
+  const rawCacheFile: IResolverCacheFile = await computeResolverCacheFromLockfileAsync({
     workspaceRoot,
     commonPrefixToTrim: rushRoot,
     platformInfo: getPlatformInfo(),
@@ -242,14 +248,31 @@ export async function afterInstallAsync(
     afterExternalPackagesAsync
   });
 
-  const serialized: string = JSON.stringify(cacheFile);
+  const extendedCacheFile: IExtendedResolverCacheFile = {
+    version: RESOLVER_CACHE_FILE_VERSION,
+    shrinkwrapHash: lockFile.hash,
+    ...rawCacheFile
+  };
+
+  const newSubPackageCache: INestedPackageJsonCache = {
+    version: RESOLVER_CACHE_FILE_VERSION,
+    subPackagesByIntegrity: Array.from(subPackagesByIntegrity)
+  };
+  const serializedSubpackageCache: string = JSON.stringify(newSubPackageCache);
+
+  const serialized: string = JSON.stringify(extendedCacheFile);
 
   await Promise.all([
     FileSystem.writeFileAsync(cacheFilePath, serialized, {
       ensureFolderExists: true
     }),
-    writeSubPackageCachePromise
+    FileSystem.writeFileAsync(subPackageCacheFilePath, serializedSubpackageCache, {
+      ensureFolderExists: true
+    })
   ]);
 
+  // Free the memory used by the lockfiles, since nothing should read the lockfile from this point on.
+  PnpmShrinkwrapFile.clearCache();
+
   terminal.writeLine(`Resolver cache written.`);
 }

rush-plugins/rush-resolver-cache-plugin/src/computeResolverCacheFromLockfileAsync.ts

@@ -49,11 +49,12 @@ function extractBundledDependencies(
   contexts: Map<string, IResolverContext>,
   context: IResolverContext
 ): void {
-  const { nestedPackageDirs } = context;
+  let { nestedPackageDirs } = context;
   if (!nestedPackageDirs) {
     return;
   }
 
+  let foundBundledDependencies: boolean = false;
   for (let i: number = nestedPackageDirs.length - 1; i >= 0; i--) {
     const nestedDir: string = nestedPackageDirs[i];
     if (!nestedDir.startsWith('node_modules/')) {
@@ -71,6 +72,12 @@ function extractBundledDependencies(
       continue;
     }
 
+    if (!foundBundledDependencies) {
+      foundBundledDependencies = true;
+      // Make a copy of the nestedPackageDirs array so that we don't mutate the version being
+      // saved into the subpackage index cache.
+      context.nestedPackageDirs = nestedPackageDirs = nestedPackageDirs.slice(0);
+    }
     // Remove this nested package from the list
     nestedPackageDirs.splice(i, 1);