diff --git a/common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-56.json b/common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-56.json new file mode 100644 index 00000000000..cec0c2118c7 --- /dev/null +++ b/common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-56.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Dedupe shrinkwrap parsing by content hash.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-57.json b/common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-57.json new file mode 100644 index 00000000000..aeee66f7ea1 --- /dev/null +++ b/common/changes/@microsoft/rush/resolver-cache-fixes_2025-08-28-19-57.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "[resolver-cache] Use shrinkwrap hash to skip resolver cache regeneration.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/libraries/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts b/libraries/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts index e6ba5b3a8d0..39872aa960f 100644 --- a/libraries/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts +++ b/libraries/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts @@ -291,10 +291,9 @@ export function normalizePnpmVersionSpecifier(versionSpecifier: IPnpmVersionSpec } } -export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { - // TODO: Implement cache eviction when a lockfile is copied back - private static _cacheByLockfilePath: Map = new Map(); +const cacheByLockfileHash: Map = new Map(); +export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { public readonly shrinkwrapFileMajorVersion: number; public readonly isWorkspaceCompatible: boolean; public readonly registry: string; @@ -304,14 +303,17 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { public readonly packages: ReadonlyMap; public readonly overrides: ReadonlyMap; public readonly packageExtensionsChecksum: undefined | string; + public readonly hash: string; private readonly _shrinkwrapJson: IPnpmShrinkwrapYaml; private readonly _integrities: Map>; private _pnpmfileConfiguration: PnpmfileConfiguration | undefined; - private constructor(shrinkwrapJson: IPnpmShrinkwrapYaml) { + private constructor(shrinkwrapJson: IPnpmShrinkwrapYaml, hash: string) { super(); + this.hash = hash; this._shrinkwrapJson = shrinkwrapJson; + cacheByLockfileHash.set(hash, this); // Normalize the data const lockfileVersion: string | number | undefined = shrinkwrapJson.lockfileVersion; @@ -362,33 +364,35 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { return dependencyPath.removeSuffix(version).includes('@', 1) ? version : `${name}@${version}`; } + /** + * Clears the cache of PnpmShrinkwrapFile instances to free up memory. + */ + public static clearCache(): void { + cacheByLockfileHash.clear(); + } + public static loadFromFile( shrinkwrapYamlFilePath: string, - { withCaching }: ILoadFromFileOptions = {} + options: ILoadFromFileOptions = {} ): PnpmShrinkwrapFile | undefined { - let loaded: PnpmShrinkwrapFile | undefined; - if (withCaching) { - loaded = PnpmShrinkwrapFile._cacheByLockfilePath.get(shrinkwrapYamlFilePath); - } - - // TODO: Promisify this - loaded ??= (() => { - try { - const shrinkwrapContent: string = FileSystem.readFile(shrinkwrapYamlFilePath); - return PnpmShrinkwrapFile.loadFromString(shrinkwrapContent); - } catch (error) { - if (FileSystem.isNotExistError(error as Error)) { - return undefined; // file does not exist - } - throw new Error(`Error reading "${shrinkwrapYamlFilePath}":\n ${(error as Error).message}`); + try { + const shrinkwrapContent: string = FileSystem.readFile(shrinkwrapYamlFilePath); + return PnpmShrinkwrapFile.loadFromString(shrinkwrapContent); + } catch (error) { + if (FileSystem.isNotExistError(error as Error)) { + return undefined; // file does not exist } - })(); - - PnpmShrinkwrapFile._cacheByLockfilePath.set(shrinkwrapYamlFilePath, loaded); - return loaded; + throw new Error(`Error reading "${shrinkwrapYamlFilePath}":\n ${(error as Error).message}`); + } } public static loadFromString(shrinkwrapContent: string): PnpmShrinkwrapFile { + const hash: string = crypto.createHash('sha-256').update(shrinkwrapContent, 'utf8').digest('hex'); + const cached: PnpmShrinkwrapFile | undefined = cacheByLockfileHash.get(hash); + if (cached) { + return cached; + } + const shrinkwrapJson: IPnpmShrinkwrapYaml = yamlModule.safeLoad(shrinkwrapContent); if ((shrinkwrapJson as LockfileFileV9).snapshots) { const lockfile: IPnpmShrinkwrapYaml | null = convertLockfileV9ToLockfileObject( @@ -418,10 +422,10 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { lockfile.dependencies[name] = PnpmShrinkwrapFile.getLockfileV9PackageId(name, versionSpecifier); } } - return new PnpmShrinkwrapFile(lockfile); + return new PnpmShrinkwrapFile(lockfile, hash); } - return new PnpmShrinkwrapFile(shrinkwrapJson); + return new PnpmShrinkwrapFile(shrinkwrapJson, hash); } public getShrinkwrapHash(experimentsConfig?: IExperimentsJson): string { diff --git a/rush-plugins/rush-resolver-cache-plugin/src/afterInstallAsync.ts b/rush-plugins/rush-resolver-cache-plugin/src/afterInstallAsync.ts index f8557042abc..72b9c631e76 100644 --- a/rush-plugins/rush-resolver-cache-plugin/src/afterInstallAsync.ts +++ b/rush-plugins/rush-resolver-cache-plugin/src/afterInstallAsync.ts @@ -41,7 +41,18 @@ function getPlatformInfo(): IPlatformInfo { } const END_TOKEN: string = '/package.json":'; -const SUBPACKAGE_CACHE_FILE_VERSION: 1 = 1; +const RESOLVER_CACHE_FILE_VERSION: 1 = 1; + +interface IExtendedResolverCacheFile extends IResolverCacheFile { + /** + * The hash of the shrinkwrap file this cache file was generated from. + */ + shrinkwrapHash: string; + /** + * The version of the resolver cache file. + */ + version: number; +} interface INestedPackageJsonCache { subPackagesByIntegrity: [string, string[] | boolean][]; @@ -73,6 +84,9 @@ export async function afterInstallAsync( terminal.writeLine(`Using pnpm-lock from: ${lockFilePath}`); terminal.writeLine(`Using pnpm store folder: ${pnpmStoreDir}`); + const workspaceRoot: string = subspace.getSubspaceTempFolderPath(); + const cacheFilePath: string = `${workspaceRoot}/resolver-cache.json`; + const lockFile: PnpmShrinkwrapFile | undefined = PnpmShrinkwrapFile.loadFromFile(lockFilePath, { withCaching: true }); @@ -80,12 +94,20 @@ export async function afterInstallAsync( throw new Error(`Failed to load shrinkwrap file: ${lockFilePath}`); } - const workspaceRoot: string = subspace.getSubspaceTempFolderPath(); + try { + const oldCacheFileContent: string = await FileSystem.readFileAsync(cacheFilePath); + const oldCache: IExtendedResolverCacheFile = JSON.parse(oldCacheFileContent); + if (oldCache.version === RESOLVER_CACHE_FILE_VERSION && oldCache.shrinkwrapHash === lockFile.hash) { + // Cache is valid, use it + return; + } + } catch (err) { + // Ignore + } const projectByImporterPath: LookupByPath = rushConfiguration.getProjectLookupForRoot(workspaceRoot); - const cacheFilePath: string = `${workspaceRoot}/resolver-cache.json`; const subPackageCacheFilePath: string = `${workspaceRoot}/subpackage-entry-cache.json`; terminal.writeLine(`Resolver cache will be written at ${cacheFilePath}`); @@ -95,9 +117,9 @@ export async function afterInstallAsync( try { const cacheContent: string = await FileSystem.readFileAsync(subPackageCacheFilePath); const cacheJson: INestedPackageJsonCache = JSON.parse(cacheContent); - if (cacheJson.version !== SUBPACKAGE_CACHE_FILE_VERSION) { + if (cacheJson.version !== RESOLVER_CACHE_FILE_VERSION) { terminal.writeLine( - `Expected subpackage cache version ${SUBPACKAGE_CACHE_FILE_VERSION}, got ${cacheJson.version}` + `Expected subpackage cache version ${RESOLVER_CACHE_FILE_VERSION}, got ${cacheJson.version}` ); } else { oldSubPackagesByIntegrity = new Map(cacheJson.subPackagesByIntegrity); @@ -204,9 +226,8 @@ export async function afterInstallAsync( terminal.writeDebugLine( `Nested "package.json" files found for package at ${descriptionFileRoot}: ${result.join(', ')}` ); - // Clone this array to ensure that mutations don't affect the subpackage cache. // eslint-disable-next-line require-atomic-updates - context.nestedPackageDirs = [...result]; + context.nestedPackageDirs = result; } } @@ -218,22 +239,7 @@ export async function afterInstallAsync( }); } - // Serialize this before `computeResolverCacheFromLockfileAsync` because bundledDependencies get removed - // from the `nestedPackageDirs` array. We clone above for safety, but this is making doubly sure. - const newSubPackageCache: INestedPackageJsonCache = { - version: SUBPACKAGE_CACHE_FILE_VERSION, - subPackagesByIntegrity: Array.from(subPackagesByIntegrity) - }; - const serializedSubpackageCache: string = JSON.stringify(newSubPackageCache); - const writeSubPackageCachePromise: Promise = FileSystem.writeFileAsync( - subPackageCacheFilePath, - serializedSubpackageCache, - { - ensureFolderExists: true - } - ); - - const cacheFile: IResolverCacheFile = await computeResolverCacheFromLockfileAsync({ + const rawCacheFile: IResolverCacheFile = await computeResolverCacheFromLockfileAsync({ workspaceRoot, commonPrefixToTrim: rushRoot, platformInfo: getPlatformInfo(), @@ -242,14 +248,31 @@ export async function afterInstallAsync( afterExternalPackagesAsync }); - const serialized: string = JSON.stringify(cacheFile); + const extendedCacheFile: IExtendedResolverCacheFile = { + version: RESOLVER_CACHE_FILE_VERSION, + shrinkwrapHash: lockFile.hash, + ...rawCacheFile + }; + + const newSubPackageCache: INestedPackageJsonCache = { + version: RESOLVER_CACHE_FILE_VERSION, + subPackagesByIntegrity: Array.from(subPackagesByIntegrity) + }; + const serializedSubpackageCache: string = JSON.stringify(newSubPackageCache); + + const serialized: string = JSON.stringify(extendedCacheFile); await Promise.all([ FileSystem.writeFileAsync(cacheFilePath, serialized, { ensureFolderExists: true }), - writeSubPackageCachePromise + FileSystem.writeFileAsync(subPackageCacheFilePath, serializedSubpackageCache, { + ensureFolderExists: true + }) ]); + // Free the memory used by the lockfiles, since nothing should read the lockfile from this point on. + PnpmShrinkwrapFile.clearCache(); + terminal.writeLine(`Resolver cache written.`); } diff --git a/rush-plugins/rush-resolver-cache-plugin/src/computeResolverCacheFromLockfileAsync.ts b/rush-plugins/rush-resolver-cache-plugin/src/computeResolverCacheFromLockfileAsync.ts index dca5e841a28..e0aff3acbf3 100644 --- a/rush-plugins/rush-resolver-cache-plugin/src/computeResolverCacheFromLockfileAsync.ts +++ b/rush-plugins/rush-resolver-cache-plugin/src/computeResolverCacheFromLockfileAsync.ts @@ -49,11 +49,12 @@ function extractBundledDependencies( contexts: Map, context: IResolverContext ): void { - const { nestedPackageDirs } = context; + let { nestedPackageDirs } = context; if (!nestedPackageDirs) { return; } + let foundBundledDependencies: boolean = false; for (let i: number = nestedPackageDirs.length - 1; i >= 0; i--) { const nestedDir: string = nestedPackageDirs[i]; if (!nestedDir.startsWith('node_modules/')) { @@ -71,6 +72,12 @@ function extractBundledDependencies( continue; } + if (!foundBundledDependencies) { + foundBundledDependencies = true; + // Make a copy of the nestedPackageDirs array so that we don't mutate the version being + // saved into the subpackage index cache. + context.nestedPackageDirs = nestedPackageDirs = nestedPackageDirs.slice(0); + } // Remove this nested package from the list nestedPackageDirs.splice(i, 1);