diff --git a/common/changes/@microsoft/rush/build-cache-override-support_2025-03-08-23-35.json b/common/changes/@microsoft/rush/build-cache-override-support_2025-03-08-23-35.json new file mode 100644 index 00000000000..09371160533 --- /dev/null +++ b/common/changes/@microsoft/rush/build-cache-override-support_2025-03-08-23-35.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add support for `RUSH_BUILD_CACHE_OVERRIDE_JSON` environment variable that takes a JSON string with the same format as the `common/config/build-cache.json` file and a `RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH` environment variable that takes a file path that can be used to override the build cache configuration that is normally provided by that file.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 42c2823ebff..798296124b0 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -237,6 +237,8 @@ export class EnvironmentConfiguration { static get allowWarningsInSuccessfulBuild(): boolean; static get buildCacheCredential(): string | undefined; static get buildCacheEnabled(): boolean | undefined; + static get buildCacheOverrideJson(): string | undefined; + static get buildCacheOverrideJsonFilePath(): string | undefined; static get buildCacheWriteAllowed(): boolean | undefined; static get cobuildContextId(): string | undefined; static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined; @@ -274,6 +276,8 @@ export const EnvironmentVariableNames: { readonly RUSH_BUILD_CACHE_CREDENTIAL: "RUSH_BUILD_CACHE_CREDENTIAL"; readonly RUSH_BUILD_CACHE_ENABLED: "RUSH_BUILD_CACHE_ENABLED"; readonly RUSH_BUILD_CACHE_WRITE_ALLOWED: "RUSH_BUILD_CACHE_WRITE_ALLOWED"; + readonly RUSH_BUILD_CACHE_OVERRIDE_JSON: "RUSH_BUILD_CACHE_OVERRIDE_JSON"; + readonly RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH: "RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH"; readonly RUSH_COBUILD_CONTEXT_ID: "RUSH_COBUILD_CONTEXT_ID"; readonly RUSH_COBUILD_RUNNER_ID: "RUSH_COBUILD_RUNNER_ID"; readonly RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED"; diff --git a/libraries/rush-lib/src/api/BuildCacheConfiguration.ts b/libraries/rush-lib/src/api/BuildCacheConfiguration.ts index 89d05d556bf..d1480c164a2 100644 --- a/libraries/rush-lib/src/api/BuildCacheConfiguration.ts +++ b/libraries/rush-lib/src/api/BuildCacheConfiguration.ts @@ -2,7 +2,6 @@ // See LICENSE in the project root for license information. import { createHash } from 'node:crypto'; -import * as path from 'path'; import { JsonFile, @@ -18,7 +17,7 @@ import { FileSystemBuildCacheProvider } from '../logic/buildCache/FileSystemBuil import { RushConstants } from '../logic/RushConstants'; import type { ICloudBuildCacheProvider } from '../logic/buildCache/ICloudBuildCacheProvider'; import { RushUserConfiguration } from './RushUserConfiguration'; -import { EnvironmentConfiguration } from './EnvironmentConfiguration'; +import { EnvironmentConfiguration, EnvironmentVariableNames } from './EnvironmentConfiguration'; import { CacheEntryId, type IGenerateCacheEntryIdOptions, @@ -83,14 +82,14 @@ interface IBuildCacheConfigurationOptions { cloudCacheProvider: ICloudBuildCacheProvider | undefined; } +const BUILD_CACHE_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson); + /** * Use this class to load and save the "common/config/rush/build-cache.json" config file. * This file provides configuration options for cached project build output. * @beta */ export class BuildCacheConfiguration { - private static _jsonSchema: JsonSchema = JsonSchema.fromLoadedObject(schemaJson); - /** * Indicates whether the build cache feature is enabled. * Typically it is enabled in the build-cache.json config file. @@ -146,11 +145,12 @@ export class BuildCacheConfiguration { rushConfiguration: RushConfiguration, rushSession: RushSession ): Promise { - const jsonFilePath: string = BuildCacheConfiguration.getBuildCacheConfigFilePath(rushConfiguration); - if (!FileSystem.exists(jsonFilePath)) { - return undefined; - } - return await BuildCacheConfiguration._loadAsync(jsonFilePath, terminal, rushConfiguration, rushSession); + const { buildCacheConfiguration } = await BuildCacheConfiguration._tryLoadInternalAsync( + terminal, + rushConfiguration, + rushSession + ); + return buildCacheConfiguration; } /** @@ -162,8 +162,13 @@ export class BuildCacheConfiguration { rushConfiguration: RushConfiguration, rushSession: RushSession ): Promise { - const jsonFilePath: string = BuildCacheConfiguration.getBuildCacheConfigFilePath(rushConfiguration); - if (!FileSystem.exists(jsonFilePath)) { + const { buildCacheConfiguration, jsonFilePath } = await BuildCacheConfiguration._tryLoadInternalAsync( + terminal, + rushConfiguration, + rushSession + ); + + if (!buildCacheConfiguration) { terminal.writeErrorLine( `The build cache feature is not enabled. This config file is missing:\n` + jsonFilePath ); @@ -171,13 +176,6 @@ export class BuildCacheConfiguration { throw new AlreadyReportedError(); } - const buildCacheConfiguration: BuildCacheConfiguration = await BuildCacheConfiguration._loadAsync( - jsonFilePath, - terminal, - rushConfiguration, - rushSession - ); - if (!buildCacheConfiguration.buildCacheEnabled) { terminal.writeErrorLine( `The build cache feature is not enabled. You can enable it by editing this config file:\n` + @@ -185,6 +183,7 @@ export class BuildCacheConfiguration { ); throw new AlreadyReportedError(); } + return buildCacheConfiguration; } @@ -192,21 +191,56 @@ export class BuildCacheConfiguration { * Gets the absolute path to the build-cache.json file in the specified rush workspace. */ public static getBuildCacheConfigFilePath(rushConfiguration: RushConfiguration): string { - return path.resolve(rushConfiguration.commonRushConfigFolder, RushConstants.buildCacheFilename); + return `${rushConfiguration.commonRushConfigFolder}/${RushConstants.buildCacheFilename}`; + } + + private static async _tryLoadInternalAsync( + terminal: ITerminal, + rushConfiguration: RushConfiguration, + rushSession: RushSession + ): Promise<{ buildCacheConfiguration: BuildCacheConfiguration | undefined; jsonFilePath: string }> { + const jsonFilePath: string = BuildCacheConfiguration.getBuildCacheConfigFilePath(rushConfiguration); + const buildCacheConfiguration: BuildCacheConfiguration | undefined = + await BuildCacheConfiguration._tryLoadAsync(jsonFilePath, terminal, rushConfiguration, rushSession); + return { buildCacheConfiguration, jsonFilePath }; } - private static async _loadAsync( + private static async _tryLoadAsync( jsonFilePath: string, terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession - ): Promise { - const buildCacheJson: IBuildCacheJson = await JsonFile.loadAndValidateAsync( - jsonFilePath, - BuildCacheConfiguration._jsonSchema - ); - const rushUserConfiguration: RushUserConfiguration = await RushUserConfiguration.initializeAsync(); + ): Promise { + let buildCacheJson: IBuildCacheJson; + const buildCacheOverrideJson: string | undefined = EnvironmentConfiguration.buildCacheOverrideJson; + if (buildCacheOverrideJson) { + buildCacheJson = JsonFile.parseString(buildCacheOverrideJson); + BUILD_CACHE_JSON_SCHEMA.validateObject( + buildCacheJson, + `${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON} environment variable` + ); + } else { + const buildCacheOverrideJsonFilePath: string | undefined = + EnvironmentConfiguration.buildCacheOverrideJsonFilePath; + if (buildCacheOverrideJsonFilePath) { + buildCacheJson = await JsonFile.loadAndValidateAsync( + buildCacheOverrideJsonFilePath, + BUILD_CACHE_JSON_SCHEMA + ); + } else { + try { + buildCacheJson = await JsonFile.loadAndValidateAsync(jsonFilePath, BUILD_CACHE_JSON_SCHEMA); + } catch (e) { + if (!FileSystem.isNotExistError(e)) { + throw e; + } else { + return undefined; + } + } + } + } + const rushUserConfiguration: RushUserConfiguration = await RushUserConfiguration.initializeAsync(); let innerGetCacheEntryId: GetCacheEntryIdFunction; try { innerGetCacheEntryId = CacheEntryId.parsePattern(buildCacheJson.cacheEntryNamePattern); @@ -218,7 +252,9 @@ export class BuildCacheConfiguration { } const { cacheHashSalt = '', cacheProvider } = buildCacheJson; - const salt: string = `${RushConstants.buildCacheVersion}${cacheHashSalt ? `${RushConstants.hashDelimiter}${cacheHashSalt}` : ''}`; + const salt: string = `${RushConstants.buildCacheVersion}${ + cacheHashSalt ? `${RushConstants.hashDelimiter}${cacheHashSalt}` : '' + }`; // Extend the cache entry id with to salt the hash // This facilitates forcing cache invalidation either when the build cache version changes (new version of Rush) // or when the user-side salt changes (need to purge bad cache entries, plugins including additional files) diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 47738daefcc..d8c9a16b4af 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -144,6 +144,34 @@ export const EnvironmentVariableNames = { */ RUSH_BUILD_CACHE_WRITE_ALLOWED: 'RUSH_BUILD_CACHE_WRITE_ALLOWED', + /** + * Set this environment variable to a JSON string to override the build cache configuration that normally lives + * at `common/config/rush/build-cache.json`. + * + * This is useful for testing purposes, or for OSS repos that are have a local-only cache, but can have + * a different cache configuration in CI/CD pipelines. + * + * @remarks + * This is similar to {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH}, but it allows you to specify + * a JSON string instead of a file path. The two environment variables are mutually exclusive, meaning you can + * only use one of them at a time. + */ + RUSH_BUILD_CACHE_OVERRIDE_JSON: 'RUSH_BUILD_CACHE_OVERRIDE_JSON', + + /** + * Set this environment variable to the path to a `build-cache.json` file to override the build cache configuration + * that normally lives at `common/config/rush/build-cache.json`. + * + * This is useful for testing purposes, or for OSS repos that are have a local-only cache, but can have + * a different cache configuration in CI/CD pipelines. + * + * @remarks + * This is similar to {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON}, but it allows you to specify + * a file path instead of a JSON string. The two environment variables are mutually exclusive, meaning you can + * only use one of them at a time. + */ + RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH: 'RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH', + /** * Setting this environment variable opts into running with cobuilds. The context id should be the same across * multiple VMs, but changed when it is a new round of cobuilds. @@ -250,6 +278,10 @@ export class EnvironmentConfiguration { private static _buildCacheWriteAllowed: boolean | undefined; + private static _buildCacheOverrideJson: string | undefined; + + private static _buildCacheOverrideJsonFilePath: string | undefined; + private static _cobuildContextId: string | undefined; private static _cobuildRunnerId: string | undefined; @@ -360,6 +392,24 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._buildCacheWriteAllowed; } + /** + * If set, overrides the build cache configuration that normally lives at `common/config/rush/build-cache.json`. + * See {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON} + */ + public static get buildCacheOverrideJson(): string | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._buildCacheOverrideJson; + } + + /** + * If set, overrides the build cache configuration that normally lives at `common/config/rush/build-cache.json`. + * See {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH} + */ + public static get buildCacheOverrideJsonFilePath(): string | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._buildCacheOverrideJsonFilePath; + } + /** * Provides a determined cobuild context id if configured * See {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID} @@ -517,6 +567,16 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON: { + EnvironmentConfiguration._buildCacheOverrideJson = value; + break; + } + + case EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH: { + EnvironmentConfiguration._buildCacheOverrideJsonFilePath = value; + break; + } + case EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID: { EnvironmentConfiguration._cobuildContextId = value; break; @@ -578,6 +638,17 @@ export class EnvironmentConfiguration { ); } + if ( + EnvironmentConfiguration._buildCacheOverrideJsonFilePath && + EnvironmentConfiguration._buildCacheOverrideJson + ) { + throw new Error( + `Environment variable ${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH} and ` + + `${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON} are mutually exclusive. ` + + `Only one may be specified.` + ); + } + // See doc comment for EnvironmentConfiguration._getRushGlobalFolderOverride(). EnvironmentConfiguration._rushGlobalFolderOverride = EnvironmentConfiguration._getRushGlobalFolderOverride(process.env);