From ceff709119049ae4940b9a9d68a3b9b1ba749b57 Mon Sep 17 00:00:00 2001 From: David Michon Date: Tue, 1 Jul 2025 21:37:41 +0000 Subject: [PATCH 1/2] [rush-bridge-cache] Support read or write of cache --- .../bridge-cache-both_2025-07-01-21-37.json | 10 ++ common/reviews/api/rush-lib.api.md | 1 + .../cli/scriptActions/PhasedScriptAction.ts | 6 +- .../src/pluginFramework/PhasedCommandHooks.ts | 4 + .../rush-bridge-cache-plugin/README.md | 36 +++- .../rush-plugin-manifest.json | 2 +- .../src/BridgeCachePlugin.ts | 168 +++++++++++------- .../schemas/bridge-cache-config.schema.json | 6 +- 8 files changed, 155 insertions(+), 78 deletions(-) create mode 100644 common/changes/@microsoft/rush/bridge-cache-both_2025-07-01-21-37.json diff --git a/common/changes/@microsoft/rush/bridge-cache-both_2025-07-01-21-37.json b/common/changes/@microsoft/rush/bridge-cache-both_2025-07-01-21-37.json new file mode 100644 index 00000000000..5638e9a7f75 --- /dev/null +++ b/common/changes/@microsoft/rush/bridge-cache-both_2025-07-01-21-37.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Include \"parallelism\" in phased operation execution context. Update \"rush-bridge-cache-plugin\" to support both cache read and cache write, selectable via command line choice parameter. Fixes an issue that the options schema for \"rush-bridge-cache-plugin\" was invalid.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 9e95089280e..1d64af11050 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -417,6 +417,7 @@ export interface ICreateOperationsContext { readonly isIncrementalBuildAllowed: boolean; readonly isInitial: boolean; readonly isWatch: boolean; + readonly parallelism: number; readonly phaseOriginal: ReadonlySet; readonly phaseSelection: ReadonlySet; readonly projectConfigurations: ReadonlyMap; diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 6c201285c97..355176763e0 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -538,6 +538,7 @@ export class PhasedScriptAction extends BaseScriptAction { isInitial: true, isWatch, rushConfiguration: this.rushConfiguration, + parallelism, phaseOriginal: new Set(this._originalPhases), phaseSelection: new Set(this._initialPhases), includePhaseDeps, @@ -821,9 +822,8 @@ export class PhasedScriptAction extends BaseScriptAction { // Loop until Ctrl+C while (!abortSignal.aborted) { // On the initial invocation, this promise will return immediately with the full set of projects - const { changedProjects, inputsSnapshot: state } = await projectWatcher.waitForChangeAsync( - onWaitingForChanges - ); + const { changedProjects, inputsSnapshot: state } = + await projectWatcher.waitForChangeAsync(onWaitingForChanges); if (abortSignal.aborted) { return; diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 4b665b7bdca..b6599398df1 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -77,6 +77,10 @@ export interface ICreateOperationsContext { * If true, the command is running in watch mode. */ readonly isWatch: boolean; + /** + * The currently configured maximum parallelism for the command. + */ + readonly parallelism: number; /** * The set of phases original for the current command execution. */ diff --git a/rush-plugins/rush-bridge-cache-plugin/README.md b/rush-plugins/rush-bridge-cache-plugin/README.md index e7e445de337..bc2ba2f975f 100644 --- a/rush-plugins/rush-bridge-cache-plugin/README.md +++ b/rush-plugins/rush-bridge-cache-plugin/README.md @@ -1,13 +1,16 @@ # @rushstack/rush-bridge-cache-plugin -This is a Rush plugin that lets you to add an optional flag to Rush's phased commands to bypass the actual _action_ of the script (build, test, lint - whatever you have configured), and just populate the cache from the action as though the action had already been performed by Rush. The flag name is configurable. +This is a Rush plugin that lets you to add an optional parameter to Rush's phased commands to bypass the actual _action_ of the script (build, test, lint - whatever you have configured), and just populate the cache from the action as though the action had already been performed by Rush, or to perform a best-effort restore from cache. The parameter name is configurable. -This is useful for integrations with other build orchestrators such as BuildXL. You can use those to do the work of actually running the task, then run the equivalent Rush command afterwards with a `--set-cache-only` to populate the Rush cache with whatever had been generated on disk, in addition to whatever cache mechanism is used by the other build orchestrator. +This is useful for integrations with other build orchestrators such as BuildXL. You can use those to do the work of actually running the task, then run the equivalent Rush command afterwards with a `--bridge-cache-action=write` to populate the Rush cache with whatever had been generated on disk, in addition to whatever cache mechanism is used by the other build orchestrator. + +Alternatively, the `--bridge-cache-action=read` parameter is useful for tasks such as GitHub Codespaces Prebuilds, where the agent has limited computational power and the job is a best-effort to accelerate the developer flow. ## Here be dragons! -This plugin assumes that the work for a particular task has already been completed and the build artifacts have been generated on disk. **If you run this command on a package where the command hasn't already been run and the build artifacts are missing or incorrect, you will cache invalid content**. Be careful and beware! +The `write` action for plugin assumes that the work for a particular task has already been completed and the build artifacts have been generated on disk. **If you run this command on a package where the command hasn't already been run and the build artifacts are missing or incorrect, you will cache invalid content**. Be careful and beware! +The `read` action for this plugin makes no guarantee that the requested operations will have their outputs restored and is purely a best-effort. ## Installation @@ -17,9 +20,20 @@ This plugin assumes that the work for a particular task has already been complet ```json { "associatedCommands": ["build", "test", "lint", "a11y", "typecheck"], - "description": "When the flag is added to any associated command, it'll bypass running the command itself, and cache whatever it finds on disk for the action. Beware! Only run when you know the build artifacts are in a valid state for the command.", - "parameterKind": "flag", - "longName": "--set-cache-only" + "description": "Danger! This parameter is meant for use in tools and as part of larger workflows that guarantee the state of the build folder.", + "parameterKind": "choice", + "longName": "--bridge-cache-action", + "required": false, + "alternatives": [ + { + "name": "read", + "description": "When specified for any associated command, attempt to restore the outputs from the build cache, but will not perform an actual build in the event of cache misses. Beware! If not all cache entries are available, some operations will be left unbuilt." + }, + { + "name": "write", + "description": "When specified for any associated command, bypass running the command itself, and cache whatever outputs exist in the output folders as-is. Beware! Only run when you know the build artifacts are in a valid state for the command." + } + ] } ``` @@ -35,16 +49,20 @@ This plugin assumes that the work for a particular task has already been complet 4. Create a configuration file for this plugin at this location: `common/config/rush-plugins/rush-bridge-cache-plugin.json` that defines the flag name you'll use to trigger the plugin: ```json { - "flagName": "--set-cache-only" + "actionParameterName": "--bridge-cache-action" } ``` ## Usage -You can now add the flag to any Rush phased command, e.g. +You can now use the parameter to have any Rush phased command either *only* restore from the cache (without any local building), or *only* write the cache, assuming all current output files are correct. -`rush build --to your-packageX --set-cache-only` +**Replay the cache entries for this command as best-effort, but don't execute any build processes** +`rush build --to your-packageX --bridge-cache-action=read` +That will populate the cache for `your-packageX` and all of its dependencies. +**Write whatever outputs are on disk for this command to the cache** +`rush build --to your-packageX --bridge-cache-action=write` That will populate the cache for `your-packageX` and all of its dependencies. diff --git a/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json index c4e1818152b..481c0fa550f 100644 --- a/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json +++ b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json @@ -3,7 +3,7 @@ "plugins": [ { "pluginName": "rush-bridge-cache-plugin", - "description": "Rush plugin that provides a --set-cache-only command flag to populate the cache from content on disk.", + "description": "Rush plugin that provides the ability to directly read or write the build cache from the command line via a custom choice parameter on build commands.", "entryPoint": "./lib/index.js", "optionsSchema": "lib/schemas/bridge-cache-config.schema.json" } diff --git a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts index fa93ecf46d0..55221a17ebb 100644 --- a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts +++ b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts @@ -4,7 +4,6 @@ import { Async } from '@rushstack/node-core-library'; import { _OperationBuildCache as OperationBuildCache } from '@rushstack/rush-sdk'; import type { - BuildCacheConfiguration, ICreateOperationsContext, IExecuteOperationsContext, ILogger, @@ -19,20 +18,25 @@ import type { CommandLineParameter } from '@rushstack/ts-command-line'; const PLUGIN_NAME: 'RushBridgeCachePlugin' = 'RushBridgeCachePlugin'; +const CACHE_ACTION_READ: 'read' = 'read'; +const CACHE_ACTION_WRITE: 'write' = 'write'; + +type CacheAction = typeof CACHE_ACTION_READ | typeof CACHE_ACTION_WRITE | undefined; + export interface IBridgeCachePluginOptions { - readonly flagName: string; + readonly actionParameterName: string; } export class BridgeCachePlugin implements IRushPlugin { public readonly pluginName: string = PLUGIN_NAME; - private readonly _flagName: string; + private readonly _actionParameterName: string; public constructor(options: IBridgeCachePluginOptions) { - this._flagName = options.flagName; + this._actionParameterName = options.actionParameterName; - if (!this._flagName) { + if (!this._actionParameterName) { throw new Error( - 'The "flagName" option must be provided for the BridgeCachePlugin. Please see the plugin README for details.' + 'The "actionParameterName" option must be provided for the BridgeCachePlugin. Please see the plugin README for details.' ); } } @@ -41,12 +45,21 @@ export class BridgeCachePlugin implements IRushPlugin { session.hooks.runAnyPhasedCommand.tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => { const logger: ILogger = session.getLogger(PLUGIN_NAME); + let cacheAction: CacheAction = undefined; + // cancel the actual operations. We don't want to run the command, just cache the output folders on disk command.hooks.createOperations.tap( { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER }, (operations: Set, context: ICreateOperationsContext): Set => { - const flagValue: boolean = this._getFlagValue(context); - if (flagValue) { + cacheAction = this._getCacheAction(context); + + if (cacheAction !== undefined) { + if (!context.buildCacheConfiguration?.buildCacheEnabled) { + throw new Error( + `The build cache must be enabled to use the "${this._actionParameterName}" parameter.` + ); + } + for (const operation of operations) { operation.enabled = false; } @@ -63,74 +76,105 @@ export class BridgeCachePlugin implements IRushPlugin { recordByOperation: Map, context: IExecuteOperationsContext ): Promise => { - if (!context.buildCacheConfiguration) { + const { buildCacheConfiguration } = context; + const { terminal } = logger; + if (!buildCacheConfiguration?.buildCacheEnabled) { + throw new Error( + `The build cache must be enabled to use the "${this._actionParameterName}" parameter.` + ); + } + + if (cacheAction === undefined) { return; } - const flagValue: boolean = this._getFlagValue(context); - if (flagValue) { - await this._setCacheAsync(logger, context.buildCacheConfiguration, recordByOperation); + const filteredOperations: Set = new Set(); + for (const operationExecutionResult of recordByOperation.values()) { + if (operationExecutionResult.operation.isNoOp) { + continue; + } + filteredOperations.add(operationExecutionResult); } + + let successCount: number = 0; + + await Async.forEachAsync( + filteredOperations, + async (operationExecutionResult: IOperationExecutionResult) => { + const projectBuildCache: OperationBuildCache = OperationBuildCache.forOperation( + operationExecutionResult, + { + buildCacheConfiguration, + terminal + } + ); + + const { operation } = operationExecutionResult; + + if (cacheAction === CACHE_ACTION_READ) { + const success: boolean = await projectBuildCache.tryRestoreFromCacheAsync(terminal); + if (success) { + ++successCount; + terminal.writeLine( + `Operation "${operation.name}": Outputs have been restored from the build cache."` + ); + } else { + terminal.writeWarningLine( + `Operation "${operation.name}": Outputs could not be restored from the build cache.` + ); + } + } else if (cacheAction === CACHE_ACTION_WRITE) { + const success: boolean = await projectBuildCache.trySetCacheEntryAsync(terminal); + if (success) { + ++successCount; + terminal.writeLine( + `Operation "${operation.name}": Existing outputs have been successfully written to the build cache."` + ); + } else { + terminal.writeErrorLine( + `Operation "${operation.name}": An error occurred while writing existing outputs to the build cache.` + ); + } + } + }, + { + concurrency: context.parallelism + } + ); + + terminal.writeLine( + `Cache operation "${cacheAction}" completed successfully for ${successCount} out of ${filteredOperations.size} operations.` + ); } ); }); } - private _getFlagValue(context: IExecuteOperationsContext): boolean { - const flagParam: CommandLineParameter | undefined = context.customParameters.get(this._flagName); - if (flagParam) { - if (flagParam.kind !== CommandLineParameterKind.Flag) { + private _getCacheAction(context: IExecuteOperationsContext): CacheAction { + const cacheActionParameter: CommandLineParameter | undefined = context.customParameters.get( + this._actionParameterName + ); + if (cacheActionParameter) { + if (cacheActionParameter.kind !== CommandLineParameterKind.Choice) { throw new Error( - `The parameter "${this._flagName}" must be a flag. Please check the plugin configuration.` + `The parameter "${this._actionParameterName}" must be a choice. Please check the plugin configuration.` ); } - return flagParam.value; + const value: string | undefined = cacheActionParameter.value; + switch (value) { + case CACHE_ACTION_READ: + case CACHE_ACTION_WRITE: + return value; + case undefined: + return undefined; + default: + throw new Error( + `The parameter "${this._actionParameterName}" must be one of: "${CACHE_ACTION_READ}" or "${CACHE_ACTION_WRITE}". Received: "${value}". Please check the plugin configuration.` + ); + } } - return false; - } - - private async _setCacheAsync( - { terminal }: ILogger, - buildCacheConfiguration: BuildCacheConfiguration, - recordByOperation: Map - ): Promise { - await Async.forEachAsync( - recordByOperation, - async ([ - { - associatedProject: { packageName }, - associatedPhase: { name: phaseName }, - isNoOp - }, - operationExecutionResult - ]) => { - if (isNoOp) { - return; - } - - const projectBuildCache: OperationBuildCache = OperationBuildCache.forOperation( - operationExecutionResult, - { - buildCacheConfiguration, - terminal - } - ); - - const success: boolean = await projectBuildCache.trySetCacheEntryAsync(terminal); - - if (success) { - terminal.writeLine( - `Cache entry set for ${phaseName} (${packageName}) from previously generated output folders` - ); - } else { - terminal.writeErrorLine( - `Error creating a cache entry set for ${phaseName} (${packageName}) from previously generated output folders` - ); - } - }, - { concurrency: 5 } - ); + return undefined; } } diff --git a/rush-plugins/rush-bridge-cache-plugin/src/schemas/bridge-cache-config.schema.json b/rush-plugins/rush-bridge-cache-plugin/src/schemas/bridge-cache-config.schema.json index ca46faa8f89..695adc8de13 100644 --- a/rush-plugins/rush-bridge-cache-plugin/src/schemas/bridge-cache-config.schema.json +++ b/rush-plugins/rush-bridge-cache-plugin/src/schemas/bridge-cache-config.schema.json @@ -5,11 +5,11 @@ "oneOf": [ { "type": "object", - "required": ["flagName"], + "required": ["actionParameterName"], "properties": { - "s3Endpoint": { + "actionParameterName": { "type": "string", - "description": "(Required) The name of the flag used to trigger this plugin on your phased commands." + "description": "(Required) The name of the choice parameter used to trigger this plugin on your phased commands. It should accept two values, 'read' and 'write'." } } } From e7eb432214e7cca2bfa29bac193b6741406e5c2d Mon Sep 17 00:00:00 2001 From: David Michon Date: Wed, 2 Jul 2025 00:47:15 +0000 Subject: [PATCH 2/2] Validate alternatives --- .../src/BridgeCachePlugin.ts | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts index 55221a17ebb..3b2ce0bf559 100644 --- a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts +++ b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts @@ -21,7 +21,7 @@ const PLUGIN_NAME: 'RushBridgeCachePlugin' = 'RushBridgeCachePlugin'; const CACHE_ACTION_READ: 'read' = 'read'; const CACHE_ACTION_WRITE: 'write' = 'write'; -type CacheAction = typeof CACHE_ACTION_READ | typeof CACHE_ACTION_WRITE | undefined; +type CacheAction = typeof CACHE_ACTION_READ | typeof CACHE_ACTION_WRITE; export interface IBridgeCachePluginOptions { readonly actionParameterName: string; @@ -45,7 +45,7 @@ export class BridgeCachePlugin implements IRushPlugin { session.hooks.runAnyPhasedCommand.tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => { const logger: ILogger = session.getLogger(PLUGIN_NAME); - let cacheAction: CacheAction = undefined; + let cacheAction: CacheAction | undefined; // cancel the actual operations. We don't want to run the command, just cache the output folders on disk command.hooks.createOperations.tap( @@ -90,10 +90,9 @@ export class BridgeCachePlugin implements IRushPlugin { const filteredOperations: Set = new Set(); for (const operationExecutionResult of recordByOperation.values()) { - if (operationExecutionResult.operation.isNoOp) { - continue; + if (!operationExecutionResult.operation.isNoOp) { + filteredOperations.add(operationExecutionResult); } - filteredOperations.add(operationExecutionResult); } let successCount: number = 0; @@ -150,7 +149,7 @@ export class BridgeCachePlugin implements IRushPlugin { }); } - private _getCacheAction(context: IExecuteOperationsContext): CacheAction { + private _getCacheAction(context: IExecuteOperationsContext): CacheAction | undefined { const cacheActionParameter: CommandLineParameter | undefined = context.customParameters.get( this._actionParameterName ); @@ -161,6 +160,16 @@ export class BridgeCachePlugin implements IRushPlugin { ); } + if ( + cacheActionParameter.alternatives.size !== 2 || + !cacheActionParameter.alternatives.has(CACHE_ACTION_READ) || + !cacheActionParameter.alternatives.has(CACHE_ACTION_WRITE) + ) { + throw new Error( + `The parameter "${this._actionParameterName}" must have exactly two choices: "${CACHE_ACTION_READ}" and "${CACHE_ACTION_WRITE}". Please check the plugin configuration.` + ); + } + const value: string | undefined = cacheActionParameter.value; switch (value) { case CACHE_ACTION_READ: