diff --git a/README.md b/README.md index bd0c6aaa51b..f8ddf37435b 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/rigs/heft-web-rig](./rigs/heft-web-rig/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig.svg)](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig) | [changelog](./rigs/heft-web-rig/CHANGELOG.md) | [@rushstack/heft-web-rig](https://www.npmjs.com/package/@rushstack/heft-web-rig) | | [/rush-plugins/rush-amazon-s3-build-cache-plugin](./rush-plugins/rush-amazon-s3-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin) | | [@rushstack/rush-amazon-s3-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-amazon-s3-build-cache-plugin) | | [/rush-plugins/rush-azure-storage-build-cache-plugin](./rush-plugins/rush-azure-storage-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin) | | [@rushstack/rush-azure-storage-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-azure-storage-build-cache-plugin) | +| [/rush-plugins/rush-bridge-cache-plugin](./rush-plugins/rush-bridge-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-bridge-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-bridge-cache-plugin) | | [@rushstack/rush-bridge-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-bridge-cache-plugin) | | [/rush-plugins/rush-buildxl-graph-plugin](./rush-plugins/rush-buildxl-graph-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-buildxl-graph-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-buildxl-graph-plugin) | | [@rushstack/rush-buildxl-graph-plugin](https://www.npmjs.com/package/@rushstack/rush-buildxl-graph-plugin) | | [/rush-plugins/rush-http-build-cache-plugin](./rush-plugins/rush-http-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin) | | [@rushstack/rush-http-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-http-build-cache-plugin) | | [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | diff --git a/apps/rush-mcp-server/bin/mcp-server b/apps/rush-mcp-server/bin/mcp-server old mode 100644 new mode 100755 diff --git a/apps/rush/src/start-dev-docs.ts b/apps/rush/src/start-dev-docs.ts index be5089d5130..4bd9539a8bb 100644 --- a/apps/rush/src/start-dev-docs.ts +++ b/apps/rush/src/start-dev-docs.ts @@ -6,4 +6,4 @@ import { Colorize, ConsoleTerminalProvider, Terminal } from '@rushstack/terminal const terminal: Terminal = new Terminal(new ConsoleTerminalProvider()); terminal.writeLine('For instructions on debugging Rush, please see this documentation:'); -terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/debugging/')); +terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/#debugging-rush')); diff --git a/common/changes/@microsoft/rush/benkeen-cache-bridge-plugin_2025-05-30-18-02.json b/common/changes/@microsoft/rush/benkeen-cache-bridge-plugin_2025-05-30-18-02.json new file mode 100644 index 00000000000..75d60bbf09d --- /dev/null +++ b/common/changes/@microsoft/rush/benkeen-cache-bridge-plugin_2025-05-30-18-02.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Introduce a `@rushstack/rush-bridge-cache-plugin` package that adds a `--set-cache-only` flag to phased commands, which sets the cache entry without performing the operation.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/changes/@rushstack/mcp-server/benkeen-cache-bridge-plugin_2025-06-10-02-59.json b/common/changes/@rushstack/mcp-server/benkeen-cache-bridge-plugin_2025-06-10-02-59.json new file mode 100644 index 00000000000..827fb92cfee --- /dev/null +++ b/common/changes/@rushstack/mcp-server/benkeen-cache-bridge-plugin_2025-06-10-02-59.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/mcp-server", + "comment": "", + "type": "none" + } + ], + "packageName": "@rushstack/mcp-server" +} \ No newline at end of file diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index c1149966a80..14eabbce0ae 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -270,6 +270,10 @@ "name": "@rushstack/rush-azure-storage-build-cache-plugin", "allowedCategories": [ "libraries" ] }, + { + "name": "@rushstack/rush-bridge-cache-plugin", + "allowedCategories": [ "libraries" ] + }, { "name": "@rushstack/rush-http-build-cache-plugin", "allowedCategories": [ "libraries" ] diff --git a/common/config/subspaces/default/pnpm-lock.yaml b/common/config/subspaces/default/pnpm-lock.yaml index 1ba0045bb90..2ef7494dd10 100644 --- a/common/config/subspaces/default/pnpm-lock.yaml +++ b/common/config/subspaces/default/pnpm-lock.yaml @@ -3297,12 +3297,12 @@ importers: '@pnpm/dependency-path': specifier: ~1000.0.9 version: 1000.0.9 - '@pnpm/dependency-path-lockfile-pre-v9': - specifier: npm:@pnpm/dependency-path@~2.1.2 - version: /@pnpm/dependency-path@2.1.8 '@pnpm/dependency-path-lockfile-pre-v10': specifier: npm:@pnpm/dependency-path@~5.1.7 version: /@pnpm/dependency-path@5.1.7 + '@pnpm/dependency-path-lockfile-pre-v9': + specifier: npm:@pnpm/dependency-path@~2.1.2 + version: /@pnpm/dependency-path@2.1.8 '@pnpm/link-bins': specifier: ~5.3.7 version: 5.3.25 @@ -4032,6 +4032,28 @@ importers: specifier: workspace:* version: link:../../rigs/local-node-rig + ../../../rush-plugins/rush-bridge-cache-plugin: + dependencies: + '@rushstack/node-core-library': + specifier: workspace:* + version: link:../../libraries/node-core-library + '@rushstack/rush-sdk': + specifier: workspace:* + version: link:../../libraries/rush-sdk + '@rushstack/terminal': + specifier: workspace:* + version: link:../../libraries/terminal + '@rushstack/ts-command-line': + specifier: workspace:* + version: link:../../libraries/ts-command-line + devDependencies: + '@rushstack/heft': + specifier: workspace:* + version: link:../../apps/heft + local-node-rig: + specifier: workspace:* + version: link:../../rigs/local-node-rig + ../../../rush-plugins/rush-buildxl-graph-plugin: dependencies: '@rushstack/node-core-library': diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 76e081722d1..1d12b07f21a 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -583,6 +583,12 @@ export class IndividualVersionPolicy extends VersionPolicy { export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase { } +// @internal (undocumented) +export interface _IOperationBuildCacheOptions { + buildCacheConfiguration: BuildCacheConfiguration; + terminal: ITerminal; +} + // @alpha export interface IOperationExecutionResult { readonly error: Error | undefined; @@ -779,6 +785,14 @@ export interface IPnpmPeerDependencyRules { export { IPrefixMatch } +// @internal (undocumented) +export type _IProjectBuildCacheOptions = _IOperationBuildCacheOptions & { + projectOutputFolderNames: ReadonlyArray; + project: RushConfigurationProject; + operationStateHash: string; + phaseName: string; +}; + // @beta export interface IRushCommand { readonly actionName: string; @@ -947,6 +961,20 @@ export class Operation { weight: number; } +// @internal (undocumented) +export class _OperationBuildCache { + // (undocumented) + get cacheId(): string | undefined; + // (undocumented) + static forOperation(executionResult: IOperationExecutionResult, options: _IOperationBuildCacheOptions): _OperationBuildCache; + // (undocumented) + static getOperationBuildCache(options: _IProjectBuildCacheOptions): _OperationBuildCache; + // (undocumented) + tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; + // (undocumented) + trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; +} + // @internal export class _OperationMetadataManager { constructor(options: _IOperationMetadataManagerOptions); diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index d7ef26a364e..685720e0629 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -197,3 +197,9 @@ export { type IRushCommandLineParameter, type IRushCommandLineAction } from './api/RushCommandLine'; + +export { OperationBuildCache as _OperationBuildCache } from './logic/buildCache/OperationBuildCache'; +export type { + IOperationBuildCacheOptions as _IOperationBuildCacheOptions, + IProjectBuildCacheOptions as _IProjectBuildCacheOptions +} from './logic/buildCache/OperationBuildCache'; diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts similarity index 90% rename from libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts rename to libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts index 861d403f029..d603c094547 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts @@ -13,8 +13,11 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider'; import { TarExecutable } from '../../utilities/TarExecutable'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; -import type { OperationExecutionRecord } from '../operations/OperationExecutionRecord'; +import type { IOperationExecutionResult } from '../operations/IOperationExecutionResult'; +/** + * @internal + */ export interface IOperationBuildCacheOptions { /** * The repo-wide configuration for the build cache. @@ -26,6 +29,9 @@ export interface IOperationBuildCacheOptions { terminal: ITerminal; } +/** + * @internal + */ export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & { /** * Value from rush-project.json @@ -50,7 +56,10 @@ interface IPathsToCache { outputFilePaths: string[]; } -export class ProjectBuildCache { +/** + * @internal + */ +export class OperationBuildCache { private static _tarUtilityPromise: Promise | undefined; private readonly _project: RushConfigurationProject; @@ -82,40 +91,40 @@ export class ProjectBuildCache { } private static _tryGetTarUtility(terminal: ITerminal): Promise { - if (!ProjectBuildCache._tarUtilityPromise) { - ProjectBuildCache._tarUtilityPromise = TarExecutable.tryInitializeAsync(terminal); + if (!OperationBuildCache._tarUtilityPromise) { + OperationBuildCache._tarUtilityPromise = TarExecutable.tryInitializeAsync(terminal); } - return ProjectBuildCache._tarUtilityPromise; + return OperationBuildCache._tarUtilityPromise; } public get cacheId(): string | undefined { return this._cacheId; } - public static getProjectBuildCache(options: IProjectBuildCacheOptions): ProjectBuildCache { - const cacheId: string | undefined = ProjectBuildCache._getCacheId(options); - return new ProjectBuildCache(cacheId, options); + public static getOperationBuildCache(options: IProjectBuildCacheOptions): OperationBuildCache { + const cacheId: string | undefined = OperationBuildCache._getCacheId(options); + return new OperationBuildCache(cacheId, options); } public static forOperation( - operation: OperationExecutionRecord, + executionResult: IOperationExecutionResult, options: IOperationBuildCacheOptions - ): ProjectBuildCache { - const outputFolders: string[] = [...(operation.operation.settings?.outputFolderNames ?? [])]; - if (operation.metadataFolderPath) { - outputFolders.push(operation.metadataFolderPath); + ): OperationBuildCache { + const outputFolders: string[] = [...(executionResult.operation.settings?.outputFolderNames ?? [])]; + if (executionResult.metadataFolderPath) { + outputFolders.push(executionResult.metadataFolderPath); } const buildCacheOptions: IProjectBuildCacheOptions = { buildCacheConfiguration: options.buildCacheConfiguration, terminal: options.terminal, - project: operation.associatedProject, - phaseName: operation.associatedPhase.name, + project: executionResult.operation.associatedProject, + phaseName: executionResult.operation.associatedPhase.name, projectOutputFolderNames: outputFolders, - operationStateHash: operation.getStateHash() + operationStateHash: executionResult.getStateHash() }; - const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions); - return new ProjectBuildCache(cacheId, buildCacheOptions); + const cacheId: string | undefined = OperationBuildCache._getCacheId(buildCacheOptions); + return new OperationBuildCache(cacheId, buildCacheOptions); } public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { @@ -175,7 +184,7 @@ export class ProjectBuildCache { ) ); - const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal); + const tarUtility: TarExecutable | undefined = await OperationBuildCache._tryGetTarUtility(terminal); let restoreSuccess: boolean = false; if (tarUtility && localCacheEntryPath) { const logFilePath: string = this._getTarLogFilePath(cacheId, 'untar'); @@ -225,7 +234,7 @@ export class ProjectBuildCache { let localCacheEntryPath: string | undefined; - const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal); + const tarUtility: TarExecutable | undefined = await OperationBuildCache._tryGetTarUtility(terminal); if (tarUtility) { const finalLocalCacheEntryPath: string = this._localBuildCacheProvider.getCacheEntryPath(cacheId); diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts similarity index 79% rename from libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts rename to libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts index 1acc025dee4..82606c11784 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts @@ -8,7 +8,7 @@ import type { RushConfigurationProject } from '../../../api/RushConfigurationPro import type { IGenerateCacheEntryIdOptions } from '../CacheEntryId'; import type { FileSystemBuildCacheProvider } from '../FileSystemBuildCacheProvider'; -import { ProjectBuildCache } from '../ProjectBuildCache'; +import { OperationBuildCache } from '../OperationBuildCache'; interface ITestOptions { enabled: boolean; @@ -16,11 +16,11 @@ interface ITestOptions { trackedProjectFiles: string[] | undefined; } -describe(ProjectBuildCache.name, () => { - function prepareSubject(options: Partial): ProjectBuildCache { +describe(OperationBuildCache.name, () => { + function prepareSubject(options: Partial): OperationBuildCache { const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - const subject: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({ + const subject: OperationBuildCache = OperationBuildCache.getOperationBuildCache({ buildCacheConfiguration: { buildCacheEnabled: options.hasOwnProperty('enabled') ? options.enabled : true, getCacheEntryId: (opts: IGenerateCacheEntryIdOptions) => @@ -46,9 +46,9 @@ describe(ProjectBuildCache.name, () => { return subject; } - describe(ProjectBuildCache.getProjectBuildCache.name, () => { - it('returns a ProjectBuildCache with a calculated cacheId value', () => { - const subject: ProjectBuildCache = prepareSubject({}); + describe(OperationBuildCache.getOperationBuildCache.name, () => { + it('returns an OperationBuildCache with a calculated cacheId value', () => { + const subject: OperationBuildCache = prepareSubject({}); expect(subject['_cacheId']).toMatchInlineSnapshot( `"acme-wizard/1926f30e8ed24cb47be89aea39e7efd70fcda075"` ); diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 44a99405aaf..9dbf7114c3d 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -6,7 +6,7 @@ import { InternalError } from '@rushstack/node-core-library'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import type { OperationStatus } from '../operations/OperationStatus'; import type { ICobuildContext } from './ICobuildLockProvider'; -import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; +import type { OperationBuildCache } from '../buildCache/OperationBuildCache'; const KEY_SEPARATOR: ':' = ':'; @@ -27,7 +27,7 @@ export interface ICobuildLockOptions { * {@inheritdoc ICobuildContext.phaseName} */ phaseName: string; - projectBuildCache: ProjectBuildCache; + operationBuildCache: OperationBuildCache; /** * The expire time of the lock in seconds. */ @@ -41,23 +41,23 @@ export interface ICobuildCompletedState { export class CobuildLock { public readonly cobuildConfiguration: CobuildConfiguration; - public readonly projectBuildCache: ProjectBuildCache; + public readonly operationBuildCache: OperationBuildCache; private _cobuildContext: ICobuildContext; public constructor(options: ICobuildLockOptions) { const { cobuildConfiguration, - projectBuildCache, + operationBuildCache, cobuildClusterId: clusterId, lockExpireTimeInSeconds, packageName, phaseName } = options; const { cobuildContextId: contextId, cobuildRunnerId: runnerId } = cobuildConfiguration; - const { cacheId } = projectBuildCache; + const { cacheId } = operationBuildCache; this.cobuildConfiguration = cobuildConfiguration; - this.projectBuildCache = projectBuildCache; + this.operationBuildCache = operationBuildCache; if (!cacheId) { // This should never happen diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts index b67089460db..bde478c4919 100644 --- a/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts +++ b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts @@ -4,7 +4,7 @@ import { CobuildLock, type ICobuildLockOptions } from '../CobuildLock'; import type { CobuildConfiguration } from '../../../api/CobuildConfiguration'; -import type { ProjectBuildCache } from '../../buildCache/ProjectBuildCache'; +import type { OperationBuildCache } from '../../buildCache/OperationBuildCache'; import type { ICobuildContext } from '../ICobuildLockProvider'; describe(CobuildLock.name, () => { @@ -14,9 +14,9 @@ describe(CobuildLock.name, () => { cobuildContextId: 'context_id', cobuildRunnerId: 'runner_id' } as unknown as CobuildConfiguration, - projectBuildCache: { + operationBuildCache: { cacheId: 'cache_id' - } as unknown as ProjectBuildCache, + } as unknown as OperationBuildCache, cobuildClusterId: 'cluster_id', lockExpireTimeInSeconds: 30, packageName: 'package_name', diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 3ccddb4bbeb..e47c454971f 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -10,7 +10,7 @@ import { SplitterTransform, type TerminalWritable, type ITerminal, Terminal } fr import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; import { OperationStatus } from './OperationStatus'; import { CobuildLock, type ICobuildCompletedState } from '../cobuild/CobuildLock'; -import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; +import { OperationBuildCache } from '../buildCache/OperationBuildCache'; import { RushConstants } from '../RushConstants'; import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { @@ -47,7 +47,7 @@ export interface IOperationBuildCacheContext { isCacheWriteAllowed: boolean; isCacheReadAllowed: boolean; - operationBuildCache: ProjectBuildCache | undefined; + operationBuildCache: OperationBuildCache | undefined; cacheDisabledReason: string | undefined; outputFolderNames: ReadonlyArray; @@ -240,7 +240,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const buildCacheTerminal: ITerminal = buildCacheContext.buildCacheTerminal; - let projectBuildCache: ProjectBuildCache | undefined = this._tryGetProjectBuildCache({ + let operationBuildCache: OperationBuildCache | undefined = this._tryGetOperationBuildCache({ buildCacheContext, buildCacheConfiguration, terminal: buildCacheTerminal, @@ -253,18 +253,18 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if ( cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && operation.consumers.size === 0 && - !projectBuildCache + !operationBuildCache ) { // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get // a log files only project build cache - projectBuildCache = await this._tryGetLogOnlyProjectBuildCacheAsync({ + operationBuildCache = await this._tryGetLogOnlyOperationBuildCacheAsync({ buildCacheConfiguration, cobuildConfiguration, buildCacheContext, record, terminal: buildCacheTerminal }); - if (projectBuildCache) { + if (operationBuildCache) { buildCacheTerminal.writeVerboseLine( `Log files only build cache is enabled for the project "${project.packageName}" because the cobuild leaf project log only is allowed` ); @@ -277,7 +277,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { cobuildLock = await this._tryGetCobuildLockAsync({ buildCacheContext, - projectBuildCache, + operationBuildCache, cobuildConfiguration, packageName: project.packageName, phaseName: phase.name @@ -305,14 +305,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { logFilenameIdentifier: operation.logFilenameIdentifier }); const restoreCacheAsync = async ( - // TODO: Investigate if `projectBuildCacheForRestore` is always the same instance as `projectBuildCache` + // TODO: Investigate if `operationBuildCacheForRestore` is always the same instance as `operationBuildCache` // above, and if it is, remove this parameter - projectBuildCacheForRestore: ProjectBuildCache | undefined, + operationBuildCacheForRestore: OperationBuildCache | undefined, specifiedCacheId?: string ): Promise => { buildCacheContext.isCacheReadAttempted = true; const restoreFromCacheSuccess: boolean | undefined = - await projectBuildCacheForRestore?.tryRestoreFromCacheAsync( + await operationBuildCacheForRestore?.tryRestoreFromCacheAsync( buildCacheTerminal, specifiedCacheId ); @@ -350,7 +350,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } const restoreFromCacheSuccess: boolean = await restoreCacheAsync( - cobuildLock.projectBuildCache, + cobuildLock.operationBuildCache, cacheId ); @@ -358,14 +358,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return status; } } else if (!buildCacheContext.isCacheReadAttempted && buildCacheContext.isCacheReadAllowed) { - const restoreFromCacheSuccess: boolean = await restoreCacheAsync(projectBuildCache); + const restoreFromCacheSuccess: boolean = await restoreCacheAsync(operationBuildCache); if (restoreFromCacheSuccess) { return OperationStatus.FromCache; } } } else if (buildCacheContext.isCacheReadAllowed) { - const restoreFromCacheSuccess: boolean = await restoreCacheAsync(projectBuildCache); + const restoreFromCacheSuccess: boolean = await restoreCacheAsync(operationBuildCache); if (restoreFromCacheSuccess) { return OperationStatus.FromCache; @@ -471,7 +471,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); }; setCacheEntryPromise = () => - cobuildLock.projectBuildCache.trySetCacheEntryAsync(buildCacheTerminal, finalCacheId); + cobuildLock.operationBuildCache.trySetCacheEntryAsync(buildCacheTerminal, finalCacheId); } } } @@ -553,7 +553,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext; } - private _tryGetProjectBuildCache({ + private _tryGetOperationBuildCache({ buildCacheConfiguration, buildCacheContext, terminal, @@ -563,7 +563,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheConfiguration: BuildCacheConfiguration | undefined; terminal: ITerminal; record: OperationExecutionRecord; - }): ProjectBuildCache | undefined { + }): OperationBuildCache | undefined { if (!buildCacheContext.operationBuildCache) { const { cacheDisabledReason } = buildCacheContext; if (cacheDisabledReason && !record.operation.settings?.allowCobuildWithoutCache) { @@ -577,7 +577,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(record, { + buildCacheContext.operationBuildCache = OperationBuildCache.forOperation(record, { buildCacheConfiguration, terminal }); @@ -586,14 +586,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext.operationBuildCache; } - // Get a ProjectBuildCache only cache/restore log files - private async _tryGetLogOnlyProjectBuildCacheAsync(options: { + // Get an OperationBuildCache only cache/restore log files + private async _tryGetLogOnlyOperationBuildCacheAsync(options: { buildCacheContext: IOperationBuildCacheContext; buildCacheConfiguration: BuildCacheConfiguration | undefined; cobuildConfiguration: CobuildConfiguration; record: IOperationRunnerContext & IOperationExecutionResult; terminal: ITerminal; - }): Promise { + }): Promise { const { buildCacheContext, buildCacheConfiguration, cobuildConfiguration, record, terminal } = options; if (!buildCacheConfiguration?.buildCacheEnabled) { @@ -617,7 +617,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const { associatedPhase, associatedProject } = record.operation; - const projectBuildCache: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({ + const operationBuildCache: OperationBuildCache = OperationBuildCache.getOperationBuildCache({ project: associatedProject, projectOutputFolderNames: outputFolderNames, buildCacheConfiguration, @@ -627,33 +627,33 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.operationBuildCache = projectBuildCache; + buildCacheContext.operationBuildCache = operationBuildCache; - return projectBuildCache; + return operationBuildCache; } private async _tryGetCobuildLockAsync({ cobuildConfiguration, buildCacheContext, - projectBuildCache, + operationBuildCache, packageName, phaseName }: { cobuildConfiguration: CobuildConfiguration | undefined; buildCacheContext: IOperationBuildCacheContext; - projectBuildCache: ProjectBuildCache | undefined; + operationBuildCache: OperationBuildCache | undefined; packageName: string; phaseName: string; }): Promise { if (!buildCacheContext.cobuildLock) { - if (projectBuildCache && cobuildConfiguration?.cobuildFeatureEnabled) { + if (operationBuildCache && cobuildConfiguration?.cobuildFeatureEnabled) { if (!buildCacheContext.cobuildClusterId) { // This should not happen throw new InternalError('Cobuild cluster id is not defined'); } buildCacheContext.cobuildLock = new CobuildLock({ cobuildConfiguration, - projectBuildCache, + operationBuildCache, cobuildClusterId: buildCacheContext.cobuildClusterId, lockExpireTimeInSeconds: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 3, packageName, diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index f0111b28a15..b10da76af58 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -34,6 +34,9 @@ import { initializeProjectLogFilesAsync } from './ProjectLogWritable'; +/** + * @internal + */ export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; onOperationStatusChanged?: (record: OperationExecutionRecord) => void; @@ -381,8 +384,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera this.status = this.operation.enabled ? await this.runner.executeAsync(this) : this.runner.isNoOp - ? OperationStatus.NoOp - : OperationStatus.Skipped; + ? OperationStatus.NoOp + : OperationStatus.Skipped; } // Make sure that the stopwatch is stopped before reporting the result, otherwise endTime is undefined. this.stopwatch.stop(); diff --git a/rush-plugins/rush-bridge-cache-plugin/.eslintrc.js b/rush-plugins/rush-bridge-cache-plugin/.eslintrc.js new file mode 100644 index 00000000000..0b04796d1ee --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/.eslintrc.js @@ -0,0 +1,13 @@ +// This is a workaround for https://github.com/eslint/eslint/issues/3458 +require('local-node-rig/profiles/default/includes/eslint/patch/modern-module-resolution'); +// This is a workaround for https://github.com/microsoft/rushstack/issues/3021 +require('local-node-rig/profiles/default/includes/eslint/patch/custom-config-package-names'); + +module.exports = { + extends: [ + 'local-node-rig/profiles/default/includes/eslint/profile/node', + 'local-node-rig/profiles/default/includes/eslint/mixins/friendly-locals', + 'local-node-rig/profiles/default/includes/eslint/mixins/tsdoc' + ], + parserOptions: { tsconfigRootDir: __dirname } +}; diff --git a/rush-plugins/rush-bridge-cache-plugin/LICENSE b/rush-plugins/rush-bridge-cache-plugin/LICENSE new file mode 100644 index 00000000000..7cfdf630251 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/LICENSE @@ -0,0 +1,24 @@ +@rushstack/rush-bridge-cache-plugin + +Copyright (c) Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/rush-plugins/rush-bridge-cache-plugin/README.md b/rush-plugins/rush-bridge-cache-plugin/README.md new file mode 100644 index 00000000000..e7e445de337 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/README.md @@ -0,0 +1,55 @@ +# @rushstack/rush-bridge-cache-plugin + +This is a Rush plugin that lets you to add an optional flag to Rush's phased commands to bypass the actual _action_ of the script (build, test, lint - whatever you have configured), and just populate the cache from the action as though the action had already been performed by Rush. The flag name is configurable. + +This is useful for integrations with other build orchestrators such as BuildXL. You can use those to do the work of actually running the task, then run the equivalent Rush command afterwards with a `--set-cache-only` to populate the Rush cache with whatever had been generated on disk, in addition to whatever cache mechanism is used by the other build orchestrator. + +## Here be dragons! + +This plugin assumes that the work for a particular task has already been completed and the build artifacts have been generated on disk. **If you run this command on a package where the command hasn't already been run and the build artifacts are missing or incorrect, you will cache invalid content**. Be careful and beware! + + +## Installation + +1. Add the `@rushstack/rush-bridge-cache-plugin` package to your autoinstaller's package.json. +2. Update your `command-line.json` file to add the new flag. Configure it to target whatever specific commands you want to have this feature. Example: + +```json +{ + "associatedCommands": ["build", "test", "lint", "a11y", "typecheck"], + "description": "When the flag is added to any associated command, it'll bypass running the command itself, and cache whatever it finds on disk for the action. Beware! Only run when you know the build artifacts are in a valid state for the command.", + "parameterKind": "flag", + "longName": "--set-cache-only" +} +``` + +3. Add a new entry in `common/config/rush/rush-plugins` to register the new plugin: +```json +{ + "packageName": "@rushstack/rush-bridge-cache-plugin", + "pluginName": "rush-bridge-cache-plugin", + "autoinstallerName": "your-auto-installer-name-here" +} +``` + +4. Create a configuration file for this plugin at this location: `common/config/rush-plugins/rush-bridge-cache-plugin.json` that defines the flag name you'll use to trigger the plugin: +```json +{ + "flagName": "--set-cache-only" +} +``` + +## Usage + +You can now add the flag to any Rush phased command, e.g. + +`rush build --to your-packageX --set-cache-only` + +That will populate the cache for `your-packageX` and all of its dependencies. + + +## Performance + +When running within a pipeline, you may want to populate the cache as quickly as possible so local Rush users will benefit from the cached entry sooner. So instead of waiting until the full build graph has been processed, running it after each individual task when it's been completed, e.g. + +`rush lint --only your-packageY --set-cache-only` diff --git a/rush-plugins/rush-bridge-cache-plugin/config/jest.config.json b/rush-plugins/rush-bridge-cache-plugin/config/jest.config.json new file mode 100644 index 00000000000..d1749681d90 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/config/jest.config.json @@ -0,0 +1,3 @@ +{ + "extends": "local-node-rig/profiles/default/config/jest.config.json" +} diff --git a/rush-plugins/rush-bridge-cache-plugin/config/rig.json b/rush-plugins/rush-bridge-cache-plugin/config/rig.json new file mode 100644 index 00000000000..d339847ee0a --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/config/rig.json @@ -0,0 +1,6 @@ +{ + // The "rig.json" file directs tools to look for their config files in an external package. + // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "local-node-rig" +} diff --git a/rush-plugins/rush-bridge-cache-plugin/package.json b/rush-plugins/rush-bridge-cache-plugin/package.json new file mode 100644 index 00000000000..ec25c13e8c3 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/package.json @@ -0,0 +1,27 @@ +{ + "name": "@rushstack/rush-bridge-cache-plugin", + "version": "0.0.1", + "private": true, + "description": "Rush plugin that provides a --set-cache-only command flag to populate the cache from content on disk.", + "license": "MIT", + "main": "./lib/index.js", + "repository": { + "url": "https://github.com/microsoft/rushstack.git", + "type": "git", + "directory": "rush-plugins/rush-bridge-cache-plugin" + }, + "scripts": { + "build": "heft test --clean", + "_phase:build": "heft run --only build -- --clean" + }, + "dependencies": { + "@rushstack/ts-command-line": "workspace:*", + "@rushstack/node-core-library": "workspace:*", + "@rushstack/rush-sdk": "workspace:*", + "@rushstack/terminal": "workspace:*" + }, + "devDependencies": { + "@rushstack/heft": "workspace:*", + "local-node-rig": "workspace:*" + } +} diff --git a/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json new file mode 100644 index 00000000000..c4e1818152b --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json", + "plugins": [ + { + "pluginName": "rush-bridge-cache-plugin", + "description": "Rush plugin that provides a --set-cache-only command flag to populate the cache from content on disk.", + "entryPoint": "./lib/index.js", + "optionsSchema": "lib/schemas/bridge-cache-config.schema.json" + } + ] +} diff --git a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts new file mode 100644 index 00000000000..fa93ecf46d0 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Async } from '@rushstack/node-core-library'; +import { _OperationBuildCache as OperationBuildCache } from '@rushstack/rush-sdk'; +import type { + BuildCacheConfiguration, + ICreateOperationsContext, + IExecuteOperationsContext, + ILogger, + IOperationExecutionResult, + IPhasedCommand, + IRushPlugin, + Operation, + RushSession +} from '@rushstack/rush-sdk'; +import { CommandLineParameterKind } from '@rushstack/ts-command-line'; +import type { CommandLineParameter } from '@rushstack/ts-command-line'; + +const PLUGIN_NAME: 'RushBridgeCachePlugin' = 'RushBridgeCachePlugin'; + +export interface IBridgeCachePluginOptions { + readonly flagName: string; +} + +export class BridgeCachePlugin implements IRushPlugin { + public readonly pluginName: string = PLUGIN_NAME; + private readonly _flagName: string; + + public constructor(options: IBridgeCachePluginOptions) { + this._flagName = options.flagName; + + if (!this._flagName) { + throw new Error( + 'The "flagName" option must be provided for the BridgeCachePlugin. Please see the plugin README for details.' + ); + } + } + + public apply(session: RushSession): void { + session.hooks.runAnyPhasedCommand.tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => { + const logger: ILogger = session.getLogger(PLUGIN_NAME); + + // cancel the actual operations. We don't want to run the command, just cache the output folders on disk + command.hooks.createOperations.tap( + { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER }, + (operations: Set, context: ICreateOperationsContext): Set => { + const flagValue: boolean = this._getFlagValue(context); + if (flagValue) { + for (const operation of operations) { + operation.enabled = false; + } + } + + return operations; + } + ); + + // populate the cache for each operation + command.hooks.beforeExecuteOperations.tap( + PLUGIN_NAME, + async ( + recordByOperation: Map, + context: IExecuteOperationsContext + ): Promise => { + if (!context.buildCacheConfiguration) { + return; + } + + const flagValue: boolean = this._getFlagValue(context); + if (flagValue) { + await this._setCacheAsync(logger, context.buildCacheConfiguration, recordByOperation); + } + } + ); + }); + } + + private _getFlagValue(context: IExecuteOperationsContext): boolean { + const flagParam: CommandLineParameter | undefined = context.customParameters.get(this._flagName); + if (flagParam) { + if (flagParam.kind !== CommandLineParameterKind.Flag) { + throw new Error( + `The parameter "${this._flagName}" must be a flag. Please check the plugin configuration.` + ); + } + + return flagParam.value; + } + + return false; + } + + private async _setCacheAsync( + { terminal }: ILogger, + buildCacheConfiguration: BuildCacheConfiguration, + recordByOperation: Map + ): Promise { + await Async.forEachAsync( + recordByOperation, + async ([ + { + associatedProject: { packageName }, + associatedPhase: { name: phaseName }, + isNoOp + }, + operationExecutionResult + ]) => { + if (isNoOp) { + return; + } + + const projectBuildCache: OperationBuildCache = OperationBuildCache.forOperation( + operationExecutionResult, + { + buildCacheConfiguration, + terminal + } + ); + + const success: boolean = await projectBuildCache.trySetCacheEntryAsync(terminal); + + if (success) { + terminal.writeLine( + `Cache entry set for ${phaseName} (${packageName}) from previously generated output folders` + ); + } else { + terminal.writeErrorLine( + `Error creating a cache entry set for ${phaseName} (${packageName}) from previously generated output folders` + ); + } + }, + { concurrency: 5 } + ); + } +} diff --git a/rush-plugins/rush-bridge-cache-plugin/src/index.ts b/rush-plugins/rush-bridge-cache-plugin/src/index.ts new file mode 100644 index 00000000000..01e83887250 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/src/index.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +export { BridgeCachePlugin as default } from './BridgeCachePlugin'; diff --git a/rush-plugins/rush-bridge-cache-plugin/src/schemas/bridge-cache-config.schema.json b/rush-plugins/rush-bridge-cache-plugin/src/schemas/bridge-cache-config.schema.json new file mode 100644 index 00000000000..ca46faa8f89 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/src/schemas/bridge-cache-config.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Configuration for bridge cache plugin", + "type": "object", + "oneOf": [ + { + "type": "object", + "required": ["flagName"], + "properties": { + "s3Endpoint": { + "type": "string", + "description": "(Required) The name of the flag used to trigger this plugin on your phased commands." + } + } + } + ] +} diff --git a/rush-plugins/rush-bridge-cache-plugin/tsconfig.json b/rush-plugins/rush-bridge-cache-plugin/tsconfig.json new file mode 100644 index 00000000000..dac21d04081 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./node_modules/local-node-rig/profiles/default/tsconfig-base.json" +} diff --git a/rush.json b/rush.json index f1ad0ab70b4..0498497f004 100644 --- a/rush.json +++ b/rush.json @@ -1328,6 +1328,12 @@ "reviewCategory": "libraries", "versionPolicyName": "rush" }, + { + "packageName": "@rushstack/rush-bridge-cache-plugin", + "projectFolder": "rush-plugins/rush-bridge-cache-plugin", + "reviewCategory": "libraries", + "versionPolicyName": "rush" + }, // "vscode-extensions" folder (alphabetical order) {