diff --git a/packages/@aws-cdk/cloud-assembly-api/lib/cloud-artifact.ts b/packages/@aws-cdk/cloud-assembly-api/lib/cloud-artifact.ts index 0f064adbe..e278d1c1f 100644 --- a/packages/@aws-cdk/cloud-assembly-api/lib/cloud-artifact.ts +++ b/packages/@aws-cdk/cloud-assembly-api/lib/cloud-artifact.ts @@ -1,3 +1,5 @@ +import * as fs from 'fs'; +import * as path from 'path'; import * as cxschema from '@aws-cdk/cloud-assembly-schema'; import type { CloudAssembly } from './cloud-assembly'; import type { MetadataEntryResult, SynthesisMessage } from './metadata'; @@ -36,6 +38,34 @@ export interface AwsCloudFormationStackProperties { * Represents an artifact within a cloud assembly. */ export class CloudArtifact { + /** + * Read the metadata for the given artifact + * + * HISTORICAL OR PRIVATE USE ONLY + * + * This is publicly exposed as a static function for downstream libraries that + * don't use the `CloudAssembly`/`CloudArtifact` API, yet still need to read + * an artifact's metadata. + * + * 99% of consumers should just access `artifact.metadata`. + */ + public static readMetadata(assemblyDirectory: string, x: cxschema.ArtifactManifest): Record { + const ret: Record = {}; + if (x.additionalMetadataFile) { + Object.assign(ret, JSON.parse(fs.readFileSync(path.join(assemblyDirectory, x.additionalMetadataFile), 'utf-8'))); + } + + for (const [p, entries] of Object.entries(x.metadata ?? {})) { + if (ret[p]) { + ret[p].push(...entries); + } else { + ret[p] = entries; + } + } + + return ret; + } + /** * Returns a subclass of `CloudArtifact` based on the artifact type defined in the artifact manifest. * @@ -77,6 +107,13 @@ export class CloudArtifact { this._dependencyIDs = manifest.dependencies || []; } + /** + * Returns the metadata associated with this Cloud Artifact + */ + public get metadata() { + return CloudArtifact.readMetadata(this.assembly.directory, this.manifest); + } + /** * Returns all the artifacts that this artifact depends on. */ @@ -100,11 +137,13 @@ export class CloudArtifact { * @returns all the metadata entries of a specific type in this artifact. */ public findMetadataByType(type: string): MetadataEntryResult[] { + const metadata = this.metadata; + const result = new Array(); - for (const path of Object.keys(this.manifest.metadata || {})) { - for (const entry of (this.manifest.metadata || {})[path]) { + for (const p of Object.keys(metadata || {})) { + for (const entry of (metadata || {})[p]) { if (entry.type === type) { - result.push({ path, ...entry }); + result.push({ path: p, ...entry }); } } } @@ -114,7 +153,7 @@ export class CloudArtifact { private renderMessages() { const messages = new Array(); - for (const [id, metadata] of Object.entries(this.manifest.metadata || { })) { + for (const [id, metadata] of Object.entries(this.metadata || { })) { for (const entry of metadata) { let level: SynthesisMessageLevel; switch (entry.type) { diff --git a/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts b/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts index c4911745b..f19379027 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts +++ b/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts @@ -86,10 +86,26 @@ export interface ArtifactManifest { /** * Associated metadata. * + * Metadata can be stored directly in the assembly manifest, as well as in a + * separate file (see `additionalMetadataFile`). It should prefer to be stored + * in the additional file, as that will reduce the size of the assembly + * manifest in cases of a lot of metdata (which CDK does emit by default). + * * @default - no metadata. */ readonly metadata?: { [path: string]: MetadataEntry[] }; + /** + * A file with additional metadata entries. + * + * The schema of this file is exactly the same as the type of the `metadata` field. + * In other words, that file contains an object mapping construct paths to arrays + * of metadata entries. + * + * @default - no additional metadata + */ + readonly additionalMetadataFile?: string; + /** * IDs of artifacts that must be deployed before this artifact. * diff --git a/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json b/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json index 36790023a..2e3e5fa40 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json +++ b/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json @@ -49,7 +49,7 @@ "type": "string" }, "metadata": { - "description": "Associated metadata. (Default - no metadata.)", + "description": "Associated metadata.\n\nMetadata can be stored directly in the assembly manifest, as well as in a\nseparate file (see `additionalMetadataFile`). It should prefer to be stored\nin the additional file, as that will reduce the size of the assembly\nmanifest in cases of a lot of metdata (which CDK does emit by default). (Default - no metadata.)", "type": "object", "additionalProperties": { "type": "array", @@ -58,6 +58,10 @@ } } }, + "additionalMetadataFile": { + "description": "A file with additional metadata entries.\n\nThe schema of this file is exactly the same as the type of the `metadata` field.\nIn other words, that file contains an object mapping construct paths to arrays\nof metadata entries. (Default - no additional metadata)", + "type": "string" + }, "dependencies": { "description": "IDs of artifacts that must be deployed before this artifact. (Default - no dependencies.)", "type": "array", diff --git a/packages/@aws-cdk/cloud-assembly-schema/schema/version.json b/packages/@aws-cdk/cloud-assembly-schema/schema/version.json index 3d43fd0c7..5b7c6a5df 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/schema/version.json +++ b/packages/@aws-cdk/cloud-assembly-schema/schema/version.json @@ -1,5 +1,5 @@ { - "schemaHash": "1b06659a117c44714e2e52854571bb1b45b765b277bb1c208bc4b7ea01f6a684", + "schemaHash": "22c511a4ddd185761b8d56ac21d48c8384873ffe4b953b3567654746f8dd26f1", "$comment": "Do not hold back the version on additions: jsonschema validation of the manifest by the consumer will trigger errors on unexpected fields.", - "revision": 50 + "revision": 52 } \ No newline at end of file diff --git a/packages/@aws-cdk/integ-runner/lib/runner/private/cloud-assembly.ts b/packages/@aws-cdk/integ-runner/lib/runner/private/cloud-assembly.ts index 146b2a3d5..7587d4e62 100644 --- a/packages/@aws-cdk/integ-runner/lib/runner/private/cloud-assembly.ts +++ b/packages/@aws-cdk/integ-runner/lib/runner/private/cloud-assembly.ts @@ -1,6 +1,7 @@ import * as path from 'path'; import type { FileManifestEntry, DockerImageManifestEntry } from '@aws-cdk/cdk-assets-lib'; import { AssetManifest } from '@aws-cdk/cdk-assets-lib'; +import { CloudArtifact } from '@aws-cdk/cloud-assembly-api'; import type { AssemblyManifest, AwsCloudFormationStackProperties, ArtifactManifest, MetadataEntry, AssetManifestProperties, ContainerImageAssetMetadataEntry, FileAssetMetadataEntry } from '@aws-cdk/cloud-assembly-schema'; import { Manifest, ArtifactType, ArtifactMetadataEntryType } from '@aws-cdk/cloud-assembly-schema'; import * as fs from 'fs-extra'; @@ -166,7 +167,7 @@ export class AssemblyManifestReader { */ private assetsFromAssemblyManifest(artifact: ArtifactManifest): (ContainerImageAssetMetadataEntry | FileAssetMetadataEntry)[] { const assets: (ContainerImageAssetMetadataEntry | FileAssetMetadataEntry)[] = []; - for (const metadata of Object.values(artifact.metadata ?? {})) { + for (const metadata of Object.values(CloudArtifact.readMetadata(this.directory, artifact) ?? {})) { metadata.forEach(data => { if (data.type === ArtifactMetadataEntryType.ASSET) { const asset = (data.data as ContainerImageAssetMetadataEntry | FileAssetMetadataEntry); diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/stack-collection.ts b/packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/stack-collection.ts index 1198f0cd3..fb6bdd04f 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/stack-collection.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/stack-collection.ts @@ -42,7 +42,11 @@ export class StackCollection { id: stack.displayName ?? stack.id, name: stack.stackName, environment: stack.environment, - metadata: stack.manifest.metadata, + + // Might be huge so load it lazily + get metadata() { + return stack.metadata; + }, dependencies: [], }; diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/refactoring/exclude.ts b/packages/@aws-cdk/toolkit-lib/lib/api/refactoring/exclude.ts deleted file mode 100644 index 3c9d23aa7..000000000 --- a/packages/@aws-cdk/toolkit-lib/lib/api/refactoring/exclude.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type { AssemblyManifest } from '@aws-cdk/cloud-assembly-schema'; -import { ArtifactMetadataEntryType, ArtifactType } from '@aws-cdk/cloud-assembly-schema'; -import type { ResourceLocation as CfnResourceLocation } from '@aws-sdk/client-cloudformation'; -import type { ResourceLocation } from './cloudformation'; - -export interface ExcludeList { - isExcluded(location: ResourceLocation): boolean; - - union(other: ExcludeList): ExcludeList; -} - -abstract class AbstractExcludeList implements ExcludeList { - abstract isExcluded(location: ResourceLocation): boolean; - - union(other: ExcludeList): ExcludeList { - return new UnionExcludeList([this, other]); - } -} - -export class ManifestExcludeList extends AbstractExcludeList { - private readonly excludedLocations: CfnResourceLocation[]; - - constructor(manifest: AssemblyManifest) { - super(); - this.excludedLocations = this.getExcludedLocations(manifest); - } - - private getExcludedLocations(asmManifest: AssemblyManifest): CfnResourceLocation[] { - // First, we need to filter the artifacts to only include CloudFormation stacks - const stackManifests = Object.entries(asmManifest.artifacts ?? {}).filter( - ([_, manifest]) => manifest.type === ArtifactType.AWS_CLOUDFORMATION_STACK, - ); - - const result: CfnResourceLocation[] = []; - for (let [stackName, manifest] of stackManifests) { - const locations = Object.values(manifest.metadata ?? {}) - // Then pick only the resources in each stack marked with DO_NOT_REFACTOR - .filter((entries) => - entries.some((entry) => entry.type === ArtifactMetadataEntryType.DO_NOT_REFACTOR && entry.data === true), - ) - // Finally, get the logical ID of each resource - .map((entries) => { - const logicalIdEntry = entries.find((entry) => entry.type === ArtifactMetadataEntryType.LOGICAL_ID); - const location: CfnResourceLocation = { - StackName: stackName, - LogicalResourceId: logicalIdEntry!.data! as string, - }; - return location; - }); - result.push(...locations); - } - return result; - } - - isExcluded(location: ResourceLocation): boolean { - return this.excludedLocations.some( - (loc) => loc.StackName === location.stack.stackName && loc.LogicalResourceId === location.logicalResourceId, - ); - } -} - -export class InMemoryExcludeList extends AbstractExcludeList { - private readonly excludedLocations: CfnResourceLocation[]; - private readonly excludedPaths: string[]; - - constructor(items: string[]) { - super(); - this.excludedLocations = []; - this.excludedPaths = []; - - if (items.length === 0) { - return; - } - - const locationRegex = /^[A-Za-z0-9]+\.[A-Za-z0-9]+$/; - - items.forEach((item: string) => { - if (locationRegex.test(item)) { - const [stackName, logicalId] = item.split('.'); - this.excludedLocations.push({ - StackName: stackName, - LogicalResourceId: logicalId, - }); - } else { - this.excludedPaths.push(item); - } - }); - } - - isExcluded(location: ResourceLocation): boolean { - const containsLocation = this.excludedLocations.some((loc) => { - return loc.StackName === location.stack.stackName && loc.LogicalResourceId === location.logicalResourceId; - }); - - const containsPath = this.excludedPaths.some((path) => location.toPath() === path); - return containsLocation || containsPath; - } -} - -export class UnionExcludeList extends AbstractExcludeList { - constructor(private readonly excludeLists: ExcludeList[]) { - super(); - } - - isExcluded(location: ResourceLocation): boolean { - return this.excludeLists.some((excludeList) => excludeList.isExcluded(location)); - } -} - -export class NeverExclude extends AbstractExcludeList { - isExcluded(_location: ResourceLocation): boolean { - return false; - } -} - -export class AlwaysExclude extends AbstractExcludeList { - isExcluded(_location: ResourceLocation): boolean { - return true; - } -} diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/refactoring/index.ts b/packages/@aws-cdk/toolkit-lib/lib/api/refactoring/index.ts index 3b01417a7..92bc904b7 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/refactoring/index.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/refactoring/index.ts @@ -17,7 +17,6 @@ import type { MappingGroup } from '../../actions'; import { ToolkitError } from '../../toolkit/toolkit-error'; import { pLimit } from '../../util/concurrency'; -export * from './exclude'; export * from './context'; interface StackGroup { diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/resource-metadata/resource-metadata.ts b/packages/@aws-cdk/toolkit-lib/lib/api/resource-metadata/resource-metadata.ts index a655d4ef5..4f9c85835 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/resource-metadata/resource-metadata.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/resource-metadata/resource-metadata.ts @@ -24,7 +24,7 @@ export interface ResourceMetadata { * @returns The resource metadata, or undefined if the resource was not found */ export function resourceMetadata(stack: CloudFormationStackArtifact, logicalId: string): ResourceMetadata | undefined { - const metadata = stack.manifest?.metadata; + const metadata = stack.metadata; if (!metadata) { return undefined; } diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts index 17bef32de..5756039f3 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts @@ -179,7 +179,7 @@ export class StackActivityMonitor { } private findMetadataFor(logicalId: string | undefined) { - const metadata = this.stack.manifest?.metadata; + const metadata = this.stack.metadata; if (!logicalId || !metadata) { return undefined; } diff --git a/packages/@aws-cdk/toolkit-lib/test/api/refactoring/exclude.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/refactoring/exclude.test.ts deleted file mode 100644 index b658ebc80..000000000 --- a/packages/@aws-cdk/toolkit-lib/test/api/refactoring/exclude.test.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { ArtifactMetadataEntryType, ArtifactType } from '@aws-cdk/cloud-assembly-schema'; -import { - AlwaysExclude, - InMemoryExcludeList, - ManifestExcludeList, - NeverExclude, - UnionExcludeList, -} from '../../../lib/api/refactoring'; -import type { CloudFormationStack } from '../../../lib/api/refactoring/cloudformation'; -import { ResourceLocation } from '../../../lib/api/refactoring/cloudformation'; - -const environment = { - name: 'prod', - account: '123456789012', - region: 'us-east-1', -}; - -const stack1: CloudFormationStack = { - stackName: 'Stack1', - environment, - template: {}, -}; -const stack2: CloudFormationStack = { - stackName: 'Stack2', - environment, - template: { - Resources: { - Resource3: { - Type: 'AWS::S3::Bucket', - Metadata: { - 'aws:cdk:path': 'Stack2/Resource3', - }, - }, - }, - }, -}; - -const resource1 = new ResourceLocation(stack1, 'Resource1'); -const resource2 = new ResourceLocation(stack2, 'Resource2'); -const resource3 = new ResourceLocation(stack2, 'Resource3'); - -describe('ManifestExcludeList', () => { - test('locations marked with DO_NOT_REFACTOR in the manifest are excluded', () => { - const manifest = { - artifacts: { - 'Stack1': { - type: ArtifactType.AWS_CLOUDFORMATION_STACK, - metadata: { - LogicalId1: [ - { type: ArtifactMetadataEntryType.DO_NOT_REFACTOR, data: true }, - { type: ArtifactMetadataEntryType.LOGICAL_ID, data: 'Resource1' }, - ], - }, - }, - 'Stack2': { - type: ArtifactType.AWS_CLOUDFORMATION_STACK, - metadata: { - LogicalId2: [ - { type: ArtifactMetadataEntryType.DO_NOT_REFACTOR, data: true }, - { type: ArtifactMetadataEntryType.LOGICAL_ID, data: 'Resource2' }, - ], - }, - }, - 'Stack1.assets': { - type: 'cdk:asset-manifest', - properties: { - file: 'Stack1.assets.json', - requiresBootstrapStackVersion: 6, - bootstrapStackVersionSsmParameter: '/cdk-bootstrap/hnb659fds/version', - }, - }, - }, - }; - - const excludeList = new ManifestExcludeList(manifest as any); - - expect(excludeList.isExcluded(resource1)).toBe(true); - expect(excludeList.isExcluded(resource2)).toBe(true); - expect(excludeList.isExcluded(resource3)).toBe(false); - }); - - test('nothing is excluded if no DO_NOT_REFACTOR entries exist', () => { - const manifest = { - artifacts: { - Stack1: { - type: ArtifactType.AWS_CLOUDFORMATION_STACK, - metadata: { - LogicalId1: [{ type: ArtifactMetadataEntryType.LOGICAL_ID, data: 'Resource1' }], - }, - }, - }, - }; - - const excludeList = new ManifestExcludeList(manifest as any); - expect(excludeList.isExcluded(resource1)).toBe(false); - }); -}); - -describe('InMemoryexcludeList', () => { - test('valid resources on a valid list are excluded', () => { - const excludeList = new InMemoryExcludeList(['Stack1.Resource1', 'Stack2/Resource3']); - expect(excludeList.isExcluded(resource1)).toBe(true); - expect(excludeList.isExcluded(resource2)).toBe(false); - expect(excludeList.isExcluded(resource3)).toBe(true); - }); - - test('nothing is excluded if no file path is provided', () => { - const excludeList = new InMemoryExcludeList([]); - expect(excludeList.isExcluded(resource1)).toBe(false); - expect(excludeList.isExcluded(resource2)).toBe(false); - expect(excludeList.isExcluded(resource3)).toBe(false); - }); -}); - -describe('UnionexcludeList', () => { - test('excludes a resource if at least one underlying list excludes', () => { - const excludeList1 = new AlwaysExclude(); - const excludeList2 = new NeverExclude(); - - const unionexcludeList = new UnionExcludeList([excludeList1, excludeList2]); - expect(unionexcludeList.isExcluded(resource1)).toBe(true); - }); - - test('does not exclude a resource if all underlying lists do not exclude', () => { - const excludeList1 = new NeverExclude(); - const excludeList2 = new NeverExclude(); - - const unionExcludeList = new UnionExcludeList([excludeList1, excludeList2]); - expect(unionExcludeList.isExcluded(resource1)).toBe(false); - }); -}); diff --git a/packages/aws-cdk/lib/cli/cdk-toolkit.ts b/packages/aws-cdk/lib/cli/cdk-toolkit.ts index 726e5f55b..fa34e2f55 100644 --- a/packages/aws-cdk/lib/cli/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cli/cdk-toolkit.ts @@ -195,7 +195,7 @@ export class CdkToolkit { public async metadata(stackName: string, json: boolean) { const stacks = await this.selectSingleStackByName(stackName); - await printSerializedObject(this.ioHost.asIoHelper(), stacks.firstStack.manifest.metadata ?? {}, json); + await printSerializedObject(this.ioHost.asIoHelper(), stacks.firstStack.metadata ?? {}, json); } public async acknowledge(noticeId: string) {