feat(cli): metadata can live in a separate file

packages/@aws-cdk/cloud-assembly-api/lib/cloud-artifact.ts

@@ -1,3 +1,5 @@
+import * as fs from 'fs';
+import * as path from 'path';
 import * as cxschema from '@aws-cdk/cloud-assembly-schema';
 import type { CloudAssembly } from './cloud-assembly';
 import type { MetadataEntryResult, SynthesisMessage } from './metadata';
@@ -36,6 +38,34 @@ export interface AwsCloudFormationStackProperties {
  * Represents an artifact within a cloud assembly.
  */
 export class CloudArtifact {
+  /**
+   * Read the metadata for the given artifact
+   *
+   * HISTORICAL OR PRIVATE USE ONLY
+   *
+   * This is publicly exposed as a static function for downstream libraries that
+   * don't use the `CloudAssembly`/`CloudArtifact` API, yet still need to read
+   * an artifact's metadata.
+   *
+   * 99% of consumers should just access `artifact.metadata`.
+   */
+  public static readMetadata(assemblyDirectory: string, x: cxschema.ArtifactManifest): Record<string, cxschema.MetadataEntry[]> {
+    const ret: Record<string, cxschema.MetadataEntry[]> = {};
+    if (x.additionalMetadataFile) {
+      Object.assign(ret, JSON.parse(fs.readFileSync(path.join(assemblyDirectory, x.additionalMetadataFile), 'utf-8')));
+    }
+
+    for (const [p, entries] of Object.entries(x.metadata ?? {})) {
+      if (ret[p]) {
+        ret[p].push(...entries);
+      } else {
+        ret[p] = entries;
+      }
+    }
+
+    return ret;
+  }
+
   /**
    * Returns a subclass of `CloudArtifact` based on the artifact type defined in the artifact manifest.
    *
@@ -77,6 +107,13 @@ export class CloudArtifact {
     this._dependencyIDs = manifest.dependencies || [];
   }
 
+  /**
+   * Returns the metadata associated with this Cloud Artifact
+   */
+  public get metadata() {
+    return CloudArtifact.readMetadata(this.assembly.directory, this.manifest);
+  }
+
   /**
    * Returns all the artifacts that this artifact depends on.
    */
@@ -100,11 +137,13 @@ export class CloudArtifact {
    * @returns all the metadata entries of a specific type in this artifact.
    */
   public findMetadataByType(type: string): MetadataEntryResult[] {
+    const metadata = this.metadata;
+
     const result = new Array<MetadataEntryResult>();
-    for (const path of Object.keys(this.manifest.metadata || {})) {
-      for (const entry of (this.manifest.metadata || {})[path]) {
+    for (const p of Object.keys(metadata || {})) {
+      for (const entry of (metadata || {})[p]) {
         if (entry.type === type) {
-          result.push({ path, ...entry });
+          result.push({ path: p, ...entry });
         }
       }
     }
@@ -114,7 +153,7 @@ export class CloudArtifact {
   private renderMessages() {
     const messages = new Array<SynthesisMessage>();
 
-    for (const [id, metadata] of Object.entries(this.manifest.metadata || { })) {
+    for (const [id, metadata] of Object.entries(this.metadata || { })) {
       for (const entry of metadata) {
         let level: SynthesisMessageLevel;
         switch (entry.type) {

packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts

@@ -86,10 +86,26 @@ export interface ArtifactManifest {
   /**
    * Associated metadata.
    *
+   * Metadata can be stored directly in the assembly manifest, as well as in a
+   * separate file (see `additionalMetadataFile`). It should prefer to be stored
+   * in the additional file, as that will reduce the size of the assembly
+   * manifest in cases of a lot of metdata (which CDK does emit by default).
+   *
    * @default - no metadata.
    */
   readonly metadata?: { [path: string]: MetadataEntry[] };
 
+  /**
+   * A file with additional metadata entries.
+   *
+   * The schema of this file is exactly the same as the type of the `metadata` field.
+   * In other words, that file contains an object mapping construct paths to arrays
+   * of metadata entries.
+   *
+   * @default - no additional metadata
+   */
+  readonly additionalMetadataFile?: string;
+
   /**
    * IDs of artifacts that must be deployed before this artifact.
    *

packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json

@@ -49,7 +49,7 @@
                     "type": "string"
                 },
                 "metadata": {
-                    "description": "Associated metadata. (Default - no metadata.)",
+                    "description": "Associated metadata.\n\nMetadata can be stored directly in the assembly manifest, as well as in a\nseparate file (see `additionalMetadataFile`). It should prefer to be stored\nin the additional file, as that will reduce the size of the assembly\nmanifest in cases of a lot of metdata (which CDK does emit by default). (Default - no metadata.)",
                     "type": "object",
                     "additionalProperties": {
                         "type": "array",
@@ -58,6 +58,10 @@
                         }
                     }
                 },
+                "additionalMetadataFile": {
+                    "description": "A file with additional metadata entries.\n\nThe schema of this file is exactly the same as the type of the `metadata` field.\nIn other words, that file contains an object mapping construct paths to arrays\nof metadata entries. (Default - no additional metadata)",
+                    "type": "string"
+                },
                 "dependencies": {
                     "description": "IDs of artifacts that must be deployed before this artifact. (Default - no dependencies.)",
                     "type": "array",

packages/@aws-cdk/cloud-assembly-schema/schema/version.json

@@ -1,5 +1,5 @@
 {
-  "schemaHash": "1b06659a117c44714e2e52854571bb1b45b765b277bb1c208bc4b7ea01f6a684",
+  "schemaHash": "22c511a4ddd185761b8d56ac21d48c8384873ffe4b953b3567654746f8dd26f1",
   "$comment": "Do not hold back the version on additions: jsonschema validation of the manifest by the consumer will trigger errors on unexpected fields.",
-  "revision": 50
+  "revision": 52
 }

packages/@aws-cdk/integ-runner/lib/runner/private/cloud-assembly.ts

@@ -1,6 +1,7 @@
 import * as path from 'path';
 import type { FileManifestEntry, DockerImageManifestEntry } from '@aws-cdk/cdk-assets-lib';
 import { AssetManifest } from '@aws-cdk/cdk-assets-lib';
+import { CloudArtifact } from '@aws-cdk/cloud-assembly-api';
 import type { AssemblyManifest, AwsCloudFormationStackProperties, ArtifactManifest, MetadataEntry, AssetManifestProperties, ContainerImageAssetMetadataEntry, FileAssetMetadataEntry } from '@aws-cdk/cloud-assembly-schema';
 import { Manifest, ArtifactType, ArtifactMetadataEntryType } from '@aws-cdk/cloud-assembly-schema';
 import * as fs from 'fs-extra';
@@ -166,7 +167,7 @@ export class AssemblyManifestReader {
    */
   private assetsFromAssemblyManifest(artifact: ArtifactManifest): (ContainerImageAssetMetadataEntry | FileAssetMetadataEntry)[] {
     const assets: (ContainerImageAssetMetadataEntry | FileAssetMetadataEntry)[] = [];
-    for (const metadata of Object.values(artifact.metadata ?? {})) {
+    for (const metadata of Object.values(CloudArtifact.readMetadata(this.directory, artifact) ?? {})) {
       metadata.forEach(data => {
         if (data.type === ArtifactMetadataEntryType.ASSET) {
           const asset = (data.data as ContainerImageAssetMetadataEntry | FileAssetMetadataEntry);

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/stack-collection.ts

@@ -42,7 +42,11 @@ export class StackCollection {
         id: stack.displayName ?? stack.id,
         name: stack.stackName,
         environment: stack.environment,
-        metadata: stack.manifest.metadata,
+
+        // Might be huge so load it lazily
+        get metadata() {
+          return stack.metadata;
+        },
         dependencies: [],
       };
 

packages/@aws-cdk/toolkit-lib/lib/api/refactoring/index.ts

@@ -17,7 +17,6 @@ import type { MappingGroup } from '../../actions';
 import { ToolkitError } from '../../toolkit/toolkit-error';
 import { pLimit } from '../../util/concurrency';
 
-export * from './exclude';
 export * from './context';
 
 interface StackGroup {

packages/@aws-cdk/toolkit-lib/lib/api/resource-metadata/resource-metadata.ts

@@ -24,7 +24,7 @@ export interface ResourceMetadata {
  * @returns The resource metadata, or undefined if the resource was not found
  */
 export function resourceMetadata(stack: CloudFormationStackArtifact, logicalId: string): ResourceMetadata | undefined {
-  const metadata = stack.manifest?.metadata;
+  const metadata = stack.metadata;
   if (!metadata) {
     return undefined;
   }

packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts

@@ -179,7 +179,7 @@ export class StackActivityMonitor {
   }
 
   private findMetadataFor(logicalId: string | undefined) {
-    const metadata = this.stack.manifest?.metadata;
+    const metadata = this.stack.metadata;
     if (!logicalId || !metadata) {
       return undefined;
     }