From 15cf71186563e0fce410089a6e351e9682819db5 Mon Sep 17 00:00:00 2001 From: Danny Steenman Date: Tue, 31 Mar 2026 21:31:50 +0200 Subject: [PATCH 1/5] fix: streamline discovery targeting and live catalog loading --- .changeset/nice-trains-prove.md | 5 + .changeset/small-bottles-laugh.md | 5 + .changeset/tame-lions-count.md | 5 + README.md | 7 +- docs/TESTING.md | 2 +- docs/architecture/cli.md | 20 +- docs/architecture/rules.md | 2 +- docs/architecture/sdk.md | 5 +- docs/reference/config-schema.md | 17 +- docs/reference/rule-ids.md | 4 +- packages/cloudburn/README.md | 8 +- packages/cloudburn/src/cli.ts | 1 + packages/cloudburn/src/commands/discover.ts | 83 ++- packages/cloudburn/src/commands/scan.ts | 4 +- packages/cloudburn/src/debug.ts | 29 + packages/cloudburn/test/cli.test.ts | 1 + packages/cloudburn/test/completion.test.ts | 6 +- packages/cloudburn/test/discover.e2e.test.ts | 125 ++-- packages/cloudburn/test/help.e2e.test.ts | 3 + .../src/aws/cloudwatch/unused-log-streams.ts | 43 +- packages/rules/src/index.ts | 1 + packages/rules/src/shared/metadata.ts | 13 + .../cloudwatch-unused-log-streams.test.ts | 79 ++- packages/rules/test/exports.test.ts | 2 + packages/rules/test/rule-metadata.test.ts | 8 +- packages/sdk/README.md | 10 +- packages/sdk/src/debug.ts | 10 + packages/sdk/src/engine/run-live.ts | 13 +- packages/sdk/src/index.ts | 1 + packages/sdk/src/providers/aws/client.ts | 52 +- .../src/providers/aws/discovery-registry.ts | 22 +- packages/sdk/src/providers/aws/discovery.ts | 259 +++++++-- packages/sdk/src/providers/aws/index.ts | 1 - .../src/providers/aws/resource-explorer.ts | 209 ++++++- .../src/providers/aws/resources/cloudfront.ts | 6 +- .../aws/resources/cloudwatch-logs.ts | 82 ++- .../src/providers/aws/resources/dynamodb.ts | 4 +- .../aws/resources/ec2-utilization.ts | 4 +- .../aws/resources/ecs-cluster-metrics.ts | 4 +- .../providers/aws/resources/elasticache.ts | 6 +- .../sdk/src/providers/aws/resources/emr.ts | 4 +- .../sdk/src/providers/aws/resources/lambda.ts | 6 +- .../providers/aws/resources/rds-activity.ts | 7 +- .../src/providers/aws/resources/redshift.ts | 6 +- .../sdk/src/providers/aws/resources/utils.ts | 5 +- packages/sdk/src/scanner.ts | 38 +- packages/sdk/src/types.ts | 7 +- .../aws-cloudwatch-logs-resource.test.ts | 93 +++ .../sdk/test/providers/aws-discovery.test.ts | 546 ++++++++++++++---- .../aws-ec2-utilization-resource.test.ts | 35 ++ .../aws-ecs-cluster-metrics-resource.test.ts | 27 + .../providers/aws-lambda-resource.test.ts | 36 ++ .../aws-rds-activity-resource.test.ts | 53 ++ .../providers/aws-resource-explorer.test.ts | 181 ++++-- packages/sdk/test/scanner.test.ts | 43 +- 55 files changed, 1738 insertions(+), 510 deletions(-) create mode 100644 .changeset/nice-trains-prove.md create mode 100644 .changeset/small-bottles-laugh.md create mode 100644 .changeset/tame-lions-count.md create mode 100644 packages/cloudburn/src/debug.ts create mode 100644 packages/sdk/src/debug.ts diff --git a/.changeset/nice-trains-prove.md b/.changeset/nice-trains-prove.md new file mode 100644 index 0000000..a9a6f0b --- /dev/null +++ b/.changeset/nice-trains-prove.md @@ -0,0 +1,5 @@ +--- +"cloudburn": patch +--- + +Restore `cloudburn discover --region` as a single-region CLI flag while keeping SDK-backed debug output streamed from the SDK and provider layers. diff --git a/.changeset/small-bottles-laugh.md b/.changeset/small-bottles-laugh.md new file mode 100644 index 0000000..d38c505 --- /dev/null +++ b/.changeset/small-bottles-laugh.md @@ -0,0 +1,5 @@ +--- +"@cloudburn/rules": patch +--- + +Redesign `CLDBRN-AWS-CLOUDWATCH-2` to flag inactive CloudWatch log groups from latest stream activity summaries instead of enumerating every log stream. diff --git a/.changeset/tame-lions-count.md b/.changeset/tame-lions-count.md new file mode 100644 index 0000000..15c2061 --- /dev/null +++ b/.changeset/tame-lions-count.md @@ -0,0 +1,5 @@ +--- +"@cloudburn/sdk": patch +--- + +Reduce live discovery fan-out with batched Resource Explorer queries, add throttling-aware retries and debug tracing, and add log-group-level CloudWatch activity hydration to avoid full log-stream enumeration for stale log group checks. diff --git a/README.md b/README.md index 38cd976..ea550e0 100644 --- a/README.md +++ b/README.md @@ -118,17 +118,18 @@ cloudburn --format json scan ./iac ### Discover -`discover` runs the same rules against live AWS resources. Initialize AWS Resource Explorer first, then run against one region or all of them. +`discover` runs the same rules against live AWS resources. Initialize AWS Resource Explorer first, then run against the current AWS region or one explicit region. ```bash cloudburn discover init cloudburn discover cloudburn discover --region eu-central-1 -cloudburn discover --region all cloudburn discover --service ec2,s3 +cloudburn --debug discover --region eu-central-1 ``` -`--region all` requires an AWS Resource Explorer aggregator index. +The CLI targets one region at a time. Multi-region discovery remains available through the SDK. +Use `--debug` to relay SDK and provider execution trace details to `stderr` while keeping normal command output on `stdout`. Generate a starter config with `cloudburn config --init`. Full details in the [config reference](docs/reference/config-schema.md). diff --git a/docs/TESTING.md b/docs/TESTING.md index 6b7f397..5cc67a9 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -68,7 +68,7 @@ Mock at the SDK boundary — do not run real scans. - `--config`, `--enabled-rules`, and `--disabled-rules` pass the expected runtime overrides to the SDK - `--exit-code` sets `process.exitCode = 1` when findings exist - `--exit-code` without findings sets `process.exitCode = 0` -- `discover list-enabled-regions`, `discover supported-resource-types`, `discover init`, `config`, `rules list`, and `estimate` all go through the shared formatter system +- `discover supported-resource-types`, `discover init`, `discover status`, `config`, `rules list`, and `estimate` all go through the shared formatter system - `table` output stays human-readable and `json` output stays machine-readable - Runtime errors remain structured JSON on `stderr` regardless of stdout format diff --git a/docs/architecture/cli.md b/docs/architecture/cli.md index 361f78c..4141aba 100644 --- a/docs/architecture/cli.md +++ b/docs/architecture/cli.md @@ -11,16 +11,15 @@ graph TD Root --> Estimate["estimate"] Root --> Completion["completion"] Rules --> RulesList["list"] - Discover --> DiscoverRegions["list-enabled-regions"] Discover --> DiscoverInit["init"] Discover --> DiscoverTypes["supported-resource-types"] Completion --> CompletionBash["bash"] Completion --> CompletionFish["fish"] Completion --> CompletionZsh["zsh"] - Root -.- RootFlags["--format json|table"] + Root -.- RootFlags["--debug\n--format json|table"] Scan -.- ScanFlags["--config path\n--enabled-rules ids\n--disabled-rules ids\n--exit-code"] - Discover -.- DiscoverFlags["--region \n--config path\n--enabled-rules ids\n--disabled-rules ids\n--exit-code"] + Discover -.- DiscoverFlags["--region region\n--config path\n--enabled-rules ids\n--disabled-rules ids\n--exit-code"] Estimate -.- EstimateFlags["--server url"] ``` @@ -38,9 +37,9 @@ graph LR All stdout-producing commands return a typed `CliResponse` and share the same format resolver. -| Format | Output | -| ------- | ---------------------------------------------------------------------------------------------------------------------------------- | -| `json` | Pretty JSON for the underlying response payload | +| Format | Output | +| ------- | --------------------------------------------------------------------------------------------- | +| `json` | Pretty JSON for the underlying response payload | | `table` | ASCII tables for scans, record lists, string lists, key/value status output, and `rules list` | ## Command Behavior @@ -50,9 +49,9 @@ All stdout-producing commands return a typed `CliResponse` and share the same fo - `discover` runs live AWS discovery and rule evaluation through `CloudBurnClient.discover({ target, config?, configPath? })`. - `discover` accepts `--config`, `--enabled-rules`, `--disabled-rules`, and `--service` for one-off overrides of discovery config. - `discover --region ` overrides the current AWS region resolved from `AWS_REGION`, `AWS_DEFAULT_REGION`, `aws_region`, then the AWS SDK region provider chain. -- `discover --region all` requires a Resource Explorer aggregator index. -- `discover --region ` targets one enabled Resource Explorer index region. -- `discover list-enabled-regions` and `discover supported-resource-types` use the shared `json|table` renderer. +- The CLI targets one explicit AWS region per discover run. +- Multi-region discovery remains an SDK capability through `target: { mode: 'regions', regions: [...] }` and requires a Resource Explorer aggregator index. +- `discover supported-resource-types` uses the shared `json|table` renderer. - `discover init` bootstraps Resource Explorer through the SDK, defaults to the current AWS region, accepts `--region ` as an override, and falls back to local-only setup when cross-region bootstrap is denied. - `discover init` status output includes the resolved setup `indexType` so users can distinguish local-only setup from aggregator setup. - `config --init` creates `.cloudburn.yml` in the git root (or current directory when no git root exists), unless a config file already exists there. @@ -60,6 +59,7 @@ All stdout-producing commands return a typed `CliResponse` and share the same fo - `config --print-template` prints the starter template without writing a file. - `rules list`, `config`, and `estimate` all use the shared formatter system instead of ad hoc string output. - `completion` is a structural parent command. `completion bash|fish|zsh` prints shell completion scripts for the selected shell. +- `--debug` is a global flag that relays SDK and provider execution tracing to `stderr` without changing normal command output on `stdout`. - `--format` is documented as a global option and defaults to `table`, except `config --print` and `config --print-template`, which preserve raw YAML by default for redirection workflows. - `scan` and `discover` can also source their default format from `.cloudburn.yml`; explicit `--format` still wins. - The hidden `__complete` command exists only as the runtime hook for generated shell scripts. @@ -78,10 +78,8 @@ cloudburn scan ./iac --enabled-rules CLDBRN-AWS-EBS-1,CLDBRN-AWS-EC2-1 cloudburn scan ./iac --service ec2,s3 cloudburn discover cloudburn discover --region eu-central-1 -cloudburn discover --region all cloudburn discover --config .cloudburn.yml --disabled-rules CLDBRN-AWS-S3-1 cloudburn discover --service ec2,s3 -cloudburn discover list-enabled-regions cloudburn discover init cloudburn config --init cloudburn config --print diff --git a/docs/architecture/rules.md b/docs/architecture/rules.md index a6c9003..d758905 100644 --- a/docs/architecture/rules.md +++ b/docs/architecture/rules.md @@ -96,7 +96,7 @@ Rule evaluators consume static and live datasets through `context.resources.get( | `CLDBRN-AWS-CLOUDTRAIL-1` | CloudTrail Redundant Global Trails | cloudtrail | discovery | Implemented | | `CLDBRN-AWS-CLOUDTRAIL-2` | CloudTrail Redundant Regional Trails | cloudtrail | discovery | Implemented | | `CLDBRN-AWS-CLOUDWATCH-1` | CloudWatch Log Group Missing Retention | cloudwatch | discovery | Implemented | -| `CLDBRN-AWS-CLOUDWATCH-2` | CloudWatch Unused Log Streams | cloudwatch | discovery | Implemented | +| `CLDBRN-AWS-CLOUDWATCH-2` | CloudWatch Log Group Inactive | cloudwatch | discovery | Implemented | | `CLDBRN-AWS-EC2-1` | EC2 Instance Type Not Preferred | ec2 | iac, discovery | Implemented | | `CLDBRN-AWS-EC2-2` | S3 Interface VPC Endpoint Used | ec2 | iac | Implemented | | `CLDBRN-AWS-EC2-3` | Elastic IP Address Unassociated | ec2 | discovery | Implemented | diff --git a/docs/architecture/sdk.md b/docs/architecture/sdk.md index d88acd2..4863404 100644 --- a/docs/architecture/sdk.md +++ b/docs/architecture/sdk.md @@ -7,7 +7,6 @@ class CloudBurnClient { +scanStatic(path: string, config?: Partial~CloudBurnConfig~, options?: { configPath?: string }) Promise~ScanResult~ +discover(options?: { target?: AwsDiscoveryTarget, config?: Partial~CloudBurnConfig~, configPath?: string }) Promise~ScanResult~ - +listEnabledDiscoveryRegions() Promise~AwsDiscoveryRegion[]~ +initializeDiscovery(options?: { region?: string }) Promise~AwsDiscoveryInitialization~ +listSupportedDiscoveryResourceTypes() Promise~AwsSupportedResourceType[]~ +loadConfig(path?: string) Promise~CloudBurnConfig~ @@ -72,11 +71,13 @@ Current live-discovery behavior: - `discover` is the live entrypoint for both the CLI and direct SDK callers. - `discoverAwsResources` in `src/providers/aws/discovery.ts` is the AWS live orchestration entrypoint. - Default discovery target is the current region, resolved from `AWS_REGION`, then `AWS_DEFAULT_REGION`, then `aws_region`, then the AWS SDK region provider chain. +- Explicit discovery uses `target: { mode: 'regions', regions: [...] }`. - Explicit single-region discovery uses the selected region as the Resource Explorer control plane instead of the ambient current region. -- `--region all` requires an aggregator index and fails fast when one is not enabled. +- Explicit multi-region discovery requires an aggregator index and fails fast when one is not enabled. - Discovery resolves the explicit default Resource Explorer view in the chosen search region and fails if no default view exists or if that default view applies additional filters. - Discovery setup returns existing local indexes without forcing aggregator creation, and `discover init` retries as local-only setup when cross-region aggregator creation is denied. - Catalog collection uses Resource Explorer `ListResources` with filter strings instead of `Search`, which avoids the 1,000-result ceiling on filter-only queries. +- Resource Explorer catalog seeding batches `resourcetype:` and `region:` filters into the smallest possible query set, raises `MaxResults` to `1000`, and retries throttled `ListResources` calls before failing. - Account-scoped or fallback-backed datasets can bypass Resource Explorer seeding entirely by declaring no `resourceTypes`; the loader then receives `[]` and owns the account-level API call. - Resource Explorer inventory failures and dataset loader failures are fatal. The SDK does not degrade to partial live results. - Missing Lambda `Architectures` values from AWS are normalized to `['x86_64']`, matching the AWS default architecture. diff --git a/docs/reference/config-schema.md b/docs/reference/config-schema.md index 9b00db4..305fb9a 100644 --- a/docs/reference/config-schema.md +++ b/docs/reference/config-schema.md @@ -11,12 +11,12 @@ Source of truth: `packages/sdk/src/types.ts` (type), `packages/sdk/src/config/de Each mode uses the same fields: -| Field | Type | Default | Description | -| ---------------- | ---------------------------- | ------- | --------------------------------------------------------------------------- | -| `enabled-rules` | `string[]` | unset | If present, only the listed rule IDs remain active for that mode. | -| `disabled-rules` | `string[]` | unset | Rule IDs to remove from the active set after `enabled-rules` is applied. | -| `services` | `string[]` | unset | Service allowlist applied before `enabled-rules` and `disabled-rules`. | -| `format` | `'json' \| 'table'` | unset | Default CLI output format for that mode when `--format` is not passed. | +| Field | Type | Default | Description | +| ---------------- | ------------------- | ------- | ------------------------------------------------------------------------ | +| `enabled-rules` | `string[]` | unset | If present, only the listed rule IDs remain active for that mode. | +| `disabled-rules` | `string[]` | unset | Rule IDs to remove from the active set after `enabled-rules` is applied. | +| `services` | `string[]` | unset | Service allowlist applied before `enabled-rules` and `disabled-rules`. | +| `format` | `'json' \| 'table'` | unset | Default CLI output format for that mode when `--format` is not passed. | ## Merge Behavior @@ -89,7 +89,8 @@ discovery: - `cloudburn discover` defaults to the current region. - Current region resolution order is `AWS_REGION`, `AWS_DEFAULT_REGION`, `aws_region`, then the AWS SDK region provider chain. -- Passing `--region ` overrides the current region and queries Resource Explorer from that selected region. +- Passing `--region ` overrides the current region for the CLI discover command. - `discover({ target })` is the SDK live-discovery entrypoint. -- `--region all` requires an aggregator index and an unfiltered default Resource Explorer view in the aggregator region. +- `discover({ target: { mode: 'regions', regions: [...] } })` is the SDK shape for explicit discovery regions. +- Multi-region SDK discovery requires an aggregator index and an unfiltered default Resource Explorer view in the aggregator region. - `cloudburn discover init` defaults to the current region, accepts `--region ` as an override, and falls back to local-only setup in that region when cross-region aggregator setup is denied. diff --git a/docs/reference/rule-ids.md b/docs/reference/rule-ids.md index e6ee011..ef065c5 100644 --- a/docs/reference/rule-ids.md +++ b/docs/reference/rule-ids.md @@ -22,7 +22,7 @@ Format: `CLDBRN-{PROVIDER}-{SERVICE}-{N}` | `CLDBRN-AWS-CLOUDTRAIL-1` | CloudTrail Redundant Global Trails | cloudtrail | discovery | Implemented | | `CLDBRN-AWS-CLOUDTRAIL-2` | CloudTrail Redundant Regional Trails | cloudtrail | discovery | Implemented | | `CLDBRN-AWS-CLOUDWATCH-1` | CloudWatch Log Group Missing Retention | cloudwatch | discovery, iac | Implemented | -| `CLDBRN-AWS-CLOUDWATCH-2` | CloudWatch Unused Log Streams | cloudwatch | discovery | Implemented | +| `CLDBRN-AWS-CLOUDWATCH-2` | CloudWatch Log Group Inactive | cloudwatch | discovery | Implemented | | `CLDBRN-AWS-CLOUDWATCH-3` | CloudWatch Log Group No Metric Filters | cloudwatch | discovery | Implemented | | `CLDBRN-AWS-COSTGUARDRAILS-1` | AWS Budgets Missing | costguardrails | discovery | Implemented | | `CLDBRN-AWS-COSTGUARDRAILS-2` | Cost Anomaly Detection Missing | costguardrails | discovery | Implemented | @@ -108,7 +108,7 @@ Format: `CLDBRN-{PROVIDER}-{SERVICE}-{N}` `CLDBRN-AWS-EBS-7` flags only `completed` snapshots with a parsed `StartTime` older than `90` days. -`CLDBRN-AWS-CLOUDWATCH-2` flags log streams with no observed event history and log streams whose `lastIngestionTime` is more than 90 days old. Delivery-managed log groups remain exempt. +`CLDBRN-AWS-CLOUDWATCH-2` flags log groups whose most recent observed stream activity is missing or older than 90 days. Delivery-managed log groups remain exempt. `CLDBRN-AWS-CLOUDWATCH-3` reviews only log groups storing at least `1 GiB` and flags them when no metric filters are configured. diff --git a/packages/cloudburn/README.md b/packages/cloudburn/README.md index 607e93f..709df12 100644 --- a/packages/cloudburn/README.md +++ b/packages/cloudburn/README.md @@ -60,21 +60,21 @@ Use `discover` to run the same rules against live AWS resources. Run `cloudburn discover init` first. It automatically configures AWS Resource Explorer indexes, which CloudBurn uses as its live service catalog before it evaluates rules. -By default, `cloudburn discover` runs against your active AWS region. You can pass `--region ` to target another region, or use `--region all` to run against all indexed regions through the AWS Resource Explorer aggregator. +By default, `cloudburn discover` runs against your active AWS region. You can pass `--region ` to target one explicit region. ```bash cloudburn discover init cloudburn discover cloudburn discover --region eu-central-1 -cloudburn discover --region all cloudburn discover --config .cloudburn.yml --enabled-rules CLDBRN-AWS-EBS-1 cloudburn discover --service ec2,s3 -cloudburn discover list-enabled-regions --format text +cloudburn --debug discover --region eu-central-1 cloudburn rules list cloudburn rules list --service ec2 --source discovery ``` -`cloudburn discover --region all` needs an AWS Resource Explorer aggregator and an unfiltered default view in the aggregator region. +The CLI targets one region per run. Multi-region discovery remains available through the SDK and still needs an AWS Resource Explorer aggregator plus an unfiltered default view in the aggregator region. +Use `--debug` to print SDK and provider execution tracing to `stderr` without changing the normal `stdout` format. ## Shell Completion diff --git a/packages/cloudburn/src/cli.ts b/packages/cloudburn/src/cli.ts index 5b006b7..2442b7e 100644 --- a/packages/cloudburn/src/cli.ts +++ b/packages/cloudburn/src/cli.ts @@ -45,6 +45,7 @@ export const createProgram = (): Command => { .usage('[command]') .description('Know what you spend. Fix what you waste.') .version(__VERSION__) + .option('--debug', 'Write execution trace messages to stderr') .option('--format ', OUTPUT_FORMAT_OPTION_DESCRIPTION, parseOutputFormat); configureCliHelp(program); diff --git a/packages/cloudburn/src/commands/discover.ts b/packages/cloudburn/src/commands/discover.ts index 3c89673..1e1ccaa 100644 --- a/packages/cloudburn/src/commands/discover.ts +++ b/packages/cloudburn/src/commands/discover.ts @@ -1,5 +1,6 @@ -import { type AwsDiscoveryTarget, assertValidAwsRegion, CloudBurnClient } from '@cloudburn/sdk'; +import { type AwsDiscoveryTarget, type AwsRegion, assertValidAwsRegion, CloudBurnClient } from '@cloudburn/sdk'; import { type Command, InvalidArgumentError } from 'commander'; +import { resolveCliDebugLogger } from '../debug.js'; import { EXIT_CODE_OK, EXIT_CODE_POLICY_VIOLATION, EXIT_CODE_RUNTIME_ERROR } from '../exit-codes.js'; import { formatError } from '../formatters/error.js'; import { type CliResponse, type OutputFormat, renderResponse, resolveOutputFormat } from '../formatters/output.js'; @@ -12,7 +13,7 @@ type DiscoverOptions = { disabledRules?: string[]; enabledRules?: string[]; exitCode?: boolean; - region?: string; + region?: AwsRegion; service?: string[]; }; @@ -127,7 +128,7 @@ const buildInitializationStatusData = ( }; }; -const parseAwsRegion = (value: string): string => { +const parseAwsRegion = (value: string): AwsRegion => { try { return assertValidAwsRegion(value); } catch (err) { @@ -135,16 +136,26 @@ const parseAwsRegion = (value: string): string => { } }; -const parseDiscoverRegion = (value: string): string => { - if (value === 'all') { - return value; +const resolveDiscoveryTarget = (region?: AwsRegion): AwsDiscoveryTarget => + region === undefined ? { mode: 'current' } : { mode: 'regions', regions: [region] }; + +const resolveNestedRegionOption = (command: Command, subcommandName: string): string | undefined => { + const rootCommand = command.parent?.parent as (Command & { rawArgs?: string[] }) | undefined; + const rawArgs = rootCommand?.rawArgs ?? []; + const subcommandIndex = rawArgs.lastIndexOf(subcommandName); + + if (subcommandIndex === -1) { + return undefined; } - return parseAwsRegion(value); -}; + const optionIndex = rawArgs.indexOf('--region', subcommandIndex + 1); + + if (optionIndex === -1 || optionIndex + 1 >= rawArgs.length) { + return undefined; + } -const resolveDiscoveryTarget = (region?: string): AwsDiscoveryTarget => - region === undefined ? { mode: 'current' } : region === 'all' ? { mode: 'all' } : { mode: 'region', region }; + return rawArgs[optionIndex + 1]; +}; const toDiscoveryConfigOverride = (options: DiscoverOptions) => { if (options.enabledRules === undefined && options.disabledRules === undefined && options.service === undefined) { @@ -180,11 +191,10 @@ export const registerDiscoverCommand = (program: Command): void => { program .command('discover') .description('Run a live AWS discovery') - .enablePositionalOptions() .option( '--region ', - 'Discovery region to use. Defaults to the current AWS region from AWS_REGION; use this flag to override it. Pass "all" to check resources in all regions that are indexed in AWS Resource Explorer.', - parseDiscoverRegion, + 'AWS region to discover. Defaults to the current AWS region from AWS_REGION when omitted.', + parseAwsRegion, ) .option('--config ', 'Explicit CloudBurn config file to load') .option( @@ -205,7 +215,8 @@ export const registerDiscoverCommand = (program: Command): void => { .option('--exit-code', 'Exit with code 1 when findings exist') .action(async (options: DiscoverOptions, command: Command) => { await runCommand(async () => { - const scanner = new CloudBurnClient(); + const debugLogger = resolveCliDebugLogger(command); + const scanner = new CloudBurnClient({ debugLogger }); const configOverride = toDiscoveryConfigOverride(options); const loadedConfig = await scanner.loadConfig(options.config); const discoveryOptions: { @@ -240,9 +251,7 @@ export const registerDiscoverCommand = (program: Command): void => { [ 'cloudburn discover', 'cloudburn discover --region eu-central-1', - 'cloudburn discover --region all', 'cloudburn discover status', - 'cloudburn discover list-enabled-regions', 'cloudburn discover init', ], ); @@ -252,10 +261,9 @@ export const registerDiscoverCommand = (program: Command): void => { .description('Show Resource Explorer status across all enabled AWS regions') .action(async (_options: DiscoverListOptions, command: Command) => { await runCommand(async () => { - const scanner = new CloudBurnClient(); - const parentRegion = discoverCommand.opts().region; - const region = parentRegion === 'all' ? undefined : parentRegion; - const status = await scanner.getDiscoveryStatus({ region }); + const debugLogger = resolveCliDebugLogger(command); + const scanner = new CloudBurnClient({ debugLogger }); + const status = await scanner.getDiscoveryStatus({ region: undefined }); const format = resolveOutputFormat(command); const rows = format === 'json' @@ -314,32 +322,6 @@ export const registerDiscoverCommand = (program: Command): void => { }); }); - discoverCommand - .command('list-enabled-regions') - .description('List AWS regions with a local or aggregator Resource Explorer index') - .action(async (_options: DiscoverListOptions, command: Command) => { - await runCommand(async () => { - const scanner = new CloudBurnClient(); - const regions = await scanner.listEnabledDiscoveryRegions(); - const format = resolveOutputFormat(command); - const output = renderResponse( - { - kind: 'record-list', - columns: [ - { key: 'region', header: 'Region' }, - { key: 'type', header: 'Type' }, - ], - emptyMessage: 'No Resource Explorer indexes are enabled.', - rows: regions, - }, - format, - ); - - process.stdout.write(`${output}\n`); - return EXIT_CODE_OK; - }); - }); - discoverCommand .command('init') .description('Set up AWS Resource Explorer for CloudBurn') @@ -350,9 +332,9 @@ export const registerDiscoverCommand = (program: Command): void => { ) .action(async (options: DiscoverInitOptions, command: Command) => { await runCommand(async () => { - const scanner = new CloudBurnClient(); - const parentRegion = discoverCommand.opts().region; - const region = options.region ?? (parentRegion === 'all' ? undefined : parentRegion); + const debugLogger = resolveCliDebugLogger(command); + const scanner = new CloudBurnClient({ debugLogger }); + const region = options.region ?? resolveNestedRegionOption(command, 'init'); const result = await scanner.initializeDiscovery({ region }); const message = describeInitializationMessage(result); const format = resolveOutputFormat(command); @@ -374,7 +356,8 @@ export const registerDiscoverCommand = (program: Command): void => { .description('List Resource Explorer supported AWS resource types') .action(async (_options: DiscoverListOptions, command: Command) => { await runCommand(async () => { - const scanner = new CloudBurnClient(); + const debugLogger = resolveCliDebugLogger(command); + const scanner = new CloudBurnClient({ debugLogger }); const resourceTypes = await scanner.listSupportedDiscoveryResourceTypes(); const format = resolveOutputFormat(command); const output = renderResponse( diff --git a/packages/cloudburn/src/commands/scan.ts b/packages/cloudburn/src/commands/scan.ts index 5e49da5..10d5e45 100644 --- a/packages/cloudburn/src/commands/scan.ts +++ b/packages/cloudburn/src/commands/scan.ts @@ -1,5 +1,6 @@ import { CloudBurnClient } from '@cloudburn/sdk'; import type { Command } from 'commander'; +import { resolveCliDebugLogger } from '../debug.js'; import { EXIT_CODE_OK, EXIT_CODE_POLICY_VIOLATION, EXIT_CODE_RUNTIME_ERROR } from '../exit-codes.js'; import { formatError } from '../formatters/error.js'; import { renderResponse, resolveOutputFormat } from '../formatters/output.js'; @@ -54,7 +55,8 @@ export const registerScanCommand = (program: Command): void => { .option('--exit-code', 'Exit with code 1 when findings exist') .action(async (path: string | undefined, options: ScanOptions, command: Command) => { try { - const scanner = new CloudBurnClient(); + const debugLogger = resolveCliDebugLogger(command); + const scanner = new CloudBurnClient({ debugLogger }); const configOverride = toScanConfigOverride(options); const loadedConfig = await scanner.loadConfig(options.config); const scanPath = path ?? process.cwd(); diff --git a/packages/cloudburn/src/debug.ts b/packages/cloudburn/src/debug.ts new file mode 100644 index 0000000..6754684 --- /dev/null +++ b/packages/cloudburn/src/debug.ts @@ -0,0 +1,29 @@ +import type { Command } from 'commander'; + +/** + * Resolves the effective CLI debug flag, including inherited global options. + * + * @param command - Active Commander command. + * @returns Whether debug output is enabled. + */ +export const isDebugEnabled = (command: Command): boolean => { + const options = command.optsWithGlobals() as { debug?: boolean }; + + return options.debug === true; +}; + +/** + * Creates a stderr debug logger for the active command when requested. + * + * @param command - Active Commander command. + * @returns Logger callback, or `undefined` when debug mode is off. + */ +export const resolveCliDebugLogger = (command: Command): ((message: string) => void) | undefined => { + if (!isDebugEnabled(command)) { + return undefined; + } + + return (message: string) => { + process.stderr.write(`[debug] ${message}\n`); + }; +}; diff --git a/packages/cloudburn/test/cli.test.ts b/packages/cloudburn/test/cli.test.ts index 84cc1f7..69fdec3 100644 --- a/packages/cloudburn/test/cli.test.ts +++ b/packages/cloudburn/test/cli.test.ts @@ -35,6 +35,7 @@ describe('cli', () => { const help = stdout.mock.calls.map(([chunk]) => String(chunk)).join(''); + expect(help).toContain('--debug'); expect(help).toContain('--format '); expect(help).toContain('completion'); expect(help).toContain('table: human-readable terminal output'); diff --git a/packages/cloudburn/test/completion.test.ts b/packages/cloudburn/test/completion.test.ts index f03ed80..6f2bfd7 100644 --- a/packages/cloudburn/test/completion.test.ts +++ b/packages/cloudburn/test/completion.test.ts @@ -45,6 +45,7 @@ describe('completion command', () => { 'rules', 'estimate', 'completion', + '--debug', '--format', '-h', '--help', @@ -59,12 +60,12 @@ describe('completion command', () => { expect(suggestions).toEqual( expect.arrayContaining([ 'init', - 'list-enabled-regions', 'supported-resource-types', '--config', '--disabled-rules', '--enabled-rules', '--region', + '--debug', '--format', '--exit-code', '-h', @@ -76,8 +77,7 @@ describe('completion command', () => { it('suggests only flags for nested discover init contexts', async () => { const suggestions = toLines(await runCompletion('discover', 'init', '--')); - expect(suggestions).toEqual(expect.arrayContaining(['--region', '--format', '--help'])); - expect(suggestions).not.toContain('list-enabled-regions'); + expect(suggestions).toEqual(expect.arrayContaining(['--region', '--debug', '--format', '--help'])); }); it('does not suggest completions while the cursor is positioned on an option value', async () => { diff --git a/packages/cloudburn/test/discover.e2e.test.ts b/packages/cloudburn/test/discover.e2e.test.ts index 98b1ad5..a0c2a5d 100644 --- a/packages/cloudburn/test/discover.e2e.test.ts +++ b/packages/cloudburn/test/discover.e2e.test.ts @@ -158,20 +158,51 @@ describe('discover command e2e', () => { await createProgram().parseAsync(['discover', '--region', 'eu-central-1'], { from: 'user' }); - expect(discover).toHaveBeenCalledWith({ target: { mode: 'region', region: 'eu-central-1' } }); + expect(discover).toHaveBeenCalledWith({ target: { mode: 'regions', regions: ['eu-central-1'] } }); expect(process.exitCode).toBe(0); }); + it('writes sdk debug tracing to stderr without adding cli-originated debug lines', async () => { + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + vi.spyOn(CloudBurnClient.prototype, 'loadConfig').mockImplementation(async function () { + (this as { options?: { debugLogger?: (message: string) => void } }).options?.debugLogger?.( + 'sdk: loading config from default search path', + ); + + return { + discovery: {}, + iac: {}, + }; + }); + vi.spyOn(CloudBurnClient.prototype, 'discover').mockImplementation(async function () { + (this as { options?: { debugLogger?: (message: string) => void } }).options?.debugLogger?.( + 'sdk: starting live discovery scan', + ); + + return { providers: [] }; + }); + + await createProgram().parseAsync(['discover', '--debug'], { from: 'user' }); + + const debugOutput = stderr.mock.calls.map(([chunk]) => String(chunk)).join(''); + + expect(debugOutput).toContain('[debug] sdk: loading config from default search path'); + expect(debugOutput).toContain('[debug] sdk: starting live discovery scan'); + expect(debugOutput).not.toContain('[debug] discover:'); + expect(stdout).toHaveBeenCalledWith('No findings.\n'); + }); + it('rejects invalid discovery regions before invoking the sdk', async () => { const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); const discover = vi.spyOn(CloudBurnClient.prototype, 'discover').mockResolvedValue({ providers: [] }); await expect( - createProgram().parseAsync(['discover', '--region', 'us-east-1 region:eu-west-1'], { from: 'user' }), + createProgram().parseAsync(['discover', '--region', 'totally-fake-1'], { from: 'user' }), ).rejects.toThrow('process.exit unexpectedly called with "1"'); expect(discover).not.toHaveBeenCalled(); - expect(stderr).toHaveBeenCalledWith(expect.stringContaining("Invalid AWS region 'us-east-1 region:eu-west-1'.")); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining("Invalid AWS region 'totally-fake-1'.")); }); it('rejects invalid service filters before invoking the sdk discover method', async () => { @@ -192,13 +223,28 @@ describe('discover command e2e', () => { expect(stderr).toHaveBeenCalled(); }); - it('passes the all-regions target to the sdk discover method', async () => { + it('rejects all as a discovery region value', async () => { + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); const discover = vi.spyOn(CloudBurnClient.prototype, 'discover').mockResolvedValue({ providers: [] }); - await createProgram().parseAsync(['discover', '--region', 'all'], { from: 'user' }); + await expect(createProgram().parseAsync(['discover', '--region', 'all'], { from: 'user' })).rejects.toThrow( + 'process.exit unexpectedly called with "1"', + ); - expect(discover).toHaveBeenCalledWith({ target: { mode: 'all' } }); - expect(process.exitCode).toBe(0); + expect(discover).not.toHaveBeenCalled(); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining("Invalid AWS region 'all'.")); + }); + + it('rejects comma-separated discovery regions in the cli before invoking the sdk', async () => { + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const discover = vi.spyOn(CloudBurnClient.prototype, 'discover').mockResolvedValue({ providers: [] }); + + await expect( + createProgram().parseAsync(['discover', '--region', 'eu-central-1,us-east-1'], { from: 'user' }), + ).rejects.toThrow('process.exit unexpectedly called with "1"'); + + expect(discover).not.toHaveBeenCalled(); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining("Invalid AWS region 'eu-central-1,us-east-1'.")); }); it('preserves the policy violation exit code when discover finds resources and --exit-code is set', async () => { @@ -305,10 +351,10 @@ describe('discover command e2e', () => { expect(help).toContain('Run a live AWS discovery'); expect(help).toContain('--region '); - expect(help).toContain('Defaults to the current'); + expect(help).toContain('AWS region to discover. Defaults'); expect(help).toContain('AWS region from AWS_REGION'); - expect(help).toContain('Pass "all" to check resources in all'); - expect(help).toContain('regions that are indexed in AWS Resource Explorer'); + expect(help).toContain('omitted.'); + expect(help).not.toContain('--regions '); expect(help).toContain('--config '); expect(help).toContain('--enabled-rules '); expect(help).toContain('When set,'); @@ -321,56 +367,7 @@ describe('discover command e2e', () => { expect(help).toContain('use this to exclude'); expect(help).toContain('specific rules'); expect(help).toContain('cloudburn discover'); - expect(help).toContain('cloudburn discover --region all'); - expect(help).toContain('cloudburn discover list-enabled-regions'); - }); - - it('lists enabled regions via the sdk', async () => { - const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); - const listEnabledRegions = vi.spyOn(CloudBurnClient.prototype, 'listEnabledDiscoveryRegions').mockResolvedValue([ - { region: 'eu-west-1', type: 'local' }, - { region: 'eu-central-1', type: 'aggregator' }, - ]); - - await createProgram().parseAsync(['discover', 'list-enabled-regions', '--format', 'json'], { from: 'user' }); - - expect(listEnabledRegions).toHaveBeenCalledTimes(1); - expect(stdout).toHaveBeenCalledWith(`[ - { - "region": "eu-west-1", - "type": "local" - }, - { - "region": "eu-central-1", - "type": "aggregator" - } -]\n`); - expect(process.exitCode).toBe(0); - }); - - it('rejects text output for list-enabled-regions before invoking the sdk', async () => { - const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); - const listEnabledRegions = vi.spyOn(CloudBurnClient.prototype, 'listEnabledDiscoveryRegions').mockResolvedValue([ - { region: 'eu-west-1', type: 'local' }, - { region: 'eu-central-1', type: 'aggregator' }, - ]); - const program = createProgram(); - const discoverCommand = program.commands.find((command) => command.name() === 'discover'); - const listCommand = discoverCommand?.commands.find((command) => command.name() === 'list-enabled-regions'); - - program.exitOverride(); - discoverCommand?.exitOverride(); - listCommand?.exitOverride(); - - await expect( - program.parseAsync(['discover', '--format', 'text', 'list-enabled-regions'], { from: 'user' }), - ).rejects.toMatchObject({ - code: 'commander.invalidArgument', - exitCode: 1, - message: expect.stringContaining('text'), - }); - expect(listEnabledRegions).not.toHaveBeenCalled(); - expect(stderr).toHaveBeenCalled(); + expect(help).toContain('cloudburn discover --region eu-central-1'); }); it('initializes resource explorer setup via the sdk', async () => { @@ -506,7 +503,7 @@ describe('discover command e2e', () => { expect(process.exitCode).toBe(0); }); - it('does not forward parent --region all into discover init', async () => { + it('does not forward discover region into discover init', async () => { const initializeDiscovery = vi.spyOn(CloudBurnClient.prototype, 'initializeDiscovery').mockResolvedValue({ aggregatorAction: 'created', status: 'CREATED', @@ -520,20 +517,20 @@ describe('discover command e2e', () => { verificationStatus: 'verified', }); - await createProgram().parseAsync(['discover', '--region', 'all', 'init'], { from: 'user' }); + await createProgram().parseAsync(['discover', '--region', 'eu-central-1', 'init'], { from: 'user' }); expect(initializeDiscovery).toHaveBeenCalledWith({ region: undefined }); expect(process.exitCode).toBe(0); }); - it('reuses the parent discovery region for discover status unless it is all', async () => { + it('does not forward discover region into discover status', async () => { const getDiscoveryStatus = vi .spyOn(CloudBurnClient.prototype, 'getDiscoveryStatus') .mockResolvedValue(observedAggregatorStatus); await createProgram().parseAsync(['discover', '--region', 'eu-central-1', 'status'], { from: 'user' }); - expect(getDiscoveryStatus).toHaveBeenCalledWith({ region: 'eu-central-1' }); + expect(getDiscoveryStatus).toHaveBeenCalledWith({ region: undefined }); expect(process.exitCode).toBe(0); }); diff --git a/packages/cloudburn/test/help.e2e.test.ts b/packages/cloudburn/test/help.e2e.test.ts index 4b883f9..43ab5cc 100644 --- a/packages/cloudburn/test/help.e2e.test.ts +++ b/packages/cloudburn/test/help.e2e.test.ts @@ -51,6 +51,7 @@ describe('cli help e2e', () => { expect(help).toContain('Usage: cloudburn'); expect(help).toContain('Available Commands:'); expect(help).toContain('Global Flags:'); + expect(help).toContain('--debug'); expect(help).toContain('completion'); expect(help).not.toContain('__complete'); expect(help).not.toContain('Use "cloudburn [command] --help" for more information about a command.'); @@ -108,6 +109,7 @@ describe('cli help e2e', () => { expect(help).toContain('Flags:'); expect(help).toContain('--no-descriptions'); expect(help).toContain('Global Flags:'); + expect(help).toContain('--debug'); expect(help).toContain('--format '); expect(help).toContain('Options: table: human-readable terminal output.'); expect(help).toContain('json: machine-readable output for automation and downstream systems.'); @@ -125,6 +127,7 @@ describe('cli help e2e', () => { expect(help).toContain('Requested aggregator region to create or reuse'); expect(help).toContain('during setup.'); expect(help).toContain('Global Flags:'); + expect(help).toContain('--debug'); expect(help).toContain('--format '); expect(help).not.toContain('Discovery region to use. Defaults to the current AWS region from AWS_REGION'); expect(help).not.toContain('--config '); diff --git a/packages/rules/src/aws/cloudwatch/unused-log-streams.ts b/packages/rules/src/aws/cloudwatch/unused-log-streams.ts index 68357e6..36b9b76 100644 --- a/packages/rules/src/aws/cloudwatch/unused-log-streams.ts +++ b/packages/rules/src/aws/cloudwatch/unused-log-streams.ts @@ -3,51 +3,60 @@ import { createFinding, createFindingMatch, createRule } from '../../shared/help const RULE_ID = 'CLDBRN-AWS-CLOUDWATCH-2'; const RULE_SERVICE = 'cloudwatch'; const RULE_MESSAGE = - 'CloudWatch log streams that have never received events or have been inactive for more than 90 days should be removed.'; + 'CloudWatch log groups whose most recent stream activity is older than 90 days should be reviewed or removed.'; const DAY_MS = 24 * 60 * 60 * 1000; const UNUSED_LOG_STREAM_DAYS = 90; const toLogGroupScopeKey = (region: string, accountId: string, logGroupName: string): string => `${region}:${accountId}:${logGroupName}`; -/** Flag CloudWatch log streams with no event history or stale ingestion outside delivery-managed log groups. */ +/** Flag CloudWatch log groups whose latest observed stream activity is stale outside delivery-managed log groups. */ export const cloudWatchUnusedLogStreamsRule = createRule({ id: RULE_ID, - name: 'CloudWatch Unused Log Streams', + name: 'CloudWatch Log Group Inactive', description: - 'Flag CloudWatch log streams that have never received events or whose last ingestion was more than 90 days ago outside delivery-managed log groups.', + 'Flag CloudWatch log groups whose most recent stream has no observed event history or whose latest stream activity is more than 90 days old outside delivery-managed log groups.', message: RULE_MESSAGE, provider: 'aws', service: RULE_SERVICE, supports: ['discovery'], - discoveryDependencies: ['aws-cloudwatch-log-groups', 'aws-cloudwatch-log-streams'], + discoveryDependencies: ['aws-cloudwatch-log-groups', 'aws-cloudwatch-log-group-recent-stream-activity'], evaluateLive: ({ resources }) => { const cutoff = Date.now() - UNUSED_LOG_STREAM_DAYS * DAY_MS; const logGroups = resources.get('aws-cloudwatch-log-groups'); - const knownLogGroups = new Set( - logGroups.map((logGroup) => toLogGroupScopeKey(logGroup.region, logGroup.accountId, logGroup.logGroupName)), + const logGroupsByScopeKey = new Map( + logGroups.map((logGroup) => [ + toLogGroupScopeKey(logGroup.region, logGroup.accountId, logGroup.logGroupName), + logGroup, + ]), ); const deliveryManagedLogGroups = new Set( logGroups .filter((logGroup) => logGroup.logGroupClass === 'DELIVERY') .map((logGroup) => toLogGroupScopeKey(logGroup.region, logGroup.accountId, logGroup.logGroupName)), ); + const recentActivityByScopeKey = new Map( + resources + .get('aws-cloudwatch-log-group-recent-stream-activity') + .map((activity) => [toLogGroupScopeKey(activity.region, activity.accountId, activity.logGroupName), activity]), + ); - const findings = resources - .get('aws-cloudwatch-log-streams') - .filter((logStream) => { - const logGroupScopeKey = toLogGroupScopeKey(logStream.region, logStream.accountId, logStream.logGroupName); + const findings = logGroups + .filter((logGroup) => { + const logGroupScopeKey = toLogGroupScopeKey(logGroup.region, logGroup.accountId, logGroup.logGroupName); + const recentActivity = recentActivityByScopeKey.get(logGroupScopeKey); + const latestActivityTimestamp = + recentActivity?.lastEventTimestamp !== undefined || recentActivity?.lastIngestionTime !== undefined + ? Math.max(recentActivity?.lastEventTimestamp ?? 0, recentActivity?.lastIngestionTime ?? 0) + : undefined; return ( - knownLogGroups.has(logGroupScopeKey) && + logGroupsByScopeKey.has(logGroupScopeKey) && !deliveryManagedLogGroups.has(logGroupScopeKey) && - ((logStream.firstEventTimestamp === undefined && - logStream.lastEventTimestamp === undefined && - logStream.lastIngestionTime === undefined) || - (logStream.lastIngestionTime !== undefined && logStream.lastIngestionTime < cutoff)) + (latestActivityTimestamp === undefined || latestActivityTimestamp < cutoff) ); }) - .map((logStream) => createFindingMatch(logStream.arn, logStream.region, logStream.accountId)); + .map((logGroup) => createFindingMatch(logGroup.logGroupArn, logGroup.region, logGroup.accountId)); return createFinding({ id: RULE_ID, service: RULE_SERVICE, message: RULE_MESSAGE }, 'discovery', findings); }, diff --git a/packages/rules/src/index.ts b/packages/rules/src/index.ts index 838ed59..afb5148 100644 --- a/packages/rules/src/index.ts +++ b/packages/rules/src/index.ts @@ -18,6 +18,7 @@ export type { AwsCloudFrontDistributionRequestActivity, AwsCloudTrailTrail, AwsCloudWatchLogGroup, + AwsCloudWatchLogGroupRecentStreamActivity, AwsCloudWatchLogMetricFilterCoverage, AwsCloudWatchLogStream, AwsCostAnomalyMonitor, diff --git a/packages/rules/src/shared/metadata.ts b/packages/rules/src/shared/metadata.ts index 74ac9a7..9d9c5e6 100644 --- a/packages/rules/src/shared/metadata.ts +++ b/packages/rules/src/shared/metadata.ts @@ -89,6 +89,17 @@ export type AwsCloudWatchLogStream = { accountId: string; }; +/** Discovered CloudWatch Logs latest-stream activity summary keyed by log group. */ +export type AwsCloudWatchLogGroupRecentStreamActivity = { + logGroupName: string; + latestStreamArn?: string; + latestStreamName?: string; + lastEventTimestamp?: number; + lastIngestionTime?: number; + region: string; + accountId: string; +}; + /** Discovered CloudWatch Logs metric-filter coverage keyed by log group. */ export type AwsCloudWatchLogMetricFilterCoverage = { logGroupName: string; @@ -647,6 +658,7 @@ export type DiscoveryDatasetKey = | 'aws-cloudfront-distributions' | 'aws-cloudfront-distribution-request-activity' | 'aws-cloudwatch-log-groups' + | 'aws-cloudwatch-log-group-recent-stream-activity' | 'aws-cloudwatch-log-metric-filter-coverage' | 'aws-cloudwatch-log-streams' | 'aws-cost-usage' @@ -702,6 +714,7 @@ export type DiscoveryDatasetMap = { 'aws-cloudfront-distributions': AwsCloudFrontDistribution[]; 'aws-cloudfront-distribution-request-activity': AwsCloudFrontDistributionRequestActivity[]; 'aws-cloudwatch-log-groups': AwsCloudWatchLogGroup[]; + 'aws-cloudwatch-log-group-recent-stream-activity': AwsCloudWatchLogGroupRecentStreamActivity[]; 'aws-cloudwatch-log-metric-filter-coverage': AwsCloudWatchLogMetricFilterCoverage[]; 'aws-cloudwatch-log-streams': AwsCloudWatchLogStream[]; 'aws-cost-usage': AwsCostUsage[]; diff --git a/packages/rules/test/cloudwatch-unused-log-streams.test.ts b/packages/rules/test/cloudwatch-unused-log-streams.test.ts index 2506224..7778ce7 100644 --- a/packages/rules/test/cloudwatch-unused-log-streams.test.ts +++ b/packages/rules/test/cloudwatch-unused-log-streams.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { cloudWatchUnusedLogStreamsRule } from '../src/aws/cloudwatch/unused-log-streams.js'; -import type { AwsCloudWatchLogGroup, AwsCloudWatchLogStream } from '../src/index.js'; +import type { AwsCloudWatchLogGroup, AwsCloudWatchLogGroupRecentStreamActivity } from '../src/index.js'; import { LiveResourceBag } from '../src/index.js'; const DAY_MS = 24 * 60 * 60 * 1000; @@ -13,11 +13,13 @@ const createLogGroup = (overrides: Partial = {}): AwsClou ...overrides, }); -const createLogStream = (overrides: Partial = {}): AwsCloudWatchLogStream => ({ +const createRecentActivity = ( + overrides: Partial = {}, +): AwsCloudWatchLogGroupRecentStreamActivity => ({ accountId: '123456789012', - arn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:2026/03/16/[$LATEST]abc', logGroupName: '/aws/lambda/app', - logStreamName: '2026/03/16/[$LATEST]abc', + latestStreamArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:2026/03/16/[$LATEST]abc', + latestStreamName: '2026/03/16/[$LATEST]abc', region: 'us-east-1', ...overrides, }); @@ -32,7 +34,7 @@ describe('cloudWatchUnusedLogStreamsRule', () => { vi.useRealTimers(); }); - it('flags log streams with no event history', () => { + it('flags log groups whose latest stream has no event history', () => { const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ catalog: { resources: [], @@ -41,7 +43,7 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }, resources: new LiveResourceBag({ 'aws-cloudwatch-log-groups': [createLogGroup()], - 'aws-cloudwatch-log-streams': [createLogStream()], + 'aws-cloudwatch-log-group-recent-stream-activity': [createRecentActivity()], }), }); @@ -50,11 +52,10 @@ describe('cloudWatchUnusedLogStreamsRule', () => { service: 'cloudwatch', source: 'discovery', message: - 'CloudWatch log streams that have never received events or have been inactive for more than 90 days should be removed.', + 'CloudWatch log groups whose most recent stream activity is older than 90 days should be reviewed or removed.', findings: [ { - resourceId: - 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:2026/03/16/[$LATEST]abc', + resourceId: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app', region: 'us-east-1', accountId: '123456789012', }, @@ -62,7 +63,7 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }); }); - it('flags log streams whose last ingestion was more than 90 days ago', () => { + it('flags log groups whose latest stream activity was more than 90 days ago', () => { const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ catalog: { resources: [], @@ -71,20 +72,22 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }, resources: new LiveResourceBag({ 'aws-cloudwatch-log-groups': [createLogGroup()], - 'aws-cloudwatch-log-streams': [createLogStream({ lastIngestionTime: Date.now() - 91 * DAY_MS })], + 'aws-cloudwatch-log-group-recent-stream-activity': [ + createRecentActivity({ lastIngestionTime: Date.now() - 91 * DAY_MS }), + ], }), }); expect(finding?.findings).toEqual([ { - resourceId: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:2026/03/16/[$LATEST]abc', + resourceId: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app', region: 'us-east-1', accountId: '123456789012', }, ]); }); - it('does not flag log streams with observed event history', () => { + it('does not flag log groups with recent observed stream activity', () => { const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ catalog: { resources: [], @@ -93,14 +96,16 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }, resources: new LiveResourceBag({ 'aws-cloudwatch-log-groups': [createLogGroup()], - 'aws-cloudwatch-log-streams': [createLogStream({ lastEventTimestamp: 1_710_000_000_000 })], + 'aws-cloudwatch-log-group-recent-stream-activity': [ + createRecentActivity({ lastEventTimestamp: 1_770_000_000_000 }), + ], }), }); expect(finding).toBeNull(); }); - it('does not flag log streams whose last ingestion was within 90 days', () => { + it('does not flag log groups whose latest stream ingestion was within 90 days', () => { const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ catalog: { resources: [], @@ -109,14 +114,16 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }, resources: new LiveResourceBag({ 'aws-cloudwatch-log-groups': [createLogGroup()], - 'aws-cloudwatch-log-streams': [createLogStream({ lastIngestionTime: Date.now() - 30 * DAY_MS })], + 'aws-cloudwatch-log-group-recent-stream-activity': [ + createRecentActivity({ lastIngestionTime: Date.now() - 30 * DAY_MS }), + ], }), }); expect(finding).toBeNull(); }); - it('does not flag log streams whose last ingestion was exactly 90 days ago', () => { + it('does not flag log groups whose latest stream activity was exactly 90 days ago', () => { const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ catalog: { resources: [], @@ -125,14 +132,16 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }, resources: new LiveResourceBag({ 'aws-cloudwatch-log-groups': [createLogGroup()], - 'aws-cloudwatch-log-streams': [createLogStream({ lastIngestionTime: Date.now() - 90 * DAY_MS })], + 'aws-cloudwatch-log-group-recent-stream-activity': [ + createRecentActivity({ lastIngestionTime: Date.now() - 90 * DAY_MS }), + ], }), }); expect(finding).toBeNull(); }); - it('does not flag streams inside delivery-managed log groups', () => { + it('does not flag delivery-managed log groups even when their latest stream is stale', () => { const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ catalog: { resources: [], @@ -141,14 +150,14 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }, resources: new LiveResourceBag({ 'aws-cloudwatch-log-groups': [createLogGroup({ logGroupClass: 'DELIVERY' })], - 'aws-cloudwatch-log-streams': [createLogStream()], + 'aws-cloudwatch-log-group-recent-stream-activity': [createRecentActivity()], }), }); expect(finding).toBeNull(); }); - it('does not flag streams when log-group metadata is unavailable', () => { + it('does not flag activity summaries when log-group metadata is unavailable', () => { const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ catalog: { resources: [], @@ -157,7 +166,7 @@ describe('cloudWatchUnusedLogStreamsRule', () => { }, resources: new LiveResourceBag({ 'aws-cloudwatch-log-groups': [], - 'aws-cloudwatch-log-streams': [createLogStream()], + 'aws-cloudwatch-log-group-recent-stream-activity': [createRecentActivity()], }), }); @@ -186,13 +195,35 @@ describe('cloudWatchUnusedLogStreamsRule', () => { logGroupClass: 'DELIVERY', }), ], - 'aws-cloudwatch-log-streams': [createLogStream()], + 'aws-cloudwatch-log-group-recent-stream-activity': [createRecentActivity()], + }), + }); + + expect(finding?.findings).toEqual([ + { + resourceId: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app', + region: 'us-east-1', + accountId: '123456789012', + }, + ]); + }); + + it('flags log groups when no streams exist yet', () => { + const finding = cloudWatchUnusedLogStreamsRule.evaluateLive?.({ + catalog: { + resources: [], + searchRegion: 'us-east-1', + indexType: 'LOCAL', + }, + resources: new LiveResourceBag({ + 'aws-cloudwatch-log-groups': [createLogGroup()], + 'aws-cloudwatch-log-group-recent-stream-activity': [], }), }); expect(finding?.findings).toEqual([ { - resourceId: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:2026/03/16/[$LATEST]abc', + resourceId: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app', region: 'us-east-1', accountId: '123456789012', }, diff --git a/packages/rules/test/exports.test.ts b/packages/rules/test/exports.test.ts index 41d2249..38cd928 100644 --- a/packages/rules/test/exports.test.ts +++ b/packages/rules/test/exports.test.ts @@ -441,6 +441,7 @@ describe('rule exports', () => { const cloudFrontRequestActivityDatasetKey: DiscoveryDatasetKey = 'aws-cloudfront-distribution-request-activity'; const datasetKey: DiscoveryDatasetKey = 'aws-rds-instances'; const cloudWatchDatasetKey: DiscoveryDatasetKey = 'aws-cloudwatch-log-groups'; + const cloudWatchRecentActivityDatasetKey: DiscoveryDatasetKey = 'aws-cloudwatch-log-group-recent-stream-activity'; const cloudWatchLogStreamDatasetKey: DiscoveryDatasetKey = 'aws-cloudwatch-log-streams'; const costUsageDatasetKey: DiscoveryDatasetKey = 'aws-cost-usage'; const dynamoDbAutoscalingDatasetKey: DiscoveryDatasetKey = 'aws-dynamodb-autoscaling'; @@ -471,6 +472,7 @@ describe('rule exports', () => { expect(cloudFrontRequestActivityDatasetKey).toBe('aws-cloudfront-distribution-request-activity'); expect(datasetKey).toBe('aws-rds-instances'); expect(cloudWatchDatasetKey).toBe('aws-cloudwatch-log-groups'); + expect(cloudWatchRecentActivityDatasetKey).toBe('aws-cloudwatch-log-group-recent-stream-activity'); expect(cloudWatchLogStreamDatasetKey).toBe('aws-cloudwatch-log-streams'); expect(costUsageDatasetKey).toBe('aws-cost-usage'); expect(dynamoDbAutoscalingDatasetKey).toBe('aws-dynamodb-autoscaling'); diff --git a/packages/rules/test/rule-metadata.test.ts b/packages/rules/test/rule-metadata.test.ts index 47b46a2..25df261 100644 --- a/packages/rules/test/rule-metadata.test.ts +++ b/packages/rules/test/rule-metadata.test.ts @@ -122,15 +122,15 @@ describe('rule metadata', () => { expect(rule).toBeDefined(); expect(rule).toMatchObject({ id: 'CLDBRN-AWS-CLOUDWATCH-2', - name: 'CloudWatch Unused Log Streams', + name: 'CloudWatch Log Group Inactive', description: - 'Flag CloudWatch log streams that have never received events or whose last ingestion was more than 90 days ago outside delivery-managed log groups.', + 'Flag CloudWatch log groups whose most recent stream has no observed event history or whose latest stream activity is more than 90 days old outside delivery-managed log groups.', message: - 'CloudWatch log streams that have never received events or have been inactive for more than 90 days should be removed.', + 'CloudWatch log groups whose most recent stream activity is older than 90 days should be reviewed or removed.', provider: 'aws', service: 'cloudwatch', supports: ['discovery'], - discoveryDependencies: ['aws-cloudwatch-log-groups', 'aws-cloudwatch-log-streams'], + discoveryDependencies: ['aws-cloudwatch-log-groups', 'aws-cloudwatch-log-group-recent-stream-activity'], }); }); diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 5062ea3..473fc77 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -46,10 +46,15 @@ const client = new CloudBurnClient(); await client.initializeDiscovery(); const currentRegion = await client.discover(); -const allRegions = await client.discover({ target: { mode: 'all' } }); +const explicitRegion = await client.discover({ + target: { mode: 'regions', regions: ['eu-central-1'] }, +}); +const multipleRegions = await client.discover({ + target: { mode: 'regions', regions: ['eu-central-1', 'us-east-1'] }, +}); ``` -`discover()` defaults to the current AWS region. You can also target a specific region with `{ target: { mode: 'region', region: 'eu-central-1' } }`. +`discover()` defaults to the current AWS region. You can also target one or more explicit AWS regions with `{ target: { mode: 'regions', regions: [...] } }`. Multi-region discovery requires an AWS Resource Explorer aggregator index. ### Lower-level helpers @@ -61,7 +66,6 @@ The `CloudBurnClient` also exposes helper methods: - `client.loadConfig(path?)` to resolve CloudBurn config from disk - `client.getDiscoveryStatus()` to inspect AWS Resource Explorer readiness -- `client.listEnabledDiscoveryRegions()` to see which regions have indexes - `client.listSupportedDiscoveryResourceTypes()` to inspect the AWS resource types discovery can search ## Docs diff --git a/packages/sdk/src/debug.ts b/packages/sdk/src/debug.ts new file mode 100644 index 0000000..e69cd34 --- /dev/null +++ b/packages/sdk/src/debug.ts @@ -0,0 +1,10 @@ +/** + * Emits an SDK debug line when a logger is configured. + * + * @param debugLogger - Optional logger callback provided by the caller. + * @param message - Human-readable trace message. + * @returns Nothing. + */ +export const emitDebugLog = (debugLogger: ((message: string) => void) | undefined, message: string): void => { + debugLogger?.(message); +}; diff --git a/packages/sdk/src/engine/run-live.ts b/packages/sdk/src/engine/run-live.ts index f891b35..a8f6207 100644 --- a/packages/sdk/src/engine/run-live.ts +++ b/packages/sdk/src/engine/run-live.ts @@ -1,11 +1,20 @@ +import { emitDebugLog } from '../debug.js'; import { discoverAwsResources } from '../providers/aws/discovery.js'; import type { AwsDiscoveryTarget, CloudBurnConfig, ScanResult } from '../types.js'; import { groupFindingsByProvider } from './group-findings.js'; import { buildRuleRegistry } from './registry.js'; -export const runLiveScan = async (config: CloudBurnConfig, target: AwsDiscoveryTarget): Promise => { +export const runLiveScan = async ( + config: CloudBurnConfig, + target: AwsDiscoveryTarget, + options?: { debugLogger?: (message: string) => void }, +): Promise => { const registry = buildRuleRegistry(config, 'discovery'); - const { diagnostics = [], ...liveContext } = await discoverAwsResources(registry.activeRules, target); + emitDebugLog(options?.debugLogger, `sdk: resolved ${registry.activeRules.length} active discovery rules`); + const { diagnostics = [], ...liveContext } = + options?.debugLogger === undefined + ? await discoverAwsResources(registry.activeRules, target) + : await discoverAwsResources(registry.activeRules, target, { debugLogger: options.debugLogger }); const findings = groupFindingsByProvider( registry.activeRules.map((rule) => { if (!rule.supports.includes('discovery') || !rule.evaluateLive) { diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index b61be5c..8b2b305 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -48,6 +48,7 @@ export type { AwsRedshiftCluster, AwsRedshiftClusterMetric, AwsRedshiftReservedNode, + AwsRegion, AwsRoute53HealthCheck, AwsRoute53Record, AwsRoute53Zone, diff --git a/packages/sdk/src/providers/aws/client.ts b/packages/sdk/src/providers/aws/client.ts index 2f7105d..4ec0dc0 100644 --- a/packages/sdk/src/providers/aws/client.ts +++ b/packages/sdk/src/providers/aws/client.ts @@ -32,6 +32,44 @@ export type AwsClientConfig = { const AWS_REGION_PATTERN = /^[a-z]{2}(?:-[a-z0-9]+)+-\d+$/; const AWS_GLOBAL_CONTROL_REGION = 'us-east-1'; +export const AWS_REGIONS = [ + 'af-south-1', + 'ap-east-1', + 'ap-east-2', + 'ap-northeast-1', + 'ap-northeast-2', + 'ap-northeast-3', + 'ap-south-1', + 'ap-south-2', + 'ap-southeast-1', + 'ap-southeast-2', + 'ap-southeast-3', + 'ap-southeast-4', + 'ap-southeast-5', + 'ap-southeast-6', + 'ap-southeast-7', + 'ca-central-1', + 'ca-west-1', + 'eu-central-1', + 'eu-central-2', + 'eu-north-1', + 'eu-south-1', + 'eu-south-2', + 'eu-west-1', + 'eu-west-2', + 'eu-west-3', + 'il-central-1', + 'me-central-1', + 'me-south-1', + 'mx-central-1', + 'sa-east-1', + 'us-east-1', + 'us-east-2', + 'us-west-1', + 'us-west-2', +] as const; + +export type AwsRegion = (typeof AWS_REGIONS)[number]; /** * Validates an AWS region string before it is used in clients or filters. @@ -39,15 +77,15 @@ const AWS_GLOBAL_CONTROL_REGION = 'us-east-1'; * @param region - AWS region to validate. * @returns The original region when valid. */ -export const assertValidAwsRegion = (region: string): string => { - if (!AWS_REGION_PATTERN.test(region)) { +export const assertValidAwsRegion = (region: string | undefined): AwsRegion => { + if (!region || !AWS_REGION_PATTERN.test(region) || !AWS_REGIONS.includes(region as AwsRegion)) { throw new AwsDiscoveryError( 'INVALID_AWS_REGION', - `Invalid AWS region '${region}'. Use a standard region name such as 'eu-central-1' or 'us-gov-west-1'.`, + `Invalid AWS region '${region ?? ''}'. Use a supported AWS region name such as 'eu-central-1' or 'us-east-1'.`, ); } - return region; + return region as AwsRegion; }; /** Creates an AWS EC2 client for a specific region. */ @@ -206,7 +244,7 @@ export const createResourceExplorerClient = (config: AwsClientConfig): ResourceE * * @returns Resolved AWS region for live discovery commands. */ -export const resolveCurrentAwsRegion = async (): Promise => { +export const resolveCurrentAwsRegion = async (): Promise => { const explicitRegion = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || process.env.aws_region; if (explicitRegion) { @@ -239,11 +277,11 @@ export const resolveAwsAccountId = async (): Promise => { * @param region - Optional preferred region for the EC2 control plane call. * @returns Region names available for Resource Explorer setup. */ -export const listEnabledAwsRegions = async (region?: string): Promise => { +export const listEnabledAwsRegions = async (region?: string): Promise => { const client = createEc2Client({ ...(region ? { region: assertValidAwsRegion(region) } : {}), }); const { Regions } = await client.send(new DescribeRegionsCommand({ AllRegions: false })); - return (Regions ?? []).flatMap((region) => (region.RegionName ? [region.RegionName] : [])); + return (Regions ?? []).flatMap((region) => (region.RegionName ? [assertValidAwsRegion(region.RegionName)] : [])); }; diff --git a/packages/sdk/src/providers/aws/discovery-registry.ts b/packages/sdk/src/providers/aws/discovery-registry.ts index fdc90ba..e0dd3b7 100644 --- a/packages/sdk/src/providers/aws/discovery-registry.ts +++ b/packages/sdk/src/providers/aws/discovery-registry.ts @@ -7,6 +7,7 @@ import { } from './resources/cloudfront.js'; import { hydrateAwsCloudTrailTrails } from './resources/cloudtrail.js'; import { + hydrateAwsCloudWatchLogGroupRecentStreamActivity, hydrateAwsCloudWatchLogGroups, hydrateAwsCloudWatchLogMetricFilterCoverage, hydrateAwsCloudWatchLogStreams, @@ -67,6 +68,16 @@ export type AwsDiscoveryDatasetLoadResult(datasetKey: K) => Promise; +}; + /** Declarative definition for one rule-facing AWS discovery dataset. */ export type AwsDiscoveryDatasetDefinition = { datasetKey: K; @@ -94,7 +105,10 @@ export type AwsDiscoveryDatasetDefinition Promise>; + load: ( + resources: AwsDiscoveredResource[], + context: AwsDiscoveryDatasetLoadContext, + ) => Promise>; }; const awsDiscoveryDatasetRegistry: { @@ -130,6 +144,12 @@ const awsDiscoveryDatasetRegistry: { service: 'cloudwatch', load: hydrateAwsCloudWatchLogGroups, }, + 'aws-cloudwatch-log-group-recent-stream-activity': { + datasetKey: 'aws-cloudwatch-log-group-recent-stream-activity', + resourceTypes: ['logs:log-group'], + service: 'cloudwatch', + load: hydrateAwsCloudWatchLogGroupRecentStreamActivity, + }, 'aws-cloudwatch-log-metric-filter-coverage': { datasetKey: 'aws-cloudwatch-log-metric-filter-coverage', resourceTypes: ['logs:log-group'], diff --git a/packages/sdk/src/providers/aws/discovery.ts b/packages/sdk/src/providers/aws/discovery.ts index ae4ef07..012b659 100644 --- a/packages/sdk/src/providers/aws/discovery.ts +++ b/packages/sdk/src/providers/aws/discovery.ts @@ -1,8 +1,14 @@ -import type { DiscoveryDatasetKey, DiscoveryDatasetMap, LiveEvaluationContext, Rule } from '@cloudburn/rules'; +import type { + AwsDiscoveredResource, + DiscoveryDatasetKey, + DiscoveryDatasetMap, + LiveEvaluationContext, + Rule, +} from '@cloudburn/rules'; import { LiveResourceBag } from '@cloudburn/rules'; +import { emitDebugLog } from '../../debug.js'; import type { AwsDiscoveryInitialization, - AwsDiscoveryRegion, AwsDiscoveryStatus, AwsDiscoveryTarget, AwsSupportedResourceType, @@ -196,6 +202,26 @@ const normalizeDatasetLoadResult = ( resources: loadResult.resources, }; +const formatElapsedMs = (startedAtMs: number): string => `${Math.max(0, Date.now() - startedAtMs)}ms`; + +const buildResourcesByTypeIndex = (resources: AwsDiscoveredResource[]): Map => { + const resourcesByType = new Map(); + + for (const resource of resources) { + const typedResources = resourcesByType.get(resource.resourceType) ?? []; + typedResources.push(resource); + resourcesByType.set(resource.resourceType, typedResources); + } + + return resourcesByType; +}; + +const appendItems = (target: T[], items: Iterable): void => { + for (const item of items) { + target.push(item); + } +}; + /** * Discovers AWS resources for live rule evaluation using Resource Explorer and * registry-driven discovery datasets. @@ -207,8 +233,13 @@ const normalizeDatasetLoadResult = ( export const discoverAwsResources = async ( rules: Rule[], target: AwsDiscoveryTarget, + options?: { debugLogger?: (message: string) => void }, ): Promise => { const datasetKeys = collectDiscoveryDependencies(rules); + emitDebugLog( + options?.debugLogger, + `aws: resolved discovery datasets ${datasetKeys.length === 0 ? 'none' : datasetKeys.join(', ')}`, + ); if (datasetKeys.length === 0) { return { @@ -234,88 +265,189 @@ export const discoverAwsResources = async ( const resourceTypes = sortUnique( datasetDefinitions.flatMap((definition) => definition.resourceTypes.map(assertValidResourceExplorerResourceType)), ); + emitDebugLog( + options?.debugLogger, + `aws: resolved Resource Explorer resource types ${resourceTypes.length === 0 ? 'none' : resourceTypes.join(', ')}`, + ); const catalog = resourceTypes.length > 0 - ? await buildAwsDiscoveryCatalog(target, resourceTypes) + ? options?.debugLogger === undefined + ? await buildAwsDiscoveryCatalog(target, resourceTypes) + : await buildAwsDiscoveryCatalog(target, resourceTypes, { debugLogger: options.debugLogger }) : { indexType: 'LOCAL' as const, resources: [], searchRegion: await resolveCurrentAwsRegion(), }; - const datasetLoads = await Promise.all( - datasetDefinitions.map(async (definition) => { - if (definition.resourceTypes.length === 0) { - const loadResult = normalizeDatasetLoadResult(await definition.load([])); + emitDebugLog( + options?.debugLogger, + `aws: catalog ready with ${catalog.resources.length} resources from ${catalog.searchRegion}`, + ); + const resourcesByType = buildResourcesByTypeIndex(catalog.resources); + const datasetLoadPromises = new Map< + DiscoveryDatasetKey, + Promise<{ + dataset: [DiscoveryDatasetKey, DiscoveryDatasetMap[DiscoveryDatasetKey]]; + diagnostics: ScanDiagnostic[]; + }> + >(); + const loadDataset = ( + datasetKey: K, + ): Promise<{ + dataset: [K, DiscoveryDatasetMap[K]]; + diagnostics: ScanDiagnostic[]; + }> => { + const cachedLoad = datasetLoadPromises.get(datasetKey); + + if (cachedLoad) { + return cachedLoad as Promise<{ + dataset: [K, DiscoveryDatasetMap[K]]; + diagnostics: ScanDiagnostic[]; + }>; + } - return { - dataset: [definition.datasetKey, loadResult.resources] as const, - diagnostics: loadResult.diagnostics, - }; - } + const definition = getAwsDiscoveryDatasetDefinition(datasetKey); - const matchingResources = catalog.resources.filter((resource) => - definition.resourceTypes.includes(resource.resourceType), - ); - const resourcesByRegion = groupResourcesByRegion(matchingResources); - const loadedResources: unknown[] = []; - const diagnostics: ScanDiagnostic[] = []; - - for (const [region, regionResources] of resourcesByRegion) { - try { - const loadResult = normalizeDatasetLoadResult(await definition.load(regionResources)); - loadedResources.push(...loadResult.resources); - diagnostics.push(...loadResult.diagnostics); - } catch (err) { - if (!isAwsAccessDeniedError(err)) { - throw err; - } + if (!definition) { + throw new Error(`Unknown discovery dataset '${datasetKey}'.`); + } + + const startedAtMs = Date.now(); + const loadPromise = (async () => { + emitDebugLog(options?.debugLogger, `aws: loading dataset ${definition.datasetKey}`); - diagnostics.push({ - code: getAwsErrorCode(err), - details: err instanceof Error ? err.message : String(err), - message: buildAccessDeniedDiagnosticMessage(definition.service, region, err), - provider: 'aws', - region, - service: definition.service, - source: 'discovery', - status: 'access_denied', - }); + try { + if (definition.resourceTypes.length === 0) { + const loadResult = normalizeDatasetLoadResult( + await definition.load([], { + loadDataset: async ( + nestedDatasetKey: T, + ): Promise => (await loadDataset(nestedDatasetKey)).dataset[1], + }), + ); + emitDebugLog( + options?.debugLogger, + `aws: completed dataset ${definition.datasetKey} with ${loadResult.resources.length} resources in ${formatElapsedMs(startedAtMs)}`, + ); + + return { + dataset: [definition.datasetKey, loadResult.resources as DiscoveryDatasetMap[K]] as [ + K, + DiscoveryDatasetMap[K], + ], + diagnostics: loadResult.diagnostics, + }; } + + const matchingResources = definition.resourceTypes.flatMap( + (resourceType) => resourcesByType.get(resourceType) ?? [], + ); + const regionResourceGroups = groupResourcesByRegion(matchingResources); + const loadedResources: unknown[] = []; + const diagnostics: ScanDiagnostic[] = []; + + for (const [region, regionResources] of regionResourceGroups) { + const regionStartedAtMs = Date.now(); + + try { + emitDebugLog( + options?.debugLogger, + `aws: loading dataset ${definition.datasetKey} in ${region} from ${regionResources.length} resources`, + ); + const loadResult = normalizeDatasetLoadResult( + await definition.load(regionResources, { + loadDataset: async ( + nestedDatasetKey: T, + ): Promise => (await loadDataset(nestedDatasetKey)).dataset[1], + }), + ); + appendItems(loadedResources, loadResult.resources); + appendItems(diagnostics, loadResult.diagnostics); + emitDebugLog( + options?.debugLogger, + `aws: completed dataset ${definition.datasetKey} in ${region} with ${loadResult.resources.length} resources in ${formatElapsedMs(regionStartedAtMs)}`, + ); + } catch (err) { + if (!isAwsAccessDeniedError(err)) { + emitDebugLog( + options?.debugLogger, + `aws: dataset ${definition.datasetKey} failed in ${region} after ${formatElapsedMs(regionStartedAtMs)}: ${err instanceof Error ? err.message : String(err)}`, + ); + throw err; + } + + diagnostics.push({ + code: getAwsErrorCode(err), + details: err instanceof Error ? err.message : String(err), + message: buildAccessDeniedDiagnosticMessage(definition.service, region, err), + provider: 'aws', + region, + service: definition.service, + source: 'discovery', + status: 'access_denied', + }); + emitDebugLog( + options?.debugLogger, + `aws: completed dataset ${definition.datasetKey} in ${region} with 0 resources in ${formatElapsedMs(regionStartedAtMs)}`, + ); + } + } + + emitDebugLog( + options?.debugLogger, + `aws: completed dataset ${definition.datasetKey} with ${loadedResources.length} resources in ${formatElapsedMs(startedAtMs)}`, + ); + + return { + dataset: [definition.datasetKey, loadedResources as DiscoveryDatasetMap[K]] as [K, DiscoveryDatasetMap[K]], + diagnostics, + }; + } catch (err) { + emitDebugLog( + options?.debugLogger, + `aws: dataset ${definition.datasetKey} failed after ${formatElapsedMs(startedAtMs)}: ${err instanceof Error ? err.message : String(err)}`, + ); + throw err; } + })(); + + datasetLoadPromises.set( + datasetKey, + loadPromise as Promise<{ + dataset: [DiscoveryDatasetKey, DiscoveryDatasetMap[DiscoveryDatasetKey]]; + diagnostics: ScanDiagnostic[]; + }>, + ); - return { - dataset: [definition.datasetKey, loadedResources] as const, - diagnostics, - }; - }), - ); + return loadPromise; + }; + const datasetLoads = await Promise.all(datasetKeys.map((datasetKey) => loadDataset(datasetKey))); + const allDatasetLoads = await Promise.all(datasetLoadPromises.values()); const resources = new LiveResourceBag( Object.fromEntries(datasetLoads.map((loadResult) => loadResult.dataset)) as Partial, ); return { catalog, - diagnostics: datasetLoads.flatMap((loadResult) => loadResult.diagnostics), + diagnostics: allDatasetLoads.flatMap((loadResult) => loadResult.diagnostics), resources, }; }; -/** - * Lists all AWS regions with an enabled Resource Explorer index. - * - * @returns Enabled local and aggregator index regions. - */ -export const listEnabledAwsDiscoveryRegions = async (): Promise => listAwsDiscoveryIndexes(); - /** * Retrieves observed Resource Explorer status across all enabled AWS regions. * * @param region - Optional explicit region to use as the preferred control region. * @returns Observed discovery status across the account. */ -export const getAwsDiscoveryStatus = async (region?: string): Promise => { +export const getAwsDiscoveryStatus = async ( + region?: string, + debugLogger?: (message: string) => void, +): Promise => { const selectedRegion = region ? assertValidAwsRegion(region) : await resolveCurrentAwsRegion(); + emitDebugLog(debugLogger, `aws: collecting discovery status from control region ${selectedRegion}`); const enabledRegions = await listEnabledAwsRegions(selectedRegion); + emitDebugLog(debugLogger, `aws: inspecting discovery status across ${enabledRegions.length} enabled regions`); const statuses = await Promise.all(enabledRegions.map((enabledRegion) => getAwsDiscoveryRegionStatus(enabledRegion))); const orderedStatuses = [...statuses].sort((left, right) => left.region.localeCompare(right.region)); const indexedRegionCount = orderedStatuses.filter((status) => status.status === 'indexed').length; @@ -343,11 +475,16 @@ export const getAwsDiscoveryStatus = async (region?: string): Promise => { +export const initializeAwsDiscovery = async ( + region?: string, + debugLogger?: (message: string) => void, +): Promise => { const explicitRegionRequested = region !== undefined; const selectedRegion = region ? assertValidAwsRegion(region) : await resolveCurrentAwsRegion(); - const observedStatus = await getAwsDiscoveryStatus(selectedRegion); + emitDebugLog(debugLogger, `aws: initializing discovery from control region ${selectedRegion}`); + const observedStatus = await getAwsDiscoveryStatus(selectedRegion, debugLogger); const enabledRegions = await listEnabledAwsRegions(selectedRegion); + emitDebugLog(debugLogger, `aws: found ${enabledRegions.length} enabled regions for initialization`); const indexes = await listAwsDiscoveryIndexes(selectedRegion); const beforeIndexedRegions = new Set(indexes.map((index) => index.region)); const aggregator = indexes.find((index) => index.type === 'aggregator'); @@ -382,7 +519,7 @@ export const initializeAwsDiscovery = async (region?: string): Promise status.region === selectedRegion && status.status === 'indexed')?.region ?? selectedRegion; @@ -480,7 +617,7 @@ export const initializeAwsDiscovery = async (region?: string): Promise => { +const sortUniqueStrings = (values: T[]): T[] => + [...new Set(values)].sort((left, right) => left.localeCompare(right)) as T[]; + +const resolveAggregatorSearchPlan = async (requestedRegions?: AwsRegion[]): Promise => { const { accessibleIndexedRegions, aggregatorRegion, sawDeniedRegion } = await findAccessibleAggregatorRegion(); + const selectedRegions: AwsRegion[] = requestedRegions + ? sortUniqueStrings(requestedRegions) + : (accessibleIndexedRegions as AwsRegion[]); if (aggregatorRegion) { + const missingRegions = selectedRegions.filter((region) => !accessibleIndexedRegions.includes(region)); + + if (missingRegions.length > 0) { + throw new AwsDiscoveryError( + 'RESOURCE_EXPLORER_REGION_NOT_ENABLED', + `AWS Resource Explorer is not enabled in ${missingRegions[0]}. Enable it first: ${RESOURCE_EXPLORER_SETUP_DOCS_URL} or run 'cloudburn discover init'.`, + ); + } + return { searchRegion: aggregatorRegion, indexType: 'AGGREGATOR', - regionFilters: accessibleIndexedRegions, + regionFilters: selectedRegions, }; } @@ -347,12 +374,88 @@ const resolveAggregatorSearchPlan = async (): Promise => { ); }; -const buildFilterString = (resourceType: string, regionFilter?: string): string => { - if (!regionFilter) { - return `resourcetype:${resourceType}`; +const buildFilterString = (resourceTypes: string[], regionFilters?: AwsRegion[]): string => { + const normalizedResourceTypes = sortUniqueStrings(resourceTypes); + + if (normalizedResourceTypes.length === 0) { + throw new Error('At least one Resource Explorer resource type is required.'); + } + + const segments = [`resourcetype:${normalizedResourceTypes.join(',')}`]; + + if (regionFilters && regionFilters.length > 0) { + segments.push(`region:${sortUniqueStrings(regionFilters).map(assertValidAwsRegion).join(',')}`); + } + + return segments.join(' '); +}; + +const chunkValuesByFilterLength = (values: T[], buildCandidate: (batch: T[]) => string): T[][] => { + const batches: T[][] = []; + let currentBatch: T[] = []; + + for (const value of values) { + const nextBatch = [...currentBatch, value]; + + if (buildCandidate(nextBatch).length <= RESOURCE_EXPLORER_FILTER_STRING_MAX_LENGTH) { + currentBatch = nextBatch; + continue; + } + + if (currentBatch.length === 0) { + throw new Error(`Resource Explorer filter value '${value}' exceeds the maximum filter length.`); + } + + batches.push(currentBatch); + currentBatch = [value]; + } + + if (currentBatch.length > 0) { + batches.push(currentBatch); + } + + return batches; +}; + +const planListResourcesQueries = (resourceTypes: string[], regionFilters?: AwsRegion[]): ListResourcesQueryPlan[] => { + const normalizedResourceTypes = sortUniqueStrings(resourceTypes); + const normalizedRegionFilters = regionFilters ? (sortUniqueStrings(regionFilters) as AwsRegion[]) : undefined; + + if (!normalizedRegionFilters || normalizedRegionFilters.length === 0) { + return chunkValuesByFilterLength(normalizedResourceTypes, (resourceTypeBatch) => + buildFilterString(resourceTypeBatch), + ).map((resourceTypeBatch) => ({ + resourceTypes: resourceTypeBatch, + })); } - return `resourcetype:${resourceType} region:${assertValidAwsRegion(regionFilter)}`; + let regionBatches: AwsRegion[][]; + + try { + regionBatches = chunkValuesByFilterLength(normalizedRegionFilters, (regionBatch) => + buildFilterString(normalizedResourceTypes, regionBatch), + ); + } catch { + regionBatches = normalizedRegionFilters.map((region) => [region]); + } + + return regionBatches.flatMap((regionBatch) => { + if (buildFilterString(normalizedResourceTypes, regionBatch).length <= RESOURCE_EXPLORER_FILTER_STRING_MAX_LENGTH) { + return [ + { + resourceTypes: normalizedResourceTypes, + regionFilters: regionBatch, + } satisfies ListResourcesQueryPlan, + ]; + } + + return chunkValuesByFilterLength(normalizedResourceTypes, (resourceTypeBatch) => + buildFilterString(resourceTypeBatch, regionBatch), + ).map((resourceTypeBatch) => ({ + resourceTypes: resourceTypeBatch, + regionFilters: regionBatch, + })); + }); }; const resolveSearchViewArn = async (searchRegion: string): Promise => { @@ -573,49 +676,91 @@ export const waitForAwsResourceExplorerSetup = async ( export const buildAwsDiscoveryCatalog = async ( target: AwsDiscoveryTarget, resourceTypes: string[], + options?: { debugLogger?: (message: string) => void }, ): Promise => { - const currentRegion = - target.mode === 'region' ? assertValidAwsRegion(target.region) : await resolveCurrentAwsRegion(); - const searchPlan = - target.mode === 'all' - ? await resolveAggregatorSearchPlan() - : await resolveRegionalSearchPlan(target.mode === 'region' ? target.region : currentRegion); + const currentRegion = await resolveCurrentAwsRegion(); + let searchPlan: SearchPlan; + + if (target.mode === 'regions') { + if (target.regions.length === 1) { + const [requestedRegion] = target.regions; + searchPlan = await resolveRegionalSearchPlan(assertValidAwsRegion(requestedRegion)); + } else { + searchPlan = await resolveAggregatorSearchPlan(target.regions); + } + } else { + searchPlan = await resolveRegionalSearchPlan(currentRegion); + } + emitDebugLog( + options?.debugLogger, + `aws: Resource Explorer using ${searchPlan.indexType.toLowerCase()} control plane ${searchPlan.searchRegion}${ + searchPlan.regionFilters ? ` for regions ${searchPlan.regionFilters.join(', ')}` : '' + }`, + ); const client = createResourceExplorerClient({ region: searchPlan.searchRegion }); const viewArn = await resolveSearchViewArn(searchPlan.searchRegion); const resourcesByArn = new Map(); - const regionFilters = searchPlan.regionFilters?.length ? searchPlan.regionFilters : [undefined]; + const queryPlans = planListResourcesQueries(resourceTypes, searchPlan.regionFilters); + emitDebugLog( + options?.debugLogger, + `aws: planned ${queryPlans.length} Resource Explorer quer${queryPlans.length === 1 ? 'y' : 'ies'} for ${resourceTypes.length} resource types`, + ); - for (const resourceType of resourceTypes) { - for (const regionFilter of regionFilters) { - let nextToken: string | undefined; + for (const [queryIndex, queryPlan] of queryPlans.entries()) { + let nextToken: string | undefined; + let page = 1; + const filterString = buildFilterString(queryPlan.resourceTypes, queryPlan.regionFilters); - do { - const response = await client - .send( + do { + emitDebugLog( + options?.debugLogger, + `aws: Resource Explorer query ${queryIndex + 1}/${queryPlans.length} page ${page} filter="${filterString}"`, + ); + const response = await withAwsServiceErrorContext( + 'AWS Resource Explorer', + 'ListResources', + searchPlan.searchRegion, + () => + client.send( new ListResourcesCommand({ Filters: { - FilterString: buildFilterString(resourceType, regionFilter), + FilterString: filterString, }, - MaxResults: 100, + MaxResults: RESOURCE_EXPLORER_LIST_RESOURCES_MAX_RESULTS, NextToken: nextToken, ViewArn: viewArn, }), - ) - .catch((err: unknown) => throwResourceExplorerOperationError(err, 'ListResources', searchPlan.searchRegion)); + ), + { + initialDelayMs: RESOURCE_EXPLORER_LIST_RESOURCES_INITIAL_DELAY_MS, + maxAttempts: RESOURCE_EXPLORER_LIST_RESOURCES_MAX_ATTEMPTS, + onRetry: ({ attempt, delayMs, maxAttempts: retryMaxAttempts }) => { + emitDebugLog( + options?.debugLogger, + `aws: retrying throttled Resource Explorer ListResources attempt ${attempt + 1}/${retryMaxAttempts} after ${delayMs}ms`, + ); + }, + }, + ); - for (const resource of response.Resources ?? []) { - const normalized = mapResource(resource); + for (const resource of response.Resources ?? []) { + const normalized = mapResource(resource); - if (normalized) { - resourcesByArn.set(normalized.arn, normalized); - } + if (normalized) { + resourcesByArn.set(normalized.arn, normalized); } + } - nextToken = response.NextToken; - } while (nextToken); - } + nextToken = response.NextToken; + page += 1; + } while (nextToken); } + emitDebugLog( + options?.debugLogger, + `aws: Resource Explorer catalog collected ${resourcesByArn.size} unique resources`, + ); + return { resources: [...resourcesByArn.values()].sort((left, right) => left.arn.localeCompare(right.arn)), searchRegion: searchPlan.searchRegion, diff --git a/packages/sdk/src/providers/aws/resources/cloudfront.ts b/packages/sdk/src/providers/aws/resources/cloudfront.ts index 16c2edc..002d7dd 100644 --- a/packages/sdk/src/providers/aws/resources/cloudfront.ts +++ b/packages/sdk/src/providers/aws/resources/cloudfront.ts @@ -5,6 +5,7 @@ import type { AwsDiscoveredResource, } from '@cloudburn/rules'; import { createCloudFrontClient, resolveAwsAccountId } from '../client.js'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { chunkItems, extractTerminalArnResourceIdentifier, withAwsServiceErrorContext } from './utils.js'; @@ -128,8 +129,11 @@ export const hydrateAwsCloudFrontDistributions = async ( */ export const hydrateAwsCloudFrontDistributionRequestActivity = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const distributions = await hydrateAwsCloudFrontDistributions(resources); + const distributions = context + ? await context.loadDataset('aws-cloudfront-distributions') + : await hydrateAwsCloudFrontDistributions(resources); if (distributions.length === 0) { return []; diff --git a/packages/sdk/src/providers/aws/resources/cloudwatch-logs.ts b/packages/sdk/src/providers/aws/resources/cloudwatch-logs.ts index 159cbd5..09d3b83 100644 --- a/packages/sdk/src/providers/aws/resources/cloudwatch-logs.ts +++ b/packages/sdk/src/providers/aws/resources/cloudwatch-logs.ts @@ -5,14 +5,16 @@ import { } from '@aws-sdk/client-cloudwatch-logs'; import type { AwsCloudWatchLogGroup, + AwsCloudWatchLogGroupRecentStreamActivity, AwsCloudWatchLogMetricFilterCoverage, AwsCloudWatchLogStream, AwsDiscoveredResource, } from '@cloudburn/rules'; import { createCloudWatchLogsClient } from '../client.js'; -import { withAwsServiceErrorContext } from './utils.js'; +import { chunkItems, withAwsServiceErrorContext } from './utils.js'; const CLOUDWATCH_LOG_GROUP_ARN_PATTERN = /^arn:[^:]+:logs:[^:]+:[^:]+:log-group:(.+)$/u; +const CLOUDWATCH_LOG_STREAM_ACTIVITY_CONCURRENCY = 10; const extractAccountIdFromArn = (arn: string): string | null => { const arnSegments = arn.split(':'); @@ -194,6 +196,84 @@ export const hydrateAwsCloudWatchLogStreams = async ( return hydratedPages.flat().sort((left, right) => left.arn.localeCompare(right.arn)); }; +/** + * Hydrates discovered CloudWatch log groups with only the latest known stream activity summary. + * + * This avoids enumerating every stream when rules only need recency at log-group scope. + * + * @param resources - Catalog resources filtered to CloudWatch Logs log groups. + * @returns One latest-stream activity summary per discovered log group. + */ +export const hydrateAwsCloudWatchLogGroupRecentStreamActivity = async ( + resources: AwsDiscoveredResource[], +): Promise => { + const resourcesByRegion = new Map(); + + for (const resource of resources) { + const logGroupName = extractLogGroupName(resource.arn); + + if (!logGroupName) { + continue; + } + + const regionResources = resourcesByRegion.get(resource.region) ?? []; + regionResources.push(resource); + resourcesByRegion.set(resource.region, regionResources); + } + + const hydratedPages = await Promise.all( + [...resourcesByRegion.entries()].map(async ([region, regionResources]) => { + const client = createCloudWatchLogsClient({ region }); + const desiredLogGroups = new Map( + regionResources.flatMap((resource) => { + const logGroupName = extractLogGroupName(resource.arn); + + return logGroupName ? [[logGroupName, resource.accountId] as const] : []; + }), + ); + const activity: AwsCloudWatchLogGroupRecentStreamActivity[] = []; + + for (const batch of chunkItems([...desiredLogGroups.entries()], CLOUDWATCH_LOG_STREAM_ACTIVITY_CONCURRENCY)) { + const hydratedBatch = await Promise.all( + batch.map(async ([logGroupName, discoveredAccountId]) => { + const response = await withAwsServiceErrorContext( + 'Amazon CloudWatch Logs', + 'DescribeLogStreams', + region, + () => + client.send( + new DescribeLogStreamsCommand({ + descending: true, + limit: 1, + logGroupName, + orderBy: 'LastEventTime', + }), + ), + ); + const latestStream = response.logStreams?.[0]; + + return { + accountId: (latestStream?.arn ? extractAccountIdFromArn(latestStream.arn) : null) ?? discoveredAccountId, + lastEventTimestamp: latestStream?.lastEventTimestamp, + lastIngestionTime: latestStream?.lastIngestionTime, + latestStreamArn: latestStream?.arn, + latestStreamName: latestStream?.logStreamName, + logGroupName, + region, + } satisfies AwsCloudWatchLogGroupRecentStreamActivity; + }), + ); + + activity.push(...hydratedBatch); + } + + return activity; + }), + ); + + return hydratedPages.flat().sort((left, right) => left.logGroupName.localeCompare(right.logGroupName)); +}; + /** * Hydrates discovered CloudWatch log groups with their metric-filter counts. * diff --git a/packages/sdk/src/providers/aws/resources/dynamodb.ts b/packages/sdk/src/providers/aws/resources/dynamodb.ts index 9f290aa..e75e0cf 100644 --- a/packages/sdk/src/providers/aws/resources/dynamodb.ts +++ b/packages/sdk/src/providers/aws/resources/dynamodb.ts @@ -7,6 +7,7 @@ import type { AwsDynamoDbTableUtilization, } from '@cloudburn/rules'; import { createApplicationAutoScalingClient, createDynamoDbClient } from '../client.js'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { chunkItems, extractTerminalArnResourceIdentifier, withAwsServiceErrorContext } from './utils.js'; @@ -192,8 +193,9 @@ export const hydrateAwsDynamoDbAutoscaling = async ( */ export const hydrateAwsDynamoDbTableUtilization = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const tables = await hydrateAwsDynamoDbTables(resources); + const tables = context ? await context.loadDataset('aws-dynamodb-tables') : await hydrateAwsDynamoDbTables(resources); const tablesByRegion = new Map(); for (const table of tables) { diff --git a/packages/sdk/src/providers/aws/resources/ec2-utilization.ts b/packages/sdk/src/providers/aws/resources/ec2-utilization.ts index ddbeebd..de5620d 100644 --- a/packages/sdk/src/providers/aws/resources/ec2-utilization.ts +++ b/packages/sdk/src/providers/aws/resources/ec2-utilization.ts @@ -1,4 +1,5 @@ import type { AwsDiscoveredResource, AwsEc2InstanceUtilization } from '@cloudburn/rules'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { hydrateAwsEc2Instances } from './ec2.js'; @@ -17,8 +18,9 @@ const toIsoDate = (timestamp: string): string => timestamp.slice(0, 10); */ export const hydrateAwsEc2InstanceUtilization = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const instances = await hydrateAwsEc2Instances(resources); + const instances = context ? await context.loadDataset('aws-ec2-instances') : await hydrateAwsEc2Instances(resources); const instancesByRegion = new Map(); for (const instance of instances) { diff --git a/packages/sdk/src/providers/aws/resources/ecs-cluster-metrics.ts b/packages/sdk/src/providers/aws/resources/ecs-cluster-metrics.ts index 3b3b152..6b5ca33 100644 --- a/packages/sdk/src/providers/aws/resources/ecs-cluster-metrics.ts +++ b/packages/sdk/src/providers/aws/resources/ecs-cluster-metrics.ts @@ -1,4 +1,5 @@ import type { AwsDiscoveredResource, AwsEcsClusterMetric } from '@cloudburn/rules'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { hydrateAwsEcsClusters } from './ecs.js'; @@ -14,8 +15,9 @@ const REQUIRED_ECS_DAILY_POINTS = FOURTEEN_DAYS_IN_SECONDS / DAILY_PERIOD_IN_SEC */ export const hydrateAwsEcsClusterMetrics = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const clusters = await hydrateAwsEcsClusters(resources); + const clusters = context ? await context.loadDataset('aws-ecs-clusters') : await hydrateAwsEcsClusters(resources); const clustersByRegion = new Map(); for (const cluster of clusters) { diff --git a/packages/sdk/src/providers/aws/resources/elasticache.ts b/packages/sdk/src/providers/aws/resources/elasticache.ts index 5c8e63c..e7c4b53 100644 --- a/packages/sdk/src/providers/aws/resources/elasticache.ts +++ b/packages/sdk/src/providers/aws/resources/elasticache.ts @@ -6,6 +6,7 @@ import type { AwsElastiCacheReservedNode, } from '@cloudburn/rules'; import { createElastiCacheClient } from '../client.js'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { extractTerminalResourceIdentifier, withAwsServiceErrorContext } from './utils.js'; @@ -179,8 +180,11 @@ export const hydrateAwsElastiCacheReservedNodes = async ( */ export const hydrateAwsElastiCacheClusterActivity = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const clusters = await hydrateAwsElastiCacheClusters(resources); + const clusters = context + ? await context.loadDataset('aws-elasticache-clusters') + : await hydrateAwsElastiCacheClusters(resources); const clustersByRegion = new Map(); for (const cluster of clusters) { diff --git a/packages/sdk/src/providers/aws/resources/emr.ts b/packages/sdk/src/providers/aws/resources/emr.ts index f89ad93..6c04c44 100644 --- a/packages/sdk/src/providers/aws/resources/emr.ts +++ b/packages/sdk/src/providers/aws/resources/emr.ts @@ -1,6 +1,7 @@ import { DescribeClusterCommand, ListInstancesCommand } from '@aws-sdk/client-emr'; import type { AwsDiscoveredResource, AwsEmrCluster, AwsEmrClusterMetric } from '@cloudburn/rules'; import { createEmrClient } from '../client.js'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { extractTerminalArnResourceIdentifier, withAwsServiceErrorContext } from './utils.js'; @@ -124,8 +125,9 @@ const listEmrClusterInstanceTypes = async ( */ export const hydrateAwsEmrClusterMetrics = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const clusters = await hydrateAwsEmrClusters(resources); + const clusters = context ? await context.loadDataset('aws-emr-clusters') : await hydrateAwsEmrClusters(resources); const clustersByRegion = new Map(); for (const cluster of clusters) { diff --git a/packages/sdk/src/providers/aws/resources/lambda.ts b/packages/sdk/src/providers/aws/resources/lambda.ts index 8621655..e7bb8dc 100644 --- a/packages/sdk/src/providers/aws/resources/lambda.ts +++ b/packages/sdk/src/providers/aws/resources/lambda.ts @@ -1,6 +1,7 @@ import { GetFunctionConfigurationCommand } from '@aws-sdk/client-lambda'; import type { AwsDiscoveredResource, AwsLambdaFunction, AwsLambdaFunctionMetric } from '@cloudburn/rules'; import { createLambdaClient } from '../client.js'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { chunkItems, withAwsServiceErrorContext } from './utils.js'; @@ -96,8 +97,11 @@ export const hydrateAwsLambdaFunctions = async (resources: AwsDiscoveredResource */ export const hydrateAwsLambdaFunctionMetrics = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const functions = await hydrateAwsLambdaFunctions(resources); + const functions = context + ? await context.loadDataset('aws-lambda-functions') + : await hydrateAwsLambdaFunctions(resources); const functionsByRegion = new Map(); for (const fn of functions) { diff --git a/packages/sdk/src/providers/aws/resources/rds-activity.ts b/packages/sdk/src/providers/aws/resources/rds-activity.ts index f674783..54a8e75 100644 --- a/packages/sdk/src/providers/aws/resources/rds-activity.ts +++ b/packages/sdk/src/providers/aws/resources/rds-activity.ts @@ -1,4 +1,5 @@ import type { AwsDiscoveredResource, AwsRdsInstanceActivity, AwsRdsInstanceCpuMetric } from '@cloudburn/rules'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { hydrateAwsRdsInstances } from './rds.js'; @@ -18,8 +19,9 @@ const REQUIRED_RDS_DAILY_CPU_POINTS = THIRTY_DAYS_IN_SECONDS / DAILY_PERIOD_IN_S */ export const hydrateAwsRdsInstanceActivity = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const instances = await hydrateAwsRdsInstances(resources); + const instances = context ? await context.loadDataset('aws-rds-instances') : await hydrateAwsRdsInstances(resources); const instancesByRegion = new Map(); for (const instance of instances) { @@ -74,8 +76,9 @@ export const hydrateAwsRdsInstanceActivity = async ( */ export const hydrateAwsRdsInstanceCpuMetrics = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const instances = await hydrateAwsRdsInstances(resources); + const instances = context ? await context.loadDataset('aws-rds-instances') : await hydrateAwsRdsInstances(resources); const instancesByRegion = new Map(); for (const instance of instances) { diff --git a/packages/sdk/src/providers/aws/resources/redshift.ts b/packages/sdk/src/providers/aws/resources/redshift.ts index a7755ac..1923a84 100644 --- a/packages/sdk/src/providers/aws/resources/redshift.ts +++ b/packages/sdk/src/providers/aws/resources/redshift.ts @@ -11,6 +11,7 @@ import type { } from '@cloudburn/rules'; import type { ScanDiagnostic } from '../../../types.js'; import { createRedshiftClient } from '../client.js'; +import type { AwsDiscoveryDatasetLoadContext } from '../discovery-registry.js'; import { getAwsErrorCode, isAwsAccessDeniedError } from '../errors.js'; import { fetchCloudWatchSignals } from './cloudwatch.js'; import { chunkItems, extractTerminalResourceIdentifier, withAwsServiceErrorContext } from './utils.js'; @@ -133,8 +134,11 @@ export const hydrateAwsRedshiftClusters = async ( */ export const hydrateAwsRedshiftClusterMetrics = async ( resources: AwsDiscoveredResource[], + context?: AwsDiscoveryDatasetLoadContext, ): Promise => { - const { resources: clusters } = await hydrateAwsRedshiftClusters(resources); + const clusters = context + ? await context.loadDataset('aws-redshift-clusters') + : (await hydrateAwsRedshiftClusters(resources)).resources; const clustersByRegion = new Map(); for (const cluster of clusters) { diff --git a/packages/sdk/src/providers/aws/resources/utils.ts b/packages/sdk/src/providers/aws/resources/utils.ts index f309987..4c0e630 100644 --- a/packages/sdk/src/providers/aws/resources/utils.ts +++ b/packages/sdk/src/providers/aws/resources/utils.ts @@ -3,6 +3,7 @@ import { AwsDiscoveryError, isAwsThrottlingError, wrapAwsServiceError } from '.. type AwsServiceErrorContextOptions = { initialDelayMs?: number; maxAttempts?: number; + onRetry?: (details: { attempt: number; delayMs: number; error: unknown; maxAttempts: number }) => void; passthrough?: (err: unknown) => boolean; }; @@ -90,7 +91,9 @@ export const withAwsServiceErrorContext = async ( } if (attempt < maxAttempts && isAwsThrottlingError(err)) { - await sleep(initialDelayMs * 2 ** (attempt - 1)); + const delayMs = initialDelayMs * 2 ** (attempt - 1); + options.onRetry?.({ attempt, delayMs, error: err, maxAttempts }); + await sleep(delayMs); continue; } diff --git a/packages/sdk/src/scanner.ts b/packages/sdk/src/scanner.ts index 5a56f1e..4448f84 100644 --- a/packages/sdk/src/scanner.ts +++ b/packages/sdk/src/scanner.ts @@ -1,16 +1,15 @@ import { loadConfig } from './config/loader.js'; import { mergeConfig } from './config/merge.js'; +import { emitDebugLog } from './debug.js'; import { runLiveScan } from './engine/run-live.js'; import { runStaticScan } from './engine/run-static.js'; import { getAwsDiscoveryStatus, initializeAwsDiscovery, - listEnabledAwsDiscoveryRegions, listSupportedAwsResourceTypes, } from './providers/aws/discovery.js'; import type { AwsDiscoveryInitialization, - AwsDiscoveryRegion, AwsDiscoveryStatus, AwsDiscoveryTarget, AwsSupportedResourceType, @@ -22,6 +21,8 @@ import type { * High-level SDK facade for CloudBurn scans and config loading. */ export class CloudBurnClient { + public constructor(private readonly options?: { debugLogger?: (message: string) => void }) {} + /** * Merges runtime config overrides onto the loaded CloudBurn config. * @@ -30,7 +31,12 @@ export class CloudBurnClient { * @returns The merged effective config for the requested operation. */ private async getEffectiveConfig(config?: Partial, configPath?: string): Promise { + emitDebugLog( + this.options?.debugLogger, + `sdk: loading config${configPath ? ` from ${configPath}` : ' from default search path'}`, + ); const loadedConfig = await this.loadConfig(configPath); + emitDebugLog(this.options?.debugLogger, 'sdk: merged runtime config overrides'); return mergeConfig(config, loadedConfig); } @@ -51,6 +57,7 @@ export class CloudBurnClient { config?: Partial, options?: { configPath?: string }, ): Promise { + emitDebugLog(this.options?.debugLogger, `sdk: starting static scan for ${path}`); const effectiveConfig = await this.getEffectiveConfig(config, options?.configPath); return runStaticScan(path, effectiveConfig); @@ -67,18 +74,12 @@ export class CloudBurnClient { config?: Partial; configPath?: string; }): Promise { + emitDebugLog(this.options?.debugLogger, 'sdk: starting live discovery scan'); const effectiveConfig = await this.getEffectiveConfig(options?.config, options?.configPath); - return runLiveScan(effectiveConfig, options?.target ?? { mode: 'current' }); - } - - /** - * Lists all AWS regions with an enabled Resource Explorer index. - * - * @returns Enabled local and aggregator index regions. - */ - public async listEnabledDiscoveryRegions(): Promise { - return listEnabledAwsDiscoveryRegions(); + return runLiveScan(effectiveConfig, options?.target ?? { mode: 'current' }, { + debugLogger: this.options?.debugLogger, + }); } /** @@ -88,7 +89,11 @@ export class CloudBurnClient { * @returns The observed discovery status. */ public async getDiscoveryStatus(options?: { region?: string }): Promise { - return getAwsDiscoveryStatus(options?.region); + emitDebugLog(this.options?.debugLogger, 'sdk: requesting discovery status'); + + return this.options?.debugLogger === undefined + ? getAwsDiscoveryStatus(options?.region) + : getAwsDiscoveryStatus(options?.region, this.options.debugLogger); } /** @@ -98,7 +103,11 @@ export class CloudBurnClient { * @returns The initialization result. */ public async initializeDiscovery(options?: { region?: string }): Promise { - return initializeAwsDiscovery(options?.region); + emitDebugLog(this.options?.debugLogger, 'sdk: initializing discovery'); + + return this.options?.debugLogger === undefined + ? initializeAwsDiscovery(options?.region) + : initializeAwsDiscovery(options?.region, this.options.debugLogger); } /** @@ -107,6 +116,7 @@ export class CloudBurnClient { * @returns Supported AWS resource types. */ public async listSupportedDiscoveryResourceTypes(): Promise { + emitDebugLog(this.options?.debugLogger, 'sdk: listing supported Resource Explorer resource types'); return listSupportedAwsResourceTypes(); } diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index de001c2..6fb41af 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -59,6 +59,8 @@ import type { StaticDatasetMap, StaticResourceBag, } from '@cloudburn/rules'; +import type { AwsRegion } from './providers/aws/client.js'; +export type { AwsRegion }; // Intent: define SDK-facing contracts for scanner orchestration. // TODO(cloudburn): extend config and result metadata as new providers/resources land. @@ -86,10 +88,9 @@ export type BuiltInRuleMetadata = Pick { }); }); +describe('hydrateAwsCloudWatchLogGroupRecentStreamActivity', () => { + beforeEach(() => { + vi.resetAllMocks(); + }); + + it('loads only the most recent stream summary for each discovered log group', async () => { + const send = vi.fn(async (command: DescribeLogStreamsCommand) => { + const input = command.input as { + descending?: boolean; + limit?: number; + logGroupName?: string; + nextToken?: string; + orderBy?: string; + }; + + expect(input.nextToken).toBeUndefined(); + expect(input.orderBy).toBe('LastEventTime'); + expect(input.descending).toBe(true); + expect(input.limit).toBe(1); + + return { + logStreams: [ + { + arn: `arn:aws:logs:us-east-1:123456789012:log-group:${input.logGroupName}:log-stream:latest`, + lastEventTimestamp: 1_770_000_000_000, + lastIngestionTime: 1_770_000_100_000, + logStreamName: 'latest', + }, + ], + }; + }); + + mockedCreateCloudWatchLogsClient.mockReturnValue({ send } as never); + + await expect( + hydrateAwsCloudWatchLogGroupRecentStreamActivity([ + { + accountId: '123456789012', + arn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app', + properties: [], + region: 'us-east-1', + resourceType: 'logs:log-group', + service: 'logs', + }, + ]), + ).resolves.toEqual([ + { + accountId: '123456789012', + lastEventTimestamp: 1_770_000_000_000, + lastIngestionTime: 1_770_000_100_000, + latestStreamArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:latest', + latestStreamName: 'latest', + logGroupName: '/aws/lambda/app', + region: 'us-east-1', + }, + ]); + + expect(send).toHaveBeenCalledTimes(1); + }); + + it('falls back to the discovered account id when a log group has no streams yet', async () => { + mockedCreateCloudWatchLogsClient.mockReturnValue({ + send: vi.fn(async (_command: DescribeLogStreamsCommand) => ({ + logStreams: [], + })), + } as never); + + await expect( + hydrateAwsCloudWatchLogGroupRecentStreamActivity([ + { + accountId: '123456789012', + arn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app', + properties: [], + region: 'us-east-1', + resourceType: 'logs:log-group', + service: 'logs', + }, + ]), + ).resolves.toEqual([ + { + accountId: '123456789012', + lastEventTimestamp: undefined, + lastIngestionTime: undefined, + latestStreamArn: undefined, + latestStreamName: undefined, + logGroupName: '/aws/lambda/app', + region: 'us-east-1', + }, + ]); + }); +}); + describe('hydrateAwsCloudWatchLogMetricFilterCoverage', () => { beforeEach(() => { vi.resetAllMocks(); diff --git a/packages/sdk/test/providers/aws-discovery.test.ts b/packages/sdk/test/providers/aws-discovery.test.ts index 5be6768..a7e407d 100644 --- a/packages/sdk/test/providers/aws-discovery.test.ts +++ b/packages/sdk/test/providers/aws-discovery.test.ts @@ -6,7 +6,6 @@ import { discoverAwsResources, getAwsDiscoveryStatus, initializeAwsDiscovery, - listEnabledAwsDiscoveryRegions, listSupportedAwsResourceTypes, } from '../../src/providers/aws/discovery.js'; import { @@ -26,6 +25,7 @@ import { } from '../../src/providers/aws/resources/cloudfront.js'; import { hydrateAwsCloudTrailTrails } from '../../src/providers/aws/resources/cloudtrail.js'; import { + hydrateAwsCloudWatchLogGroupRecentStreamActivity, hydrateAwsCloudWatchLogGroups, hydrateAwsCloudWatchLogMetricFilterCoverage, hydrateAwsCloudWatchLogStreams, @@ -153,6 +153,7 @@ vi.mock('../../src/providers/aws/resources/cloudfront.js', () => ({ vi.mock('../../src/providers/aws/resources/cloudwatch-logs.js', () => ({ hydrateAwsCloudWatchLogGroups: vi.fn(), + hydrateAwsCloudWatchLogGroupRecentStreamActivity: vi.fn(), hydrateAwsCloudWatchLogMetricFilterCoverage: vi.fn(), hydrateAwsCloudWatchLogStreams: vi.fn(), })); @@ -264,6 +265,9 @@ const _mockedHydrateAwsCloudFrontDistributionRequestActivity = vi.mocked( ); const mockedHydrateAwsCloudTrailTrails = vi.mocked(hydrateAwsCloudTrailTrails); const mockedHydrateAwsCloudWatchLogGroups = vi.mocked(hydrateAwsCloudWatchLogGroups); +const mockedHydrateAwsCloudWatchLogGroupRecentStreamActivity = vi.mocked( + hydrateAwsCloudWatchLogGroupRecentStreamActivity, +); const mockedHydrateAwsCloudWatchLogMetricFilterCoverage = vi.mocked(hydrateAwsCloudWatchLogMetricFilterCoverage); const mockedHydrateAwsCloudWatchLogStreams = vi.mocked(hydrateAwsCloudWatchLogStreams); const mockedHydrateAwsCostUsage = vi.mocked(hydrateAwsCostUsage); @@ -309,6 +313,7 @@ const mockedHydrateAwsRoute53Zones = vi.mocked(hydrateAwsRoute53Zones); const mockedHydrateAwsS3BucketAnalyses = vi.mocked(hydrateAwsS3BucketAnalyses); const mockedHydrateAwsSageMakerNotebookInstances = vi.mocked(hydrateAwsSageMakerNotebookInstances); const mockedHydrateAwsSecretsManagerSecrets = vi.mocked(hydrateAwsSecretsManagerSecrets); +const loadContextMatcher = expect.objectContaining({ loadDataset: expect.any(Function) }); const catalog: AwsDiscoveryCatalog = { indexType: 'LOCAL', @@ -593,21 +598,21 @@ describe('discoverAwsResources', () => { service: 's3', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'ec2:instance', 'ec2:volume', 'ecr:repository', 'lambda:function', 's3:bucket', ]); - expect(mockedHydrateAwsEbsVolumes).toHaveBeenCalledWith([catalog.resources[0]]); - expect(mockedHydrateAwsEc2Instances).toHaveBeenCalledWith([catalog.resources[1]]); - expect(mockedHydrateAwsEcrRepositories).toHaveBeenCalledWith([catalog.resources[2]]); - expect(mockedHydrateAwsLambdaFunctions).toHaveBeenCalledWith([catalog.resources[3]]); - expect(mockedHydrateAwsS3BucketAnalyses).toHaveBeenCalledWith([catalog.resources[4]]); + expect(mockedHydrateAwsEbsVolumes).toHaveBeenCalledWith([catalog.resources[0]], loadContextMatcher); + expect(mockedHydrateAwsEc2Instances).toHaveBeenCalledWith([catalog.resources[1]], loadContextMatcher); + expect(mockedHydrateAwsEcrRepositories).toHaveBeenCalledWith([catalog.resources[2]], loadContextMatcher); + expect(mockedHydrateAwsLambdaFunctions).toHaveBeenCalledWith([catalog.resources[3]], loadContextMatcher); + expect(mockedHydrateAwsS3BucketAnalyses).toHaveBeenCalledWith([catalog.resources[4]], loadContextMatcher); expect(result.catalog).toEqual(catalog); expect(result.resources).toBeInstanceOf(LiveResourceBag); expect(result.resources.get('aws-ebs-volumes')).toEqual([ @@ -851,10 +856,10 @@ describe('discoverAwsResources', () => { discoveryDependencies: ['aws-secretsmanager-secrets'], }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'apigateway:restapis/stages', 'cloudfront:distribution', 'dynamodb:table', @@ -862,16 +867,31 @@ describe('discoverAwsResources', () => { 'route53:hostedzone', 'secretsmanager:secret', ]); - expect(mockedHydrateAwsApiGatewayStages).toHaveBeenCalledWith([extendedCatalog.resources[0]]); - expect(mockedHydrateAwsCloudFrontDistributions).toHaveBeenCalledWith([extendedCatalog.resources[1]]); - expect(mockedHydrateAwsCostUsage).toHaveBeenCalledWith([]); - expect(mockedHydrateAwsDynamoDbTables).toHaveBeenCalledWith([extendedCatalog.resources[2]]); - expect(mockedHydrateAwsDynamoDbAutoscaling).toHaveBeenCalledWith([extendedCatalog.resources[2]]); - expect(mockedHydrateAwsDynamoDbTableUtilization).toHaveBeenCalledWith([extendedCatalog.resources[2]]); - expect(mockedHydrateAwsRoute53Zones).toHaveBeenCalledWith([extendedCatalog.resources[3]]); - expect(mockedHydrateAwsRoute53Records).toHaveBeenCalledWith([extendedCatalog.resources[3]]); - expect(mockedHydrateAwsRoute53HealthChecks).toHaveBeenCalledWith([extendedCatalog.resources[4]]); - expect(mockedHydrateAwsSecretsManagerSecrets).toHaveBeenCalledWith([extendedCatalog.resources[5]]); + expect(mockedHydrateAwsApiGatewayStages).toHaveBeenCalledWith([extendedCatalog.resources[0]], loadContextMatcher); + expect(mockedHydrateAwsCloudFrontDistributions).toHaveBeenCalledWith( + [extendedCatalog.resources[1]], + loadContextMatcher, + ); + expect(mockedHydrateAwsCostUsage).toHaveBeenCalledWith([], loadContextMatcher); + expect(mockedHydrateAwsDynamoDbTables).toHaveBeenCalledWith([extendedCatalog.resources[2]], loadContextMatcher); + expect(mockedHydrateAwsDynamoDbAutoscaling).toHaveBeenCalledWith( + [extendedCatalog.resources[2]], + loadContextMatcher, + ); + expect(mockedHydrateAwsDynamoDbTableUtilization).toHaveBeenCalledWith( + [extendedCatalog.resources[2]], + loadContextMatcher, + ); + expect(mockedHydrateAwsRoute53Zones).toHaveBeenCalledWith([extendedCatalog.resources[3]], loadContextMatcher); + expect(mockedHydrateAwsRoute53Records).toHaveBeenCalledWith([extendedCatalog.resources[3]], loadContextMatcher); + expect(mockedHydrateAwsRoute53HealthChecks).toHaveBeenCalledWith( + [extendedCatalog.resources[4]], + loadContextMatcher, + ); + expect(mockedHydrateAwsSecretsManagerSecrets).toHaveBeenCalledWith( + [extendedCatalog.resources[5]], + loadContextMatcher, + ); expect(result.resources.get('aws-cost-usage')).toHaveLength(1); expect(result.resources.get('aws-route53-records')).toHaveLength(1); }); @@ -919,13 +939,13 @@ describe('discoverAwsResources', () => { discoveryDependencies: ['aws-cost-anomaly-monitors'], }), ], - { mode: 'region', region: 'eu-west-1' }, + { mode: 'regions', regions: ['eu-west-1'] }, ); expect(mockedBuildAwsDiscoveryCatalog).not.toHaveBeenCalled(); - expect(mockedHydrateAwsCostUsage).toHaveBeenCalledWith([]); - expect(mockedHydrateAwsCostGuardrailBudgets).toHaveBeenCalledWith([]); - expect(mockedHydrateAwsCostAnomalyMonitors).toHaveBeenCalledWith([]); + expect(mockedHydrateAwsCostUsage).toHaveBeenCalledWith([], loadContextMatcher); + expect(mockedHydrateAwsCostGuardrailBudgets).toHaveBeenCalledWith([], loadContextMatcher); + expect(mockedHydrateAwsCostAnomalyMonitors).toHaveBeenCalledWith([], loadContextMatcher); expect(result.catalog).toEqual({ indexType: 'LOCAL', resources: [], @@ -981,13 +1001,13 @@ describe('discoverAwsResources', () => { service: 'cloudtrail', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'cloudtrail:trail', ]); - expect(mockedHydrateAwsCloudTrailTrails).toHaveBeenCalledWith([catalog.resources[6]]); + expect(mockedHydrateAwsCloudTrailTrails).toHaveBeenCalledWith([catalog.resources[6]], loadContextMatcher); expect(result.resources.get('aws-cloudtrail-trails')).toEqual([ { accountId: '123456789012', @@ -1025,13 +1045,82 @@ describe('discoverAwsResources', () => { service: 'lambda', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'lambda:function', ]); - expect(mockedHydrateAwsLambdaFunctionMetrics).toHaveBeenCalledWith([catalog.resources[3]]); + expect(mockedHydrateAwsLambdaFunctionMetrics).toHaveBeenCalledWith([catalog.resources[3]], loadContextMatcher); + expect(result.resources.get('aws-lambda-function-metrics')).toEqual([ + { + accountId: '123456789012', + averageDurationMsLast7Days: 2_500, + functionName: 'my-func', + region: 'us-east-1', + totalErrorsLast7Days: 12, + totalInvocationsLast7Days: 100, + }, + ]); + }); + + it('reuses memoized base datasets when metrics and base datasets are requested together', async () => { + mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ + indexType: 'LOCAL', + resources: [catalog.resources[3]], + searchRegion: 'us-east-1', + }); + mockedHydrateAwsLambdaFunctions.mockResolvedValue([ + { + accountId: '123456789012', + architectures: ['x86_64'], + functionName: 'my-func', + memorySizeMb: 512, + region: 'us-east-1', + timeoutSeconds: 60, + }, + ]); + mockedHydrateAwsLambdaFunctionMetrics.mockImplementation(async (_resources, context) => { + const functions = await context.loadDataset('aws-lambda-functions'); + + return functions.map((fn) => ({ + accountId: fn.accountId, + averageDurationMsLast7Days: 2_500, + functionName: fn.functionName, + region: fn.region, + totalErrorsLast7Days: 12, + totalInvocationsLast7Days: 100, + })); + }); + + const result = await discoverAwsResources( + [ + createRule({ + discoveryDependencies: ['aws-lambda-functions', 'aws-lambda-function-metrics'], + service: 'lambda', + }), + ], + { mode: 'regions', regions: ['us-east-1'] }, + ); + + expect(mockedHydrateAwsLambdaFunctions).toHaveBeenCalledTimes(1); + expect(mockedHydrateAwsLambdaFunctions).toHaveBeenCalledWith([catalog.resources[3]], loadContextMatcher); + expect(mockedHydrateAwsLambdaFunctionMetrics).toHaveBeenCalledWith( + [catalog.resources[3]], + expect.objectContaining({ + loadDataset: expect.any(Function), + }), + ); + expect(result.resources.get('aws-lambda-functions')).toEqual([ + { + accountId: '123456789012', + architectures: ['x86_64'], + functionName: 'my-func', + memorySizeMb: 512, + region: 'us-east-1', + timeoutSeconds: 60, + }, + ]); expect(result.resources.get('aws-lambda-function-metrics')).toEqual([ { accountId: '123456789012', @@ -1044,6 +1133,45 @@ describe('discoverAwsResources', () => { ]); }); + it('emits dataset completion timing in debug mode so slow hydrators are visible', async () => { + mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ + indexType: 'LOCAL', + resources: [catalog.resources[3]], + searchRegion: 'us-east-1', + }); + mockedHydrateAwsLambdaFunctions.mockResolvedValue([ + { + accountId: '123456789012', + architectures: ['x86_64'], + functionName: 'my-func', + memorySizeMb: 512, + region: 'us-east-1', + timeoutSeconds: 60, + }, + ]); + const debugLogger = vi.fn(); + + await discoverAwsResources( + [ + createRule({ + discoveryDependencies: ['aws-lambda-functions'], + service: 'lambda', + }), + ], + { mode: 'regions', regions: ['us-east-1'] }, + { debugLogger }, + ); + + expect(debugLogger).toHaveBeenCalledWith('aws: loading dataset aws-lambda-functions'); + expect(debugLogger).toHaveBeenCalledWith('aws: loading dataset aws-lambda-functions in us-east-1 from 1 resources'); + expect(debugLogger.mock.calls.map(([message]) => message)).toEqual( + expect.arrayContaining([ + expect.stringMatching(/^aws: completed dataset aws-lambda-functions in us-east-1 with 1 resources in \d+ms$/), + expect.stringMatching(/^aws: completed dataset aws-lambda-functions with 1 resources in \d+ms$/), + ]), + ); + }); + it('hydrates ECS and EKS datasets from their discovery resource types', async () => { mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ indexType: 'LOCAL', @@ -1137,21 +1265,21 @@ describe('discoverAwsResources', () => { service: 'eks', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'ecs:cluster', 'ecs:container-instance', 'ecs:service', 'eks:cluster', ]); - expect(mockedHydrateAwsEcsContainerInstances).toHaveBeenCalledWith([catalog.resources[11]]); - expect(mockedHydrateAwsEcsClusters).toHaveBeenCalledWith([catalog.resources[12]]); - expect(mockedHydrateAwsEcsClusterMetrics).toHaveBeenCalledWith([catalog.resources[12]]); - expect(mockedHydrateAwsEcsServices).toHaveBeenCalledWith([catalog.resources[13]]); - expect(mockedHydrateAwsEcsAutoscaling).toHaveBeenCalledWith([catalog.resources[13]]); - expect(mockedHydrateAwsEksNodegroups).toHaveBeenCalledWith([catalog.resources[14]]); + expect(mockedHydrateAwsEcsContainerInstances).toHaveBeenCalledWith([catalog.resources[11]], loadContextMatcher); + expect(mockedHydrateAwsEcsClusters).toHaveBeenCalledWith([catalog.resources[12]], loadContextMatcher); + expect(mockedHydrateAwsEcsClusterMetrics).toHaveBeenCalledWith([catalog.resources[12]], loadContextMatcher); + expect(mockedHydrateAwsEcsServices).toHaveBeenCalledWith([catalog.resources[13]], loadContextMatcher); + expect(mockedHydrateAwsEcsAutoscaling).toHaveBeenCalledWith([catalog.resources[13]], loadContextMatcher); + expect(mockedHydrateAwsEksNodegroups).toHaveBeenCalledWith([catalog.resources[14]], loadContextMatcher); expect(result.resources.get('aws-ecs-container-instances')).toEqual([ { accountId: '123456789012', @@ -1321,22 +1449,22 @@ describe('discoverAwsResources', () => { service: 'redshift', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'elasticache:cluster', 'elasticache:reserved-instance', 'elasticmapreduce:cluster', 'redshift:cluster', ]); - expect(mockedHydrateAwsElastiCacheClusters).toHaveBeenCalledWith([catalog.resources[15]]); - expect(mockedHydrateAwsElastiCacheReservedNodes).toHaveBeenCalledWith([catalog.resources[16]]); - expect(mockedHydrateAwsEmrClusters).toHaveBeenCalledWith([catalog.resources[17]]); - expect(mockedHydrateAwsEmrClusterMetrics).toHaveBeenCalledWith([catalog.resources[17]]); - expect(mockedHydrateAwsRedshiftClusters).toHaveBeenCalledWith([catalog.resources[18]]); - expect(mockedHydrateAwsRedshiftClusterMetrics).toHaveBeenCalledWith([catalog.resources[18]]); - expect(mockedHydrateAwsRedshiftReservedNodes).toHaveBeenCalledWith([catalog.resources[18]]); + expect(mockedHydrateAwsElastiCacheClusters).toHaveBeenCalledWith([catalog.resources[15]], loadContextMatcher); + expect(mockedHydrateAwsElastiCacheReservedNodes).toHaveBeenCalledWith([catalog.resources[16]], loadContextMatcher); + expect(mockedHydrateAwsEmrClusters).toHaveBeenCalledWith([catalog.resources[17]], loadContextMatcher); + expect(mockedHydrateAwsEmrClusterMetrics).toHaveBeenCalledWith([catalog.resources[17]], loadContextMatcher); + expect(mockedHydrateAwsRedshiftClusters).toHaveBeenCalledWith([catalog.resources[18]], loadContextMatcher); + expect(mockedHydrateAwsRedshiftClusterMetrics).toHaveBeenCalledWith([catalog.resources[18]], loadContextMatcher); + expect(mockedHydrateAwsRedshiftReservedNodes).toHaveBeenCalledWith([catalog.resources[18]], loadContextMatcher); expect(result.resources.get('aws-elasticache-clusters')).toHaveLength(1); expect(result.resources.get('aws-emr-cluster-metrics')).toHaveLength(1); expect(result.resources.get('aws-redshift-reserved-nodes')).toHaveLength(1); @@ -1366,13 +1494,13 @@ describe('discoverAwsResources', () => { service: 'cloudwatch', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'logs:log-group', ]); - expect(mockedHydrateAwsCloudWatchLogGroups).toHaveBeenCalledWith([catalog.resources[7]]); + expect(mockedHydrateAwsCloudWatchLogGroups).toHaveBeenCalledWith([catalog.resources[7]], loadContextMatcher); expect(result.resources.get('aws-cloudwatch-log-groups')).toEqual([ { accountId: '123456789012', @@ -1419,14 +1547,14 @@ describe('discoverAwsResources', () => { service: 'cloudwatch', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'logs:log-group', ]); - expect(mockedHydrateAwsCloudWatchLogGroups).toHaveBeenCalledWith([catalog.resources[7]]); - expect(mockedHydrateAwsCloudWatchLogStreams).toHaveBeenCalledWith([catalog.resources[7]]); + expect(mockedHydrateAwsCloudWatchLogGroups).toHaveBeenCalledWith([catalog.resources[7]], loadContextMatcher); + expect(mockedHydrateAwsCloudWatchLogStreams).toHaveBeenCalledWith([catalog.resources[7]], loadContextMatcher); expect(result.resources.get('aws-cloudwatch-log-groups')).toEqual([ { accountId: '123456789012', @@ -1449,6 +1577,66 @@ describe('discoverAwsResources', () => { ]); }); + it('hydrates CloudWatch log-group recent stream activity from log-group catalog resources', async () => { + mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ + indexType: 'LOCAL', + resources: [catalog.resources[7]], + searchRegion: 'us-east-1', + }); + mockedHydrateAwsCloudWatchLogGroups.mockResolvedValue([ + { + accountId: '123456789012', + logGroupArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app', + logGroupClass: 'STANDARD', + logGroupName: '/aws/lambda/app', + region: 'us-east-1', + retentionInDays: 30, + storedBytes: 2048, + }, + ]); + mockedHydrateAwsCloudWatchLogGroupRecentStreamActivity.mockResolvedValue([ + { + accountId: '123456789012', + lastEventTimestamp: 1_770_000_000_000, + lastIngestionTime: 1_770_000_100_000, + latestStreamArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:latest', + latestStreamName: 'latest', + logGroupName: '/aws/lambda/app', + region: 'us-east-1', + }, + ]); + + const result = await discoverAwsResources( + [ + createRule({ + discoveryDependencies: ['aws-cloudwatch-log-groups', 'aws-cloudwatch-log-group-recent-stream-activity'], + service: 'cloudwatch', + }), + ], + { mode: 'regions', regions: ['us-east-1'] }, + ); + + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ + 'logs:log-group', + ]); + expect(mockedHydrateAwsCloudWatchLogGroups).toHaveBeenCalledWith([catalog.resources[7]], loadContextMatcher); + expect(mockedHydrateAwsCloudWatchLogGroupRecentStreamActivity).toHaveBeenCalledWith( + [catalog.resources[7]], + loadContextMatcher, + ); + expect(result.resources.get('aws-cloudwatch-log-group-recent-stream-activity')).toEqual([ + { + accountId: '123456789012', + lastEventTimestamp: 1_770_000_000_000, + lastIngestionTime: 1_770_000_100_000, + latestStreamArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:latest', + latestStreamName: 'latest', + logGroupName: '/aws/lambda/app', + region: 'us-east-1', + }, + ]); + }); + it('hydrates CloudWatch log metric-filter coverage from log-group catalog resources', async () => { mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ indexType: 'LOCAL', @@ -1471,13 +1659,16 @@ describe('discoverAwsResources', () => { service: 'cloudwatch', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'logs:log-group', ]); - expect(mockedHydrateAwsCloudWatchLogMetricFilterCoverage).toHaveBeenCalledWith([catalog.resources[7]]); + expect(mockedHydrateAwsCloudWatchLogMetricFilterCoverage).toHaveBeenCalledWith( + [catalog.resources[7]], + loadContextMatcher, + ); expect(result.resources.get('aws-cloudwatch-log-metric-filter-coverage')).toEqual([ { accountId: '123456789012', @@ -1511,11 +1702,13 @@ describe('discoverAwsResources', () => { service: 's3', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, ['s3:bucket']); - expect(mockedHydrateAwsS3BucketAnalyses).toHaveBeenCalledWith([catalog.resources[4]]); + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ + 's3:bucket', + ]); + expect(mockedHydrateAwsS3BucketAnalyses).toHaveBeenCalledWith([catalog.resources[4]], loadContextMatcher); expect(mockedHydrateAwsEbsVolumes).not.toHaveBeenCalled(); expect(mockedHydrateAwsEc2Instances).not.toHaveBeenCalled(); expect(mockedHydrateAwsLambdaFunctions).not.toHaveBeenCalled(); @@ -1569,10 +1762,10 @@ describe('discoverAwsResources', () => { service: 'elb', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'ec2:reserved-instances', 'elasticloadbalancing:loadbalancer', 'elasticloadbalancing:loadbalancer/app', @@ -1580,9 +1773,9 @@ describe('discoverAwsResources', () => { 'elasticloadbalancing:loadbalancer/net', 'elasticloadbalancing:targetgroup', ]); - expect(mockedHydrateAwsEc2ReservedInstances).toHaveBeenCalledWith([catalog.resources[8]]); - expect(mockedHydrateAwsEc2LoadBalancers).toHaveBeenCalledWith([catalog.resources[9]]); - expect(mockedHydrateAwsEc2TargetGroups).toHaveBeenCalledWith([catalog.resources[10]]); + expect(mockedHydrateAwsEc2ReservedInstances).toHaveBeenCalledWith([catalog.resources[8]], loadContextMatcher); + expect(mockedHydrateAwsEc2LoadBalancers).toHaveBeenCalledWith([catalog.resources[9]], loadContextMatcher); + expect(mockedHydrateAwsEc2TargetGroups).toHaveBeenCalledWith([catalog.resources[10]], loadContextMatcher); expect(result.resources.get('aws-ec2-reserved-instances')).toEqual([ { accountId: '123456789012', @@ -1638,11 +1831,13 @@ describe('discoverAwsResources', () => { service: 'rds', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, ['rds:db']); - expect(mockedHydrateAwsRdsInstances).toHaveBeenCalledWith([catalog.resources[5]]); + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ + 'rds:db', + ]); + expect(mockedHydrateAwsRdsInstances).toHaveBeenCalledWith([catalog.resources[5]], loadContextMatcher); expect(result.resources.get('aws-rds-instances' as never)).toEqual([ { accountId: '123456789012', @@ -1680,11 +1875,13 @@ describe('discoverAwsResources', () => { service: 'rds', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, ['rds:db']); - expect(mockedHydrateAwsRdsInstanceCpuMetrics).toHaveBeenCalledWith([catalog.resources[5]]); + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ + 'rds:db', + ]); + expect(mockedHydrateAwsRdsInstanceCpuMetrics).toHaveBeenCalledWith([catalog.resources[5]], loadContextMatcher); expect(result.resources.get('aws-rds-instance-cpu-metrics')).toEqual([ { accountId: '123456789012', @@ -1721,11 +1918,13 @@ describe('discoverAwsResources', () => { service: 'rds', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, ['rds:db']); - expect(mockedHydrateAwsRdsReservedInstances).toHaveBeenCalledWith([catalog.resources[5]]); + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ + 'rds:db', + ]); + expect(mockedHydrateAwsRdsReservedInstances).toHaveBeenCalledWith([catalog.resources[5]], loadContextMatcher); expect(result.resources.get('aws-rds-reserved-instances')).toEqual([ { accountId: '123456789012', @@ -1765,13 +1964,13 @@ describe('discoverAwsResources', () => { service: 'ec2', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'ec2:instance', ]); - expect(mockedHydrateAwsEc2InstanceUtilization).toHaveBeenCalledWith([catalog.resources[1]]); + expect(mockedHydrateAwsEc2InstanceUtilization).toHaveBeenCalledWith([catalog.resources[1]], loadContextMatcher); expect(result.resources.get('aws-ec2-instance-utilization')).toEqual([ { accountId: '123456789012', @@ -1819,22 +2018,25 @@ describe('discoverAwsResources', () => { service: 'ec2', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'ec2:natgateway', ]); - expect(mockedHydrateAwsEc2NatGatewayActivity).toHaveBeenCalledWith([ - { - accountId: '123456789012', - arn: 'arn:aws:ec2:us-east-1:123456789012:natgateway/nat-123', - properties: [], - region: 'us-east-1', - resourceType: 'ec2:natgateway', - service: 'ec2', - }, - ]); + expect(mockedHydrateAwsEc2NatGatewayActivity).toHaveBeenCalledWith( + [ + { + accountId: '123456789012', + arn: 'arn:aws:ec2:us-east-1:123456789012:natgateway/nat-123', + properties: [], + region: 'us-east-1', + resourceType: 'ec2:natgateway', + service: 'ec2', + }, + ], + loadContextMatcher, + ); expect(result.resources.get('aws-ec2-nat-gateway-activity')).toEqual([ { accountId: '123456789012', @@ -1871,11 +2073,13 @@ describe('discoverAwsResources', () => { service: 'rds', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, ['rds:db']); - expect(mockedHydrateAwsRdsInstanceActivity).toHaveBeenCalledWith([catalog.resources[5]]); + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ + 'rds:db', + ]); + expect(mockedHydrateAwsRdsInstanceActivity).toHaveBeenCalledWith([catalog.resources[5]], loadContextMatcher); expect(result.resources.get('aws-rds-instance-activity')).toEqual([ { accountId: '123456789012', @@ -1912,13 +2116,13 @@ describe('discoverAwsResources', () => { service: 'ebs', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'ec2:snapshot', ]); - expect(mockedHydrateAwsEbsSnapshots).toHaveBeenCalledWith([catalog.resources[19]]); + expect(mockedHydrateAwsEbsSnapshots).toHaveBeenCalledWith([catalog.resources[19]], loadContextMatcher); expect(result.resources.get('aws-ebs-snapshots')).toEqual([ { accountId: '123456789012', @@ -1956,13 +2160,13 @@ describe('discoverAwsResources', () => { service: 'rds', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'us-east-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['us-east-1'] }, [ 'rds:snapshot', ]); - expect(mockedHydrateAwsRdsSnapshots).toHaveBeenCalledWith([catalog.resources[20]]); + expect(mockedHydrateAwsRdsSnapshots).toHaveBeenCalledWith([catalog.resources[20]], loadContextMatcher); expect(result.resources.get('aws-rds-snapshots')).toEqual([ { accountId: '123456789012', @@ -2008,22 +2212,25 @@ describe('discoverAwsResources', () => { service: 'sagemaker', }), ], - { mode: 'region', region: 'eu-west-1' }, + { mode: 'regions', regions: ['eu-west-1'] }, ); - expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'region', region: 'eu-west-1' }, [ + expect(mockedBuildAwsDiscoveryCatalog).toHaveBeenCalledWith({ mode: 'regions', regions: ['eu-west-1'] }, [ 'sagemaker:notebook-instance', ]); - expect(mockedHydrateAwsSageMakerNotebookInstances).toHaveBeenCalledWith([ - { - accountId: '123456789012', - arn: 'arn:aws:sagemaker:eu-west-1:123456789012:notebook-instance/analytics-notebook', - properties: [], - region: 'eu-west-1', - resourceType: 'sagemaker:notebook-instance', - service: 'sagemaker', - }, - ]); + expect(mockedHydrateAwsSageMakerNotebookInstances).toHaveBeenCalledWith( + [ + { + accountId: '123456789012', + arn: 'arn:aws:sagemaker:eu-west-1:123456789012:notebook-instance/analytics-notebook', + properties: [], + region: 'eu-west-1', + resourceType: 'sagemaker:notebook-instance', + service: 'sagemaker', + }, + ], + loadContextMatcher, + ); expect(result.resources.get('aws-sagemaker-notebook-instances')).toEqual([ { accountId: '123456789012', @@ -2076,7 +2283,7 @@ describe('discoverAwsResources', () => { service: 'lambda', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); expect(result.resources.get('aws-ebs-volumes')).toEqual([ @@ -2135,7 +2342,7 @@ describe('discoverAwsResources', () => { service: 'ecr', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); expect(result.resources.get('aws-ecr-repositories')).toEqual([]); @@ -2184,7 +2391,7 @@ describe('discoverAwsResources', () => { service: 'ec2', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); expect(result.resources.get('aws-ec2-instance-utilization')).toEqual([]); @@ -2249,7 +2456,7 @@ describe('discoverAwsResources', () => { service: 'redshift', }), ], - { mode: 'region', region: 'us-east-1' }, + { mode: 'regions', regions: ['us-east-1'] }, ); expect(result.resources.get('aws-redshift-clusters')).toEqual([ @@ -2284,6 +2491,40 @@ describe('discoverAwsResources', () => { ]); }); + it('merges very large dataset loads without overflowing the call stack', async () => { + mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ + indexType: 'LOCAL', + resources: [catalog.resources[7]], + searchRegion: 'us-east-1', + }); + mockedHydrateAwsCloudWatchLogStreams.mockResolvedValue( + Array.from({ length: 150_000 }, (_, index) => ({ + accountId: '123456789012', + arn: `arn:aws:logs:us-east-1:123456789012:log-group:/aws/lambda/app:log-stream:stream-${index}`, + creationTime: undefined, + firstEventTimestamp: undefined, + lastEventTimestamp: undefined, + lastIngestionTime: undefined, + logGroupName: '/aws/lambda/app', + logStreamName: `stream-${index}`, + region: 'us-east-1', + })), + ); + + const result = await discoverAwsResources( + [ + createRule({ + discoveryDependencies: ['aws-cloudwatch-log-streams'], + service: 'cloudwatch', + }), + ], + { mode: 'regions', regions: ['us-east-1'] }, + ); + + expect(mockedHydrateAwsCloudWatchLogStreams).toHaveBeenCalledWith([catalog.resources[7]], loadContextMatcher); + expect(result.resources.get('aws-cloudwatch-log-streams')).toHaveLength(150_000); + }); + it('returns an empty catalog without Resource Explorer calls when no live rules require discovery metadata', async () => { mockedResolveCurrentAwsRegion.mockResolvedValue('us-east-1'); @@ -2309,6 +2550,79 @@ describe('discoverAwsResources', () => { expect(result.resources.get('aws-s3-bucket-analyses')).toEqual([]); }); + it('emits dataset completion timing to the debug logger', async () => { + mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ + indexType: 'LOCAL', + resources: [catalog.resources[1]], + searchRegion: 'us-east-1', + }); + mockedHydrateAwsEc2Instances.mockResolvedValue([ + { + accountId: '123456789012', + instanceId: 'i-123', + instanceType: 'c6i.large', + region: 'us-east-1', + }, + ]); + const debugLines: string[] = []; + + await discoverAwsResources( + [ + createRule({ + discoveryDependencies: ['aws-ec2-instances'], + }), + ], + { mode: 'regions', regions: ['us-east-1'] }, + { + debugLogger: (message) => debugLines.push(message), + }, + ); + + expect(debugLines).toContain('aws: loading dataset aws-ec2-instances'); + expect(debugLines).toContain('aws: loading dataset aws-ec2-instances in us-east-1 from 1 resources'); + expect( + debugLines.some((line) => + /^aws: completed dataset aws-ec2-instances in us-east-1 with 1 resources in \d+ms$/.test(line), + ), + ).toBe(true); + expect( + debugLines.some((line) => /^aws: completed dataset aws-ec2-instances with 1 resources in \d+ms$/.test(line)), + ).toBe(true); + }); + + it('emits dataset failure timing before surfacing non-access-denied errors', async () => { + mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ + indexType: 'LOCAL', + resources: [catalog.resources[1]], + searchRegion: 'us-east-1', + }); + mockedHydrateAwsEc2Instances.mockRejectedValue(new Error('boom')); + const debugLines: string[] = []; + + await expect( + discoverAwsResources( + [ + createRule({ + discoveryDependencies: ['aws-ec2-instances'], + }), + ], + { mode: 'regions', regions: ['us-east-1'] }, + { + debugLogger: (message) => debugLines.push(message), + }, + ), + ).rejects.toThrow('boom'); + + expect(debugLines).toContain('aws: loading dataset aws-ec2-instances'); + expect(debugLines).toContain('aws: loading dataset aws-ec2-instances in us-east-1 from 1 resources'); + expect( + debugLines.some((line) => /^aws: dataset aws-ec2-instances failed in us-east-1 after \d+ms: boom$/.test(line)), + ).toBe(true); + expect(debugLines.some((line) => /^aws: dataset aws-ec2-instances failed after \d+ms: boom$/.test(line))).toBe( + true, + ); + }); + it('fails fast when a discovery rule has an evaluator but no discoveryDependencies metadata', async () => { await expect( discoverAwsResources( @@ -2927,11 +3241,9 @@ describe('discovery support commands', () => { }); }); - it('delegates region listing and supported resource type listing to the resource explorer module', async () => { - mockedListAwsDiscoveryIndexes.mockResolvedValue([{ region: 'eu-west-1', type: 'local' }]); + it('delegates supported resource type listing to the resource explorer module', async () => { mockedListAwsDiscoverySupportedResourceTypes.mockResolvedValue([{ resourceType: 'ec2:volume', service: 'ec2' }]); - await expect(listEnabledAwsDiscoveryRegions()).resolves.toEqual([{ region: 'eu-west-1', type: 'local' }]); await expect(listSupportedAwsResourceTypes()).resolves.toEqual([{ resourceType: 'ec2:volume', service: 'ec2' }]); }); }); diff --git a/packages/sdk/test/providers/aws-ec2-utilization-resource.test.ts b/packages/sdk/test/providers/aws-ec2-utilization-resource.test.ts index a55b283..847c62f 100644 --- a/packages/sdk/test/providers/aws-ec2-utilization-resource.test.ts +++ b/packages/sdk/test/providers/aws-ec2-utilization-resource.test.ts @@ -72,4 +72,39 @@ describe('hydrateAwsEc2InstanceUtilization', () => { }, ]); }); + + it('reuses the shared EC2 instance dataset when a discovery context provides preloaded instances', async () => { + mockedFetchCloudWatchSignals.mockResolvedValue( + new Map([ + ['cpu0', [{ timestamp: '2026-03-01T00:00:00.000Z', value: 5 }]], + ['in0', [{ timestamp: '2026-03-01T00:00:00.000Z', value: 1024 }]], + ['out0', [{ timestamp: '2026-03-01T00:00:00.000Z', value: 1024 }]], + ]), + ); + + await expect( + hydrateAwsEc2InstanceUtilization([], { + loadDataset: async () => [ + { + accountId: '123456789012', + instanceId: 'i-123', + instanceType: 'm6i.large', + region: 'us-east-1', + }, + ], + }), + ).resolves.toEqual([ + { + accountId: '123456789012', + averageCpuUtilizationLast14Days: 5, + averageDailyNetworkBytesLast14Days: 2048, + instanceId: 'i-123', + instanceType: 'm6i.large', + lowUtilizationDays: 1, + region: 'us-east-1', + }, + ]); + + expect(mockedHydrateAwsEc2Instances).not.toHaveBeenCalled(); + }); }); diff --git a/packages/sdk/test/providers/aws-ecs-cluster-metrics-resource.test.ts b/packages/sdk/test/providers/aws-ecs-cluster-metrics-resource.test.ts index 19c045d..66c6064 100644 --- a/packages/sdk/test/providers/aws-ecs-cluster-metrics-resource.test.ts +++ b/packages/sdk/test/providers/aws-ecs-cluster-metrics-resource.test.ts @@ -68,4 +68,31 @@ describe('hydrateAwsEcsClusterMetrics', () => { }, ]); }); + + it('reuses the shared ECS cluster dataset when a discovery context provides preloaded clusters', async () => { + mockedFetchCloudWatchSignals.mockResolvedValue(new Map([['ecsCluster0', createDailyPoints(14, 5)]])); + + await expect( + hydrateAwsEcsClusterMetrics([], { + loadDataset: async () => [ + { + accountId: '123456789012', + clusterArn: 'arn:aws:ecs:us-east-1:123456789012:cluster/production', + clusterName: 'production', + region: 'us-east-1', + }, + ], + }), + ).resolves.toEqual([ + { + accountId: '123456789012', + averageCpuUtilizationLast14Days: 5, + clusterArn: 'arn:aws:ecs:us-east-1:123456789012:cluster/production', + clusterName: 'production', + region: 'us-east-1', + }, + ]); + + expect(mockedHydrateAwsEcsClusters).not.toHaveBeenCalled(); + }); }); diff --git a/packages/sdk/test/providers/aws-lambda-resource.test.ts b/packages/sdk/test/providers/aws-lambda-resource.test.ts index bae5db0..f954f28 100644 --- a/packages/sdk/test/providers/aws-lambda-resource.test.ts +++ b/packages/sdk/test/providers/aws-lambda-resource.test.ts @@ -379,4 +379,40 @@ describe('hydrateAwsLambdaFunctionMetrics', () => { }, ]); }); + + it('reuses the shared lambda dataset when a discovery context provides preloaded functions', async () => { + mockedFetchCloudWatchSignals.mockResolvedValue( + new Map([ + ['invocations0', [{ timestamp: '2026-03-24T00:00:00.000Z', value: 100 }]], + ['errors0', [{ timestamp: '2026-03-24T00:00:00.000Z', value: 12 }]], + ['duration0', [{ timestamp: '2026-03-24T00:00:00.000Z', value: 2_500 }]], + ]), + ); + + await expect( + hydrateAwsLambdaFunctionMetrics([], { + loadDataset: async () => [ + { + accountId: '123456789012', + architectures: ['x86_64'], + functionName: 'shared-function', + memorySizeMb: 512, + region: 'us-east-1', + timeoutSeconds: 60, + }, + ], + }), + ).resolves.toEqual([ + { + accountId: '123456789012', + averageDurationMsLast7Days: 2_500, + functionName: 'shared-function', + region: 'us-east-1', + totalErrorsLast7Days: 12, + totalInvocationsLast7Days: 100, + }, + ]); + + expect(mockedCreateLambdaClient).not.toHaveBeenCalled(); + }); }); diff --git a/packages/sdk/test/providers/aws-rds-activity-resource.test.ts b/packages/sdk/test/providers/aws-rds-activity-resource.test.ts index d03782e..45f7a5b 100644 --- a/packages/sdk/test/providers/aws-rds-activity-resource.test.ts +++ b/packages/sdk/test/providers/aws-rds-activity-resource.test.ts @@ -92,6 +92,33 @@ describe('hydrateAwsRdsInstanceActivity', () => { }, ]); }); + + it('reuses the shared RDS instance dataset when a discovery context provides preloaded instances', async () => { + mockedFetchCloudWatchSignals.mockResolvedValue(new Map([['rds0', createDailyPoints(7, 0)]])); + + await expect( + hydrateAwsRdsInstanceActivity([], { + loadDataset: async () => [ + { + accountId: '123456789012', + dbInstanceIdentifier: 'legacy-db', + instanceClass: 'db.m6i.large', + region: 'us-east-1', + }, + ], + }), + ).resolves.toEqual([ + { + accountId: '123456789012', + dbInstanceIdentifier: 'legacy-db', + instanceClass: 'db.m6i.large', + maxDatabaseConnectionsLast7Days: 0, + region: 'us-east-1', + }, + ]); + + expect(mockedHydrateAwsRdsInstances).not.toHaveBeenCalled(); + }); }); describe('hydrateAwsRdsInstanceCpuMetrics', () => { @@ -140,4 +167,30 @@ describe('hydrateAwsRdsInstanceCpuMetrics', () => { }, ]); }); + + it('reuses the shared RDS instance dataset for CPU metrics when a discovery context provides preloaded instances', async () => { + mockedFetchCloudWatchSignals.mockResolvedValue(new Map([['cpu0', createDailyPoints(30, 8)]])); + + await expect( + hydrateAwsRdsInstanceCpuMetrics([], { + loadDataset: async () => [ + { + accountId: '123456789012', + dbInstanceIdentifier: 'legacy-db', + instanceClass: 'db.m6i.large', + region: 'us-east-1', + }, + ], + }), + ).resolves.toEqual([ + { + accountId: '123456789012', + averageCpuUtilizationLast30Days: 8, + dbInstanceIdentifier: 'legacy-db', + region: 'us-east-1', + }, + ]); + + expect(mockedHydrateAwsRdsInstances).not.toHaveBeenCalled(); + }); }); diff --git a/packages/sdk/test/providers/aws-resource-explorer.test.ts b/packages/sdk/test/providers/aws-resource-explorer.test.ts index 95e32d1..6bb31ca 100644 --- a/packages/sdk/test/providers/aws-resource-explorer.test.ts +++ b/packages/sdk/test/providers/aws-resource-explorer.test.ts @@ -36,7 +36,8 @@ describe('resource explorer discovery', () => { }, })) .mockImplementationOnce(async (command) => { - expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume region:eu-central-1'); + expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume,lambda:function region:eu-central-1'); + expect(command.input.MaxResults).toBe(1000); expect(command.input.ViewArn).toBe('arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default'); return { @@ -49,16 +50,6 @@ describe('resource explorer discovery', () => { Service: 'ec2', Properties: [], }, - ], - ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', - }; - }) - .mockImplementationOnce(async (command) => { - expect(command.input.Filters?.FilterString).toBe('resourcetype:lambda:function region:eu-central-1'); - expect(command.input.ViewArn).toBe('arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default'); - - return { - Resources: [ { Arn: 'arn:aws:lambda:eu-central-1:123456789012:function:my-func', OwningAccountId: '123456789012', @@ -67,14 +58,6 @@ describe('resource explorer discovery', () => { Service: 'lambda', Properties: [], }, - { - Arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', - OwningAccountId: '123456789012', - Region: 'eu-central-1', - ResourceType: 'ec2:volume', - Service: 'ec2', - Properties: [], - }, ], ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', }; @@ -218,6 +201,42 @@ describe('resource explorer discovery', () => { ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', }, }) + .mockRejectedValueOnce( + Object.assign(new Error('Rate exceeded'), { + name: 'ThrottlingException', + $metadata: { + httpStatusCode: 429, + requestId: 'request-123', + }, + }), + ) + .mockRejectedValueOnce( + Object.assign(new Error('Rate exceeded'), { + name: 'ThrottlingException', + $metadata: { + httpStatusCode: 429, + requestId: 'request-123', + }, + }), + ) + .mockRejectedValueOnce( + Object.assign(new Error('Rate exceeded'), { + name: 'ThrottlingException', + $metadata: { + httpStatusCode: 429, + requestId: 'request-123', + }, + }), + ) + .mockRejectedValueOnce( + Object.assign(new Error('Rate exceeded'), { + name: 'ThrottlingException', + $metadata: { + httpStatusCode: 429, + requestId: 'request-123', + }, + }), + ) .mockRejectedValueOnce( Object.assign(new Error('Rate exceeded'), { name: 'ThrottlingException', @@ -235,6 +254,62 @@ describe('resource explorer discovery', () => { }); }); + it('retries throttled list resources calls before succeeding', async () => { + vi.spyOn(clientModule, 'resolveCurrentAwsRegion').mockResolvedValue('eu-central-1'); + vi.spyOn(clientModule, 'createResourceExplorerClient').mockReturnValue({ + send: vi + .fn() + .mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-central-1']); + + return { + Indexes: [ + { + Region: 'eu-central-1', + Type: 'AGGREGATOR', + }, + ], + }; + }) + .mockResolvedValueOnce({ + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }) + .mockResolvedValueOnce({ + View: { + Filters: { + FilterString: '', + }, + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }, + }) + .mockRejectedValueOnce( + Object.assign(new Error('Rate exceeded'), { + name: 'ThrottlingException', + $metadata: { + httpStatusCode: 429, + requestId: 'request-123', + }, + }), + ) + .mockResolvedValueOnce({ + Resources: [ + { + Arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + OwningAccountId: '123456789012', + Region: 'eu-central-1', + ResourceType: 'ec2:volume', + Service: 'ec2', + Properties: [], + }, + ], + }), + } as never); + + await expect(buildAwsDiscoveryCatalog({ mode: 'current' }, ['ec2:volume'])).resolves.toMatchObject({ + resources: [{ arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123' }], + }); + }); + it('fails when an explicit discovery region is malformed', async () => { vi.spyOn(clientModule, 'resolveCurrentAwsRegion').mockResolvedValue('eu-central-1'); vi.spyOn(clientModule, 'createResourceExplorerClient').mockReturnValue({ @@ -249,7 +324,9 @@ describe('resource explorer discovery', () => { } as never); await expect( - buildAwsDiscoveryCatalog({ mode: 'region', region: 'eu-central-1 resourcetype:s3:bucket' }, ['ec2:volume']), + buildAwsDiscoveryCatalog({ mode: 'regions', regions: ['eu-central-1 resourcetype:s3:bucket' as never] }, [ + 'ec2:volume', + ]), ).rejects.toMatchObject({ code: 'INVALID_AWS_REGION', }); @@ -317,23 +394,23 @@ describe('resource explorer discovery', () => { return euCentralClient; }); - await expect(buildAwsDiscoveryCatalog({ mode: 'region', region: 'eu-central-1' }, ['ec2:volume'])).resolves.toEqual( - { - resources: [ - { - arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', - accountId: '123456789012', - region: 'eu-central-1', - resourceType: 'ec2:volume', - service: 'ec2', - properties: [], - }, - ], - searchRegion: 'eu-central-1', - indexType: 'LOCAL', - viewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', - }, - ); + await expect( + buildAwsDiscoveryCatalog({ mode: 'regions', regions: ['eu-central-1'] }, ['ec2:volume']), + ).resolves.toEqual({ + resources: [ + { + arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + accountId: '123456789012', + region: 'eu-central-1', + resourceType: 'ec2:volume', + service: 'ec2', + properties: [], + }, + ], + searchRegion: 'eu-central-1', + indexType: 'LOCAL', + viewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }); expect(createResourceExplorerClient).toHaveBeenCalled(); }); @@ -397,6 +474,7 @@ describe('resource explorer discovery', () => { }) .mockImplementationOnce(async (command) => { expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume region:eu-west-1'); + expect(command.input.MaxResults).toBe(1000); return { Resources: [ @@ -425,7 +503,9 @@ describe('resource explorer discovery', () => { throw new Error(`Unexpected client region ${region}`); }); - await expect(buildAwsDiscoveryCatalog({ mode: 'region', region: 'eu-west-1' }, ['ec2:volume'])).resolves.toEqual({ + await expect( + buildAwsDiscoveryCatalog({ mode: 'regions', regions: ['eu-west-1'] }, ['ec2:volume']), + ).resolves.toEqual({ resources: [ { arn: 'arn:aws:ec2:eu-west-1:123456789012:volume/vol-123', @@ -522,7 +602,7 @@ describe('resource explorer discovery', () => { }); }); - it('limits all-region discovery to accessible indexed regions while skipping denied regions', async () => { + it('limits multi-region discovery to accessible indexed regions while skipping denied regions', async () => { vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['ap-south-1', 'eu-central-1', 'eu-west-1']); const apSouthClient = { @@ -562,7 +642,8 @@ describe('resource explorer discovery', () => { }, }) .mockImplementationOnce(async (command) => { - expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume region:eu-central-1'); + expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume region:eu-central-1,eu-west-1'); + expect(command.input.MaxResults).toBe(1000); return { Resources: [ @@ -574,14 +655,6 @@ describe('resource explorer discovery', () => { Service: 'ec2', Properties: [], }, - ], - }; - }) - .mockImplementationOnce(async (command) => { - expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume region:eu-west-1'); - - return { - Resources: [ { Arn: 'arn:aws:ec2:eu-west-1:123456789012:volume/vol-456', OwningAccountId: '123456789012', @@ -625,7 +698,9 @@ describe('resource explorer discovery', () => { throw new Error(`Unexpected client region ${region}`); }); - await expect(buildAwsDiscoveryCatalog({ mode: 'all' }, ['ec2:volume'])).resolves.toEqual({ + await expect( + buildAwsDiscoveryCatalog({ mode: 'regions', regions: ['eu-central-1', 'eu-west-1'] }, ['ec2:volume']), + ).resolves.toEqual({ resources: [ { arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', @@ -650,10 +725,10 @@ describe('resource explorer discovery', () => { }); expect(apSouthClient.send).toHaveBeenCalledTimes(1); expect(euWestClient.send).toHaveBeenCalledTimes(1); - expect(euCentralClient.send).toHaveBeenCalledTimes(5); + expect(euCentralClient.send).toHaveBeenCalledTimes(4); }); - it('fails all-region discovery when denied regions are skipped and no accessible aggregator exists', async () => { + it('fails multi-region discovery when denied regions are skipped and no accessible aggregator exists', async () => { vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['ap-south-1', 'eu-central-1']); const apSouthClient = { @@ -693,7 +768,9 @@ describe('resource explorer discovery', () => { throw new Error(`Unexpected client region ${region}`); }); - await expect(buildAwsDiscoveryCatalog({ mode: 'all' }, ['ec2:volume'])).rejects.toMatchObject({ + await expect( + buildAwsDiscoveryCatalog({ mode: 'regions', regions: ['ap-south-1', 'eu-central-1'] }, ['ec2:volume']), + ).rejects.toMatchObject({ code: 'RESOURCE_EXPLORER_AGGREGATOR_REQUIRED', }); }); diff --git a/packages/sdk/test/scanner.test.ts b/packages/sdk/test/scanner.test.ts index 61f5e0e..9eb772a 100644 --- a/packages/sdk/test/scanner.test.ts +++ b/packages/sdk/test/scanner.test.ts @@ -50,14 +50,14 @@ describe('CloudBurnClient', () => { const result = await scanner.discover({ target: { - mode: 'region', - region: 'us-east-1', + mode: 'regions', + regions: ['us-east-1'], }, }); expect(mockedDiscoverAwsResources).toHaveBeenCalledWith(expect.any(Array), { - mode: 'region', - region: 'us-east-1', + mode: 'regions', + regions: ['us-east-1'], }); expect(result).toEqual({ @@ -99,8 +99,8 @@ describe('CloudBurnClient', () => { const result = await scanner.discover({ target: { - mode: 'region', - region: 'us-east-1', + mode: 'regions', + regions: ['us-east-1'], }, }); @@ -153,8 +153,8 @@ describe('CloudBurnClient', () => { const result = await scanner.discover({ target: { - mode: 'region', - region: 'us-east-1', + mode: 'regions', + regions: ['us-east-1'], }, }); @@ -207,8 +207,8 @@ describe('CloudBurnClient', () => { const result = await scanner.discover({ target: { - mode: 'region', - region: 'us-east-1', + mode: 'regions', + regions: ['us-east-1'], }, }); @@ -264,6 +264,29 @@ describe('CloudBurnClient', () => { }); }); + it('forwards the configured debug logger into live discovery', async () => { + mockedDiscoverAwsResources.mockResolvedValue({ + catalog: discoveryCatalog, + resources: new LiveResourceBag(), + }); + + const debugLogger = vi.fn(); + const scanner = new CloudBurnClient({ debugLogger }); + + await scanner.discover(); + + expect(mockedDiscoverAwsResources).toHaveBeenCalledWith( + expect.any(Array), + { + mode: 'current', + }, + { + debugLogger, + }, + ); + expect(debugLogger).toHaveBeenCalledWith('sdk: starting live discovery scan'); + }); + it('passes an explicit config path through discovery config loading', async () => { mockedDiscoverAwsResources.mockResolvedValue({ catalog: discoveryCatalog, From 0eba4e6e1f6ed72aaa16b2dc6db37c504aa9eba6 Mon Sep 17 00:00:00 2001 From: Danny Steenman Date: Tue, 31 Mar 2026 21:38:07 +0200 Subject: [PATCH 2/5] fix(sdk): avoid duplicate single-region debug logs --- packages/sdk/src/providers/aws/discovery.ts | 10 ++-- .../sdk/test/providers/aws-discovery.test.ts | 54 +++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/packages/sdk/src/providers/aws/discovery.ts b/packages/sdk/src/providers/aws/discovery.ts index 012b659..7465be3 100644 --- a/packages/sdk/src/providers/aws/discovery.ts +++ b/packages/sdk/src/providers/aws/discovery.ts @@ -393,10 +393,12 @@ export const discoverAwsResources = async ( } } - emitDebugLog( - options?.debugLogger, - `aws: completed dataset ${definition.datasetKey} with ${loadedResources.length} resources in ${formatElapsedMs(startedAtMs)}`, - ); + if (regionResourceGroups.size > 1) { + emitDebugLog( + options?.debugLogger, + `aws: completed dataset ${definition.datasetKey} with ${loadedResources.length} resources in ${formatElapsedMs(startedAtMs)}`, + ); + } return { dataset: [definition.datasetKey, loadedResources as DiscoveryDatasetMap[K]] as [K, DiscoveryDatasetMap[K]], diff --git a/packages/sdk/test/providers/aws-discovery.test.ts b/packages/sdk/test/providers/aws-discovery.test.ts index a7e407d..f4bf3cf 100644 --- a/packages/sdk/test/providers/aws-discovery.test.ts +++ b/packages/sdk/test/providers/aws-discovery.test.ts @@ -1167,6 +1167,10 @@ describe('discoverAwsResources', () => { expect(debugLogger.mock.calls.map(([message]) => message)).toEqual( expect.arrayContaining([ expect.stringMatching(/^aws: completed dataset aws-lambda-functions in us-east-1 with 1 resources in \d+ms$/), + ]), + ); + expect(debugLogger.mock.calls.map(([message]) => message)).not.toEqual( + expect.arrayContaining([ expect.stringMatching(/^aws: completed dataset aws-lambda-functions with 1 resources in \d+ms$/), ]), ); @@ -2587,6 +2591,56 @@ describe('discoverAwsResources', () => { ).toBe(true); expect( debugLines.some((line) => /^aws: completed dataset aws-ec2-instances with 1 resources in \d+ms$/.test(line)), + ).toBe(false); + }); + + it('emits one aggregate completion timing when a dataset spans multiple regions', async () => { + mockedBuildAwsDiscoveryCatalog.mockResolvedValue({ + indexType: 'AGGREGATOR', + resources: [ + catalog.resources[1], + { + ...catalog.resources[1], + arn: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-456', + region: 'eu-west-1', + }, + ], + searchRegion: 'us-east-1', + }); + mockedHydrateAwsEc2Instances.mockImplementation(async (resources) => + resources.map((resource) => ({ + accountId: resource.accountId, + instanceId: resource.arn.split('/').at(-1) ?? 'unknown', + instanceType: 'c6i.large', + region: resource.region, + })), + ); + const debugLines: string[] = []; + + await discoverAwsResources( + [ + createRule({ + discoveryDependencies: ['aws-ec2-instances'], + }), + ], + { mode: 'regions', regions: ['eu-west-1', 'us-east-1'] }, + { + debugLogger: (message) => debugLines.push(message), + }, + ); + + expect( + debugLines.some((line) => + /^aws: completed dataset aws-ec2-instances in us-east-1 with 1 resources in \d+ms$/.test(line), + ), + ).toBe(true); + expect( + debugLines.some((line) => + /^aws: completed dataset aws-ec2-instances in eu-west-1 with 1 resources in \d+ms$/.test(line), + ), + ).toBe(true); + expect( + debugLines.some((line) => /^aws: completed dataset aws-ec2-instances with 2 resources in \d+ms$/.test(line)), ).toBe(true); }); From 2aede0044ac2a9d2af237279af381a7b6abf8c74 Mon Sep 17 00:00:00 2001 From: Danny Steenman Date: Tue, 31 Mar 2026 21:41:16 +0200 Subject: [PATCH 3/5] fix(cli): list supported regions in validation errors --- packages/cloudburn/test/discover.e2e.test.ts | 3 +++ packages/sdk/src/providers/aws/client.ts | 3 ++- packages/sdk/test/providers/aws-client.test.ts | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/cloudburn/test/discover.e2e.test.ts b/packages/cloudburn/test/discover.e2e.test.ts index a0c2a5d..49d9b68 100644 --- a/packages/cloudburn/test/discover.e2e.test.ts +++ b/packages/cloudburn/test/discover.e2e.test.ts @@ -203,6 +203,9 @@ describe('discover command e2e', () => { expect(discover).not.toHaveBeenCalled(); expect(stderr).toHaveBeenCalledWith(expect.stringContaining("Invalid AWS region 'totally-fake-1'.")); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('Supported regions:')); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('eu-central-1')); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('us-east-1')); }); it('rejects invalid service filters before invoking the sdk discover method', async () => { diff --git a/packages/sdk/src/providers/aws/client.ts b/packages/sdk/src/providers/aws/client.ts index 4ec0dc0..a7e16f6 100644 --- a/packages/sdk/src/providers/aws/client.ts +++ b/packages/sdk/src/providers/aws/client.ts @@ -70,6 +70,7 @@ export const AWS_REGIONS = [ ] as const; export type AwsRegion = (typeof AWS_REGIONS)[number]; +const SUPPORTED_AWS_REGIONS_MESSAGE = AWS_REGIONS.join(', '); /** * Validates an AWS region string before it is used in clients or filters. @@ -81,7 +82,7 @@ export const assertValidAwsRegion = (region: string | undefined): AwsRegion => { if (!region || !AWS_REGION_PATTERN.test(region) || !AWS_REGIONS.includes(region as AwsRegion)) { throw new AwsDiscoveryError( 'INVALID_AWS_REGION', - `Invalid AWS region '${region ?? ''}'. Use a supported AWS region name such as 'eu-central-1' or 'us-east-1'.`, + `Invalid AWS region '${region ?? ''}'. Supported regions: ${SUPPORTED_AWS_REGIONS_MESSAGE}.`, ); } diff --git a/packages/sdk/test/providers/aws-client.test.ts b/packages/sdk/test/providers/aws-client.test.ts index 1527c2c..da491c4 100644 --- a/packages/sdk/test/providers/aws-client.test.ts +++ b/packages/sdk/test/providers/aws-client.test.ts @@ -52,4 +52,12 @@ describe('resolveCurrentAwsRegion', { timeout: 30_000 }, () => { code: 'INVALID_AWS_REGION', }); }); + + it('lists supported regions when region validation fails', async () => { + const clientModule = await importClientModule(); + + expect(() => clientModule.assertValidAwsRegion('bla')).toThrowError("Invalid AWS region 'bla'. Supported regions:"); + expect(() => clientModule.assertValidAwsRegion('bla')).toThrowError('eu-central-1'); + expect(() => clientModule.assertValidAwsRegion('bla')).toThrowError('us-east-1'); + }); }); From e9984095d60388db224bd3e313937c5c88465cde Mon Sep 17 00:00:00 2001 From: Danny Steenman Date: Wed, 1 Apr 2026 08:05:50 +0200 Subject: [PATCH 4/5] fix(sdk): restore discovery target compatibility --- packages/cloudburn/src/commands/discover.ts | 4 +- packages/sdk/src/index.ts | 2 +- packages/sdk/src/providers/aws/client.ts | 22 ++- .../src/providers/aws/resource-explorer.ts | 4 + packages/sdk/src/types.ts | 5 + .../sdk/test/providers/aws-client.test.ts | 14 +- .../providers/aws-resource-explorer.test.ts | 186 ++++++++++++++++++ 7 files changed, 230 insertions(+), 7 deletions(-) diff --git a/packages/cloudburn/src/commands/discover.ts b/packages/cloudburn/src/commands/discover.ts index 1e1ccaa..f3a79dc 100644 --- a/packages/cloudburn/src/commands/discover.ts +++ b/packages/cloudburn/src/commands/discover.ts @@ -1,4 +1,4 @@ -import { type AwsDiscoveryTarget, type AwsRegion, assertValidAwsRegion, CloudBurnClient } from '@cloudburn/sdk'; +import { type AwsDiscoveryTarget, type AwsRegion, assertSupportedAwsRegion, CloudBurnClient } from '@cloudburn/sdk'; import { type Command, InvalidArgumentError } from 'commander'; import { resolveCliDebugLogger } from '../debug.js'; import { EXIT_CODE_OK, EXIT_CODE_POLICY_VIOLATION, EXIT_CODE_RUNTIME_ERROR } from '../exit-codes.js'; @@ -130,7 +130,7 @@ const buildInitializationStatusData = ( const parseAwsRegion = (value: string): AwsRegion => { try { - return assertValidAwsRegion(value); + return assertSupportedAwsRegion(value); } catch (err) { throw new InvalidArgumentError(err instanceof Error ? err.message : 'Invalid AWS region.'); } diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 8b2b305..0abeda6 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -3,7 +3,7 @@ export { awsCorePreset } from '@cloudburn/rules'; export { builtInRuleMetadata } from './built-in-rules.js'; export { parseIaC } from './parsers/index.js'; -export { assertValidAwsRegion } from './providers/aws/client.js'; +export { assertSupportedAwsRegion, assertValidAwsRegion } from './providers/aws/client.js'; export { isAwsDiscoveryErrorCode } from './providers/aws/errors.js'; export { CloudBurnClient } from './scanner.js'; export type { diff --git a/packages/sdk/src/providers/aws/client.ts b/packages/sdk/src/providers/aws/client.ts index a7e16f6..6678b21 100644 --- a/packages/sdk/src/providers/aws/client.ts +++ b/packages/sdk/src/providers/aws/client.ts @@ -72,13 +72,33 @@ export const AWS_REGIONS = [ export type AwsRegion = (typeof AWS_REGIONS)[number]; const SUPPORTED_AWS_REGIONS_MESSAGE = AWS_REGIONS.join(', '); +const assertAwsRegionShape = (region: string | undefined): string => { + if (!region || !AWS_REGION_PATTERN.test(region)) { + throw new AwsDiscoveryError( + 'INVALID_AWS_REGION', + `Invalid AWS region '${region ?? ''}'. Use a valid AWS region name such as 'eu-central-1' or 'us-east-1'.`, + ); + } + + return region; +}; + /** * Validates an AWS region string before it is used in clients or filters. * * @param region - AWS region to validate. * @returns The original region when valid. */ -export const assertValidAwsRegion = (region: string | undefined): AwsRegion => { +export const assertValidAwsRegion = (region: string | undefined): AwsRegion => + assertAwsRegionShape(region) as AwsRegion; + +/** + * Validates that a CLI-supplied AWS region is one of the known supported regions. + * + * @param region - AWS region supplied by the caller. + * @returns The original region when it is part of the supported region list. + */ +export const assertSupportedAwsRegion = (region: string | undefined): AwsRegion => { if (!region || !AWS_REGION_PATTERN.test(region) || !AWS_REGIONS.includes(region as AwsRegion)) { throw new AwsDiscoveryError( 'INVALID_AWS_REGION', diff --git a/packages/sdk/src/providers/aws/resource-explorer.ts b/packages/sdk/src/providers/aws/resource-explorer.ts index c9a6961..0351945 100644 --- a/packages/sdk/src/providers/aws/resource-explorer.ts +++ b/packages/sdk/src/providers/aws/resource-explorer.ts @@ -688,6 +688,10 @@ export const buildAwsDiscoveryCatalog = async ( } else { searchPlan = await resolveAggregatorSearchPlan(target.regions); } + } else if (target.mode === 'region') { + searchPlan = await resolveRegionalSearchPlan(assertValidAwsRegion(target.region)); + } else if (target.mode === 'all') { + searchPlan = await resolveAggregatorSearchPlan(); } else { searchPlan = await resolveRegionalSearchPlan(currentRegion); } diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 6fb41af..40b931a 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -88,6 +88,11 @@ export type BuiltInRuleMetadata = Pick { }); }); + it('accepts valid AWS region strings outside the commercial allowlist at runtime', async () => { + const clientModule = await importClientModule(); + + expect(clientModule.assertValidAwsRegion('us-gov-west-1')).toBe('us-gov-west-1'); + }); + it('lists supported regions when region validation fails', async () => { const clientModule = await importClientModule(); - expect(() => clientModule.assertValidAwsRegion('bla')).toThrowError("Invalid AWS region 'bla'. Supported regions:"); - expect(() => clientModule.assertValidAwsRegion('bla')).toThrowError('eu-central-1'); - expect(() => clientModule.assertValidAwsRegion('bla')).toThrowError('us-east-1'); + expect(() => clientModule.assertSupportedAwsRegion('bla')).toThrowError( + "Invalid AWS region 'bla'. Supported regions:", + ); + expect(() => clientModule.assertSupportedAwsRegion('bla')).toThrowError('eu-central-1'); + expect(() => clientModule.assertSupportedAwsRegion('bla')).toThrowError('us-east-1'); }); }); diff --git a/packages/sdk/test/providers/aws-resource-explorer.test.ts b/packages/sdk/test/providers/aws-resource-explorer.test.ts index 6bb31ca..b0628e9 100644 --- a/packages/sdk/test/providers/aws-resource-explorer.test.ts +++ b/packages/sdk/test/providers/aws-resource-explorer.test.ts @@ -414,6 +414,86 @@ describe('resource explorer discovery', () => { expect(createResourceExplorerClient).toHaveBeenCalled(); }); + it('accepts the deprecated single-region discovery target shape', async () => { + vi.spyOn(clientModule, 'resolveCurrentAwsRegion').mockResolvedValue('us-east-1'); + vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['eu-central-1']); + + const euCentralClient = { + send: vi + .fn() + .mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-central-1']); + + return { + Indexes: [ + { + Region: 'eu-central-1', + Type: 'LOCAL', + }, + ], + }; + }) + .mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-central-1']); + + return { + Indexes: [ + { + Region: 'eu-central-1', + Type: 'LOCAL', + }, + ], + }; + }) + .mockResolvedValueOnce({ + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }) + .mockResolvedValueOnce({ + View: { + Filters: { + FilterString: '', + }, + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }, + }) + .mockResolvedValueOnce({ + Resources: [ + { + Arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + OwningAccountId: '123456789012', + Region: 'eu-central-1', + ResourceType: 'ec2:volume', + Service: 'ec2', + Properties: [], + }, + ], + }), + } as never; + + vi.spyOn(clientModule, 'createResourceExplorerClient').mockImplementation(({ region }) => { + expect(region).toBe('eu-central-1'); + return euCentralClient; + }); + + await expect(buildAwsDiscoveryCatalog({ mode: 'region', region: 'eu-central-1' }, ['ec2:volume'])).resolves.toEqual( + { + resources: [ + { + arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + accountId: '123456789012', + region: 'eu-central-1', + resourceType: 'ec2:volume', + service: 'ec2', + properties: [], + }, + ], + searchRegion: 'eu-central-1', + indexType: 'LOCAL', + viewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }, + ); + }); + it('uses the aggregator control plane for an explicit local region when the aggregator lives elsewhere', async () => { vi.spyOn(clientModule, 'resolveCurrentAwsRegion').mockResolvedValue('us-east-1'); vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['eu-west-1', 'eu-central-1']); @@ -728,6 +808,112 @@ describe('resource explorer discovery', () => { expect(euCentralClient.send).toHaveBeenCalledTimes(4); }); + it('accepts the deprecated all-regions discovery target shape', async () => { + vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['eu-central-1', 'eu-west-1']); + + const euCentralClient = { + send: vi + .fn() + .mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-central-1']); + + return { + Indexes: [ + { + Region: 'eu-central-1', + Type: 'AGGREGATOR', + }, + ], + }; + }) + .mockResolvedValueOnce({ + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }) + .mockResolvedValueOnce({ + View: { + Filters: { + FilterString: '', + }, + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }, + }) + .mockImplementationOnce(async (command) => { + expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume region:eu-central-1,eu-west-1'); + + return { + Resources: [ + { + Arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + OwningAccountId: '123456789012', + Region: 'eu-central-1', + ResourceType: 'ec2:volume', + Service: 'ec2', + Properties: [], + }, + { + Arn: 'arn:aws:ec2:eu-west-1:123456789012:volume/vol-456', + OwningAccountId: '123456789012', + Region: 'eu-west-1', + ResourceType: 'ec2:volume', + Service: 'ec2', + Properties: [], + }, + ], + }; + }), + } as never; + const euWestClient = { + send: vi.fn().mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-west-1']); + + return { + Indexes: [ + { + Region: 'eu-west-1', + Type: 'LOCAL', + }, + ], + }; + }), + } as never; + + vi.spyOn(clientModule, 'createResourceExplorerClient').mockImplementation(({ region }) => { + if (region === 'eu-central-1') { + return euCentralClient; + } + + if (region === 'eu-west-1') { + return euWestClient; + } + + throw new Error(`Unexpected client region ${region}`); + }); + + await expect(buildAwsDiscoveryCatalog({ mode: 'all' }, ['ec2:volume'])).resolves.toEqual({ + resources: [ + { + arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + accountId: '123456789012', + region: 'eu-central-1', + resourceType: 'ec2:volume', + service: 'ec2', + properties: [], + }, + { + arn: 'arn:aws:ec2:eu-west-1:123456789012:volume/vol-456', + accountId: '123456789012', + region: 'eu-west-1', + resourceType: 'ec2:volume', + service: 'ec2', + properties: [], + }, + ], + searchRegion: 'eu-central-1', + indexType: 'AGGREGATOR', + viewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }); + }); + it('fails multi-region discovery when denied regions are skipped and no accessible aggregator exists', async () => { vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['ap-south-1', 'eu-central-1']); From 2a041fe5bb0a5550369b7c02fb15bdb9602a768b Mon Sep 17 00:00:00 2001 From: Danny Steenman Date: Wed, 1 Apr 2026 08:44:15 +0200 Subject: [PATCH 5/5] fix(sdk): avoid ambient region lookup for explicit discovery targets --- .../src/providers/aws/resource-explorer.ts | 3 +- .../providers/aws-resource-explorer.test.ts | 201 +++++++++++++++++- 2 files changed, 201 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/providers/aws/resource-explorer.ts b/packages/sdk/src/providers/aws/resource-explorer.ts index 0351945..d04ef14 100644 --- a/packages/sdk/src/providers/aws/resource-explorer.ts +++ b/packages/sdk/src/providers/aws/resource-explorer.ts @@ -678,7 +678,6 @@ export const buildAwsDiscoveryCatalog = async ( resourceTypes: string[], options?: { debugLogger?: (message: string) => void }, ): Promise => { - const currentRegion = await resolveCurrentAwsRegion(); let searchPlan: SearchPlan; if (target.mode === 'regions') { @@ -693,7 +692,7 @@ export const buildAwsDiscoveryCatalog = async ( } else if (target.mode === 'all') { searchPlan = await resolveAggregatorSearchPlan(); } else { - searchPlan = await resolveRegionalSearchPlan(currentRegion); + searchPlan = await resolveRegionalSearchPlan(await resolveCurrentAwsRegion()); } emitDebugLog( options?.debugLogger, diff --git a/packages/sdk/test/providers/aws-resource-explorer.test.ts b/packages/sdk/test/providers/aws-resource-explorer.test.ts index b0628e9..2da350d 100644 --- a/packages/sdk/test/providers/aws-resource-explorer.test.ts +++ b/packages/sdk/test/providers/aws-resource-explorer.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import * as clientModule from '../../src/providers/aws/client.js'; import { buildAwsDiscoveryCatalog, @@ -7,6 +7,10 @@ import { } from '../../src/providers/aws/resource-explorer.js'; describe('resource explorer discovery', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + it('builds a deduplicated catalog and applies a region filter when listing from an aggregator region', async () => { vi.spyOn(clientModule, 'resolveCurrentAwsRegion').mockResolvedValue('eu-central-1'); @@ -494,6 +498,90 @@ describe('resource explorer discovery', () => { ); }); + it('accepts the deprecated single-region discovery target shape without ambient region state', async () => { + const ambientRegionError = new Error('ambient region unavailable'); + const resolveCurrentAwsRegion = vi + .spyOn(clientModule, 'resolveCurrentAwsRegion') + .mockRejectedValue(ambientRegionError); + vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['eu-central-1']); + + const euCentralClient = { + send: vi + .fn() + .mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-central-1']); + + return { + Indexes: [ + { + Region: 'eu-central-1', + Type: 'LOCAL', + }, + ], + }; + }) + .mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-central-1']); + + return { + Indexes: [ + { + Region: 'eu-central-1', + Type: 'LOCAL', + }, + ], + }; + }) + .mockResolvedValueOnce({ + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }) + .mockResolvedValueOnce({ + View: { + Filters: { + FilterString: '', + }, + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }, + }) + .mockResolvedValueOnce({ + Resources: [ + { + Arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + OwningAccountId: '123456789012', + Region: 'eu-central-1', + ResourceType: 'ec2:volume', + Service: 'ec2', + Properties: [], + }, + ], + }), + } as never; + + vi.spyOn(clientModule, 'createResourceExplorerClient').mockImplementation(({ region }) => { + expect(region).toBe('eu-central-1'); + return euCentralClient; + }); + + await expect(buildAwsDiscoveryCatalog({ mode: 'region', region: 'eu-central-1' }, ['ec2:volume'])).resolves.toEqual( + { + resources: [ + { + arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + accountId: '123456789012', + region: 'eu-central-1', + resourceType: 'ec2:volume', + service: 'ec2', + properties: [], + }, + ], + searchRegion: 'eu-central-1', + indexType: 'LOCAL', + viewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }, + ); + expect(resolveCurrentAwsRegion).not.toHaveBeenCalled(); + }); + it('uses the aggregator control plane for an explicit local region when the aggregator lives elsewhere', async () => { vi.spyOn(clientModule, 'resolveCurrentAwsRegion').mockResolvedValue('us-east-1'); vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['eu-west-1', 'eu-central-1']); @@ -914,6 +1002,117 @@ describe('resource explorer discovery', () => { }); }); + it('accepts the deprecated all-regions discovery target shape without ambient region state', async () => { + const ambientRegionError = new Error('ambient region unavailable'); + const resolveCurrentAwsRegion = vi + .spyOn(clientModule, 'resolveCurrentAwsRegion') + .mockRejectedValue(ambientRegionError); + vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['eu-central-1', 'eu-west-1']); + + const euCentralClient = { + send: vi + .fn() + .mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-central-1']); + + return { + Indexes: [ + { + Region: 'eu-central-1', + Type: 'AGGREGATOR', + }, + ], + }; + }) + .mockResolvedValueOnce({ + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }) + .mockResolvedValueOnce({ + View: { + Filters: { + FilterString: '', + }, + ViewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }, + }) + .mockImplementationOnce(async (command) => { + expect(command.input.Filters?.FilterString).toBe('resourcetype:ec2:volume region:eu-central-1,eu-west-1'); + + return { + Resources: [ + { + Arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + OwningAccountId: '123456789012', + Region: 'eu-central-1', + ResourceType: 'ec2:volume', + Service: 'ec2', + Properties: [], + }, + { + Arn: 'arn:aws:ec2:eu-west-1:123456789012:volume/vol-456', + OwningAccountId: '123456789012', + Region: 'eu-west-1', + ResourceType: 'ec2:volume', + Service: 'ec2', + Properties: [], + }, + ], + }; + }), + } as never; + const euWestClient = { + send: vi.fn().mockImplementationOnce(async (command) => { + expect(command.input.Regions).toEqual(['eu-west-1']); + + return { + Indexes: [ + { + Region: 'eu-west-1', + Type: 'LOCAL', + }, + ], + }; + }), + } as never; + + vi.spyOn(clientModule, 'createResourceExplorerClient').mockImplementation(({ region }) => { + if (region === 'eu-central-1') { + return euCentralClient; + } + + if (region === 'eu-west-1') { + return euWestClient; + } + + throw new Error(`Unexpected client region ${region}`); + }); + + await expect(buildAwsDiscoveryCatalog({ mode: 'all' }, ['ec2:volume'])).resolves.toEqual({ + resources: [ + { + arn: 'arn:aws:ec2:eu-central-1:123456789012:volume/vol-123', + accountId: '123456789012', + region: 'eu-central-1', + resourceType: 'ec2:volume', + service: 'ec2', + properties: [], + }, + { + arn: 'arn:aws:ec2:eu-west-1:123456789012:volume/vol-456', + accountId: '123456789012', + region: 'eu-west-1', + resourceType: 'ec2:volume', + service: 'ec2', + properties: [], + }, + ], + searchRegion: 'eu-central-1', + indexType: 'AGGREGATOR', + viewArn: 'arn:aws:resource-explorer-2:eu-central-1:123456789012:view/default', + }); + expect(resolveCurrentAwsRegion).not.toHaveBeenCalled(); + }); + it('fails multi-region discovery when denied regions are skipped and no accessible aggregator exists', async () => { vi.spyOn(clientModule, 'listEnabledAwsRegions').mockResolvedValue(['ap-south-1', 'eu-central-1']);