fix(sdk): degrade gracefully on discovery dataset failures

.changeset/good-bears-invent.md

@@ -0,0 +1,5 @@
+---
+"@cloudburn/sdk": patch
+---
+
+Gracefully degrade AWS discovery when required datasets are throttled or otherwise unavailable by retrying longer and surfacing skipped-rule diagnostics instead of aborting the run.

.changeset/neat-socks-laugh.md

@@ -0,0 +1,5 @@
+---
+"cloudburn": patch
+---
+
+Improve discover table output by separating diagnostics into a dedicated table so skipped rules and access-denied discovery results stay readable.

packages/cloudburn/src/formatters/output.ts

@@ -74,6 +74,16 @@ const scanColumns: ColumnSpec[] = [
   { key: 'message', header: 'Message' },
 ];
 
+const diagnosticColumns: ColumnSpec[] = [
+  { key: 'provider', header: 'Provider' },
+  { key: 'status', header: 'Status' },
+  { key: 'ruleId', header: 'RuleId' },
+  { key: 'source', header: 'Source' },
+  { key: 'service', header: 'Service' },
+  { key: 'region', header: 'Region' },
+  { key: 'message', header: 'Message' },
+];
+
 const ruleListColumns: ColumnSpec[] = [
   { key: 'ruleId', header: 'RuleId' },
   { key: 'provider', header: 'Provider' },
@@ -176,8 +186,22 @@ const renderTable = (response: CliResponse): string => {
     case 'rule-list':
       return renderRuleTable(response.rules, response.emptyMessage);
     case 'scan-result': {
-      const rows = projectScanRows(response.result);
-      return rows.length === 0 ? 'No findings.' : renderAsciiTable(rows, scanColumns);
+      const findingRows = projectFindingRows(response.result);
+      const diagnosticRows = projectDiagnosticRows(response.result);
+
+      if (findingRows.length === 0 && diagnosticRows.length === 0) {
+        return 'No findings.';
+      }
+
+      if (findingRows.length === 0) {
+        return `Diagnostics\n${renderAsciiTable(diagnosticRows, diagnosticColumns)}`;
+      }
+
+      if (diagnosticRows.length === 0) {
+        return renderAsciiTable(findingRows, scanColumns);
+      }
+
+      return `${renderAsciiTable(findingRows, scanColumns)}\n\nDiagnostics\n${renderAsciiTable(diagnosticRows, diagnosticColumns)}`;
     }
     case 'status':
       return renderAsciiTable(
@@ -197,8 +221,8 @@ const renderTable = (response: CliResponse): string => {
   }
 };
 
-const projectScanRows = (result: ScanResult): RecordRow[] => [
-  ...flattenScanResult(result).map(({ finding, message, provider, ruleId, service, source }) => ({
+const projectFindingRows = (result: ScanResult): RecordRow[] =>
+  flattenScanResult(result).map(({ finding, message, provider, ruleId, service, source }) => ({
     accountId: finding.accountId ?? '',
     message,
     path: finding.location?.path ?? '',
@@ -210,21 +234,18 @@ const projectScanRows = (result: ScanResult): RecordRow[] => [
     source,
     column: finding.location?.column ?? '',
     line: finding.location?.line ?? '',
-  })),
-  ...getScanDiagnostics(result).map((diagnostic) => ({
-    accountId: '',
-    column: '',
-    line: '',
+  }));
+
+const projectDiagnosticRows = (result: ScanResult): RecordRow[] =>
+  getScanDiagnostics(result).map((diagnostic) => ({
     message: diagnostic.message,
-    path: '',
     provider: diagnostic.provider,
     region: diagnostic.region ?? '',
-    resourceId: '',
-    ruleId: '',
+    ruleId: diagnostic.ruleId ?? '',
     service: diagnostic.service,
     source: diagnostic.source,
-  })),
-];
+    status: diagnostic.status,
+  }));
 
 const inferColumns = (rows: RecordRow[]): ColumnSpec[] => {
   const keys = Array.from(new Set(rows.flatMap((row) => Object.keys(row)))).sort((left, right) =>

packages/cloudburn/test/formatters.test.ts

@@ -51,6 +51,36 @@ const resultWithLocation = {
   ],
 };
 
+const resultWithSkippedRuleDiagnostic = {
+  diagnostics: [
+    {
+      details:
+        'Amazon CloudWatch Logs DescribeMetricFilters failed in us-east-1 with ThrottlingException: Rate exceeded.',
+      message: 'Skipped rule CLDBRN-AWS-CLOUDWATCH-3 because required discovery datasets were unavailable.',
+      provider: 'aws' as const,
+      ruleId: 'CLDBRN-AWS-CLOUDWATCH-3',
+      service: 'cloudwatch',
+      source: 'discovery' as const,
+      status: 'skipped' as const,
+    },
+  ],
+  providers: [],
+};
+
+const resultWithFindingAndDiagnostic = {
+  diagnostics: [
+    {
+      message: 'Skipped lambda discovery in us-east-1 because access is denied by AWS permissions.',
+      provider: 'aws' as const,
+      region: 'us-east-1',
+      service: 'lambda',
+      source: 'discovery' as const,
+      status: 'access_denied' as const,
+    },
+  ],
+  providers: resultWithoutLocation.providers,
+};
+
 const withStdoutColumns = (columns: number, run: () => void): void => {
   const descriptor = Object.getOwnPropertyDescriptor(process.stdout, 'columns');
 
@@ -85,6 +115,27 @@ describe('renderResponse', () => {
     `);
   });
 
+  it('renders skipped-rule diagnostics with their rule id in table mode', () => {
+    const output = renderResponse({ kind: 'scan-result', result: resultWithSkippedRuleDiagnostic }, 'table');
+
+    expect(output).toContain('Diagnostics');
+    expect(output).toContain('Status');
+    expect(output).toContain('CLDBRN-AWS-CLOUDWATCH-3');
+    expect(output).toContain('Skipped rule CLDBRN-AWS-CLOUDWATCH-3');
+    expect(output).not.toContain('ResourceId');
+    expect(output).not.toContain('AccountId');
+  });
+
+  it('renders diagnostics in a separate table when findings also exist', () => {
+    const output = renderResponse({ kind: 'scan-result', result: resultWithFindingAndDiagnostic }, 'table');
+
+    expect(output).toContain('CLDBRN-AWS-EBS-1');
+    expect(output).toContain('vol-123');
+    expect(output).toContain('Diagnostics');
+    expect(output).toContain('access_denied');
+    expect(output).toContain('Skipped lambda discovery in us-east-1');
+  });
+
   it('wraps long status values to the available terminal width in table mode', () => {
     withStdoutColumns(60, () => {
       const output = renderResponse(

packages/sdk/src/engine/run-live.ts

@@ -11,10 +11,23 @@ export const runLiveScan = async (
 ): Promise<ScanResult> => {
   const registry = buildRuleRegistry(config, 'discovery');
   emitDebugLog(options?.debugLogger, `sdk: resolved ${registry.activeRules.length} active discovery rules`);
-  const { diagnostics = [], ...liveContext } =
-    options?.debugLogger === undefined
-      ? await discoverAwsResources(registry.activeRules, target)
-      : await discoverAwsResources(registry.activeRules, target, { debugLogger: options.debugLogger });
+  const {
+    diagnostics = [],
+    unavailableDatasets = new Map(),
+    ...liveContext
+  } = options?.debugLogger === undefined
+    ? await discoverAwsResources(registry.activeRules, target)
+    : await discoverAwsResources(registry.activeRules, target, { debugLogger: options.debugLogger });
+  const unresolvedUnavailableDatasets: unknown = unavailableDatasets;
+  const unavailableDatasetDiagnostics =
+    unresolvedUnavailableDatasets instanceof Map
+      ? unresolvedUnavailableDatasets
+      : new Map(
+          unresolvedUnavailableDatasets instanceof Set
+            ? [...unresolvedUnavailableDatasets].map((datasetKey) => [datasetKey, []] as const)
+            : [],
+        );
+  const scanDiagnostics = [...diagnostics];
   const findings = groupFindingsByProvider(
     registry.activeRules.map((rule) => {
       if (!rule.supports.includes('discovery') || !rule.evaluateLive) {
@@ -24,6 +37,32 @@ export const runLiveScan = async (
         };
       }
 
+      const unavailableDependencies = (rule.discoveryDependencies ?? []).filter((dependency) =>
+        unavailableDatasetDiagnostics.has(dependency),
+      );
+
+      if (unavailableDependencies.length > 0) {
+        scanDiagnostics.push({
+          details: unavailableDependencies
+            .flatMap((dependency) => unavailableDatasetDiagnostics.get(dependency) ?? [])
+            .map((diagnostic) => diagnostic.details)
+            .filter((detail): detail is string => detail !== undefined)
+            .filter((detail, index, details) => details.indexOf(detail) === index)
+            .join('\n'),
+          message: `Skipped rule ${rule.id} because required discovery datasets were unavailable: ${unavailableDependencies.join(', ')}.`,
+          provider: rule.provider,
+          ruleId: rule.id,
+          service: rule.service,
+          source: 'discovery',
+          status: 'skipped',
+        });
+
+        return {
+          provider: rule.provider,
+          finding: null,
+        };
+      }
+
       return {
         provider: rule.provider,
         finding: rule.evaluateLive(liveContext),
@@ -32,7 +71,7 @@ export const runLiveScan = async (
   );
 
   return {
-    ...(diagnostics.length > 0 ? { diagnostics } : {}),
+    ...(scanDiagnostics.length > 0 ? { diagnostics: scanDiagnostics } : {}),
     providers: findings,
   };
 };