Skip to content

fix(import): aggregate CRD imports from same API group #3798

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mateo-moon wants to merge 1 commit into
base: 2.x
Choose a base branch
from
Open

fix(import): aggregate CRD imports from same API group #3798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/import/crd.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,33 @@ export class ImportCustomResourceDefinition extends ImportBase {
return new ImportCustomResourceDefinition(manifest);
}

/**
* Creates an importer from multiple import specs by downloading and combining
* their manifests. This ensures CRDs from the same API group across different
* sources are consolidated into a single module.
*
* @param importSpecs Array of import specifications to aggregate
* @returns A single ImportCustomResourceDefinition containing all CRDs
*/
public static async fromSpecs(importSpecs: ImportSpec[]): Promise<ImportCustomResourceDefinition> {
const manifests = await Promise.all(
importSpecs.map(spec => download(spec.source)),
);
// Combine all manifests with YAML document separator
const combinedManifest = manifests.join('\n---\n');
return new ImportCustomResourceDefinition(combinedManifest);
}

/**
* Creates an importer directly from a raw manifest string.
*
* @param manifest Raw YAML manifest string containing one or more CRDs
* @returns An ImportCustomResourceDefinition instance
*/
public static fromManifest(manifest: string): ImportCustomResourceDefinition {
return new ImportCustomResourceDefinition(manifest);
}

public readonly rawManifest: string;
private readonly groups: Record<string, CustomResourceDefinition[]> = { };

Expand Down
123 changes: 116 additions & 7 deletions src/import/dispatch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,133 @@ import { ImportKubernetesApi } from './k8s';
import { ImportSpec, addImportToConfig } from '../config';
import { PREFIX_DELIM } from '../util';

export async function importDispatch(imports: ImportSpec[], argv: any, options: ImportOptions) {
/**
* Categorized import specs for processing.
*/
interface CategorizedImports {
/** Kubernetes API imports (k8s@version) */
k8s: Array<{ spec: ImportSpec; importer: ImportKubernetesApi }>;
/** Helm chart imports */
helm: ImportSpec[];
/** CRD imports without module name prefix (will be aggregated) */
crdUnprefixed: ImportSpec[];
/** CRD imports with module name prefix (processed individually to preserve prefix) */
crdPrefixed: ImportSpec[];
}

/**
* Categorizes import specs by type for appropriate processing.
*
* CRD imports without a module name prefix are grouped together so they can be
* aggregated - this ensures CRDs from the same API group across different sources
* are consolidated into a single module instead of overwriting each other.
*
* CRD imports with a module name prefix are kept separate since the prefix
* indicates the user wants them in distinct modules.
*/
async function categorizeImports(imports: ImportSpec[], argv: any): Promise<CategorizedImports> {
const result: CategorizedImports = {
k8s: [],
helm: [],
crdUnprefixed: [],
crdPrefixed: [],
};

for (const importSpec of imports) {
const importer = await matchImporter(importSpec, argv);
// Check if it's a k8s@ import
const k8sMatch = await ImportKubernetesApi.match(importSpec, argv);
if (k8sMatch) {
result.k8s.push({ spec: importSpec, importer: new ImportKubernetesApi(k8sMatch) });
continue;
}

const prefix = importSpec.source.split(':')[0];

// Check if it's a helm import
if (prefix === 'helm') {
result.helm.push(importSpec);
continue;
}

// It's a CRD import - categorize by whether it has a module name prefix
if (importSpec.moduleNamePrefix) {
result.crdPrefixed.push(importSpec);
} else {
result.crdUnprefixed.push(importSpec);
}
}

if (!importer) {
throw new Error(`unable to determine import type for "${importSpec}"`);
return result;
}

export async function importDispatch(imports: ImportSpec[], argv: any, options: ImportOptions) {
const categorized = await categorizeImports(imports, argv);

// Process k8s imports
for (const { spec, importer } of categorized.k8s) {
console.error('Importing resources, this may take a few moments...');
await importer.import({
moduleNamePrefix: spec.moduleNamePrefix,
...options,
});
if (options.save ?? true) {
const specStr = spec.moduleNamePrefix ? `${spec.moduleNamePrefix}${PREFIX_DELIM}${spec.source}` : spec.source;
await addImportToConfig(specStr);
}
}

// Process helm imports
for (const spec of categorized.helm) {
const importer = await ImportHelm.fromSpec(spec);
console.error('Importing resources, this may take a few moments...');
await importer.import({
moduleNamePrefix: spec.moduleNamePrefix,
...options,
});
if (options.save ?? true) {
const specStr = spec.moduleNamePrefix ? `${spec.moduleNamePrefix}${PREFIX_DELIM}${spec.source}` : spec.source;
await addImportToConfig(specStr);
}
}

// Process CRD imports with prefix (individually, to preserve their prefixes)
for (const spec of categorized.crdPrefixed) {
// Check for crds.dev URL format
const crdsDevUrl = matchCrdsDevUrl(spec.source);
const importer = crdsDevUrl
? await ImportCustomResourceDefinition.fromSpec({ source: crdsDevUrl, moduleNamePrefix: spec.moduleNamePrefix })
: await ImportCustomResourceDefinition.fromSpec(spec);

console.error('Importing resources, this may take a few moments...');
await importer.import({
moduleNamePrefix: importSpec.moduleNamePrefix,
moduleNamePrefix: spec.moduleNamePrefix,
...options,
});
if (options.save ?? true) {
const specStr = `${spec.moduleNamePrefix}${PREFIX_DELIM}${spec.source}`;
await addImportToConfig(specStr);
}
}

// Process unprefixed CRD imports together (aggregated)
// This ensures CRDs from the same API group across different sources
// are consolidated into a single module
if (categorized.crdUnprefixed.length > 0) {
// Transform sources to handle crds.dev URLs
const resolvedSpecs = categorized.crdUnprefixed.map(spec => {
const crdsDevUrl = matchCrdsDevUrl(spec.source);
return crdsDevUrl ? { ...spec, source: crdsDevUrl } : spec;
});

console.error('Importing resources, this may take a few moments...');
const importer = await ImportCustomResourceDefinition.fromSpecs(resolvedSpecs);
await importer.import(options);

// Save all sources to config
if (options.save ?? true) {
const spec = importSpec.moduleNamePrefix ? `${importSpec.moduleNamePrefix}${PREFIX_DELIM}${importSpec.source}` : importSpec.source;
await addImportToConfig(spec);
for (const spec of categorized.crdUnprefixed) {
await addImportToConfig(spec.source);
}
}
}
}
Expand Down
Loading