[OCPNODE-673] Add kubeletconfig to bootstrap mode

pkg/controller/bootstrap/bootstrap.go

@@ -74,6 +74,7 @@ func (b *Bootstrap) Run(destDir string) error {
 
 	var cconfig *mcfgv1.ControllerConfig
 	var featureGate *apicfgv1.FeatureGate
+	var kconfigs []*mcfgv1.KubeletConfig
 	var pools []*mcfgv1.MachineConfigPool
 	var configs []*mcfgv1.MachineConfig
 	var icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy
@@ -112,6 +113,8 @@ func (b *Bootstrap) Run(destDir string) error {
 				configs = append(configs, obj)
 			case *mcfgv1.ControllerConfig:
 				cconfig = obj
+			case *mcfgv1.KubeletConfig:
+				kconfigs = append(kconfigs, obj)
 			case *apioperatorsv1alpha1.ImageContentSourcePolicy:
 				icspRules = append(icspRules, obj)
 			case *apicfgv1.Image:
@@ -139,6 +142,7 @@ func (b *Bootstrap) Run(destDir string) error {
 	if err != nil {
 		return err
 	}
+
 	configs = append(configs, rconfigs...)
 
 	if featureGate != nil {
@@ -148,6 +152,13 @@ func (b *Bootstrap) Run(destDir string) error {
 		}
 		configs = append(configs, kConfigs...)
 	}
+	if len(kconfigs) > 0 {
+		kconfigs, err := kubeletconfig.RunKubeletBootstrap(b.templatesDir, kconfigs, cconfig, featureGate, pools)
+		if err != nil {
+			return err
+		}
+		configs = append(configs, kconfigs...)
+	}
 
 	fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig)
 	if err != nil {

pkg/controller/common/constants.go

@@ -19,6 +19,9 @@ const (
 	// MCNameSuffixAnnotationKey is used to keep track of the machine config name associated with a CR
 	MCNameSuffixAnnotationKey = "machineconfiguration.openshift.io/mc-name-suffix"
 
+	// MaxMCNameSuffix is the maximum value of the name suffix of the machine config associated with kubeletconfig and containerruntime objects
+	MaxMCNameSuffix int = 9
+
 	// ClusterFeatureInstanceName is a singleton name for featureGate configuration
 	ClusterFeatureInstanceName = "cluster"
 )

pkg/controller/kubelet-config/helpers.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 
 	ign3types "github.com/coreos/ignition/v2/config/v3_2/types"
+	"github.com/imdario/mergo"
 	osev1 "github.com/openshift/api/config/v1"
 	"github.com/vincent-petithory/dataurl"
 	corev1 "k8s.io/api/core/v1"
@@ -206,7 +207,7 @@ func getManagedKubeletConfigKey(pool *mcfgv1.MachineConfigPool, client mcfgclien
 	// then if the user creates a kc-new it will map to mc-3. This is what we want as the latest kubelet config created should be higher in priority
 	// so that those changes can be rolled out to the nodes. But users will have to be mindful of how many kubelet config CRs they create. Don't think
 	// anyone should ever have the need to create 10 when they can simply update an existing kubelet config unless it is to apply to another pool.
-	if suffixNum+1 > 9 {
+	if suffixNum+1 > ctrlcommon.MaxMCNameSuffix {
 		return "", fmt.Errorf("max number of supported kubelet config (10) has been reached. Please delete old kubelet configs before retrying")
 	}
 	// Return the default MC name with the suffixNum+1 value appended to it
@@ -336,3 +337,57 @@ func kubeletConfigToIgnFile(cfg *kubeletconfigv1beta1.KubeletConfiguration) (*ig
 	cfgIgn := createNewKubeletIgnition(cfgJSON)
 	return cfgIgn, nil
 }
+
+// generateKubeletIgnFiles generates the Ignition files from the kubelet config
+func generateKubeletIgnFiles(kubeletConfig *mcfgv1.KubeletConfig, originalKubeConfig *kubeletconfigv1beta1.KubeletConfiguration, userDefinedSystemReserved map[string]string) (*ign3types.File, *ign3types.File, *ign3types.File, error) {
+	var (
+		kubeletIgnition            *ign3types.File
+		logLevelIgnition           *ign3types.File
+		autoSizingReservedIgnition *ign3types.File
+	)
+
+	if kubeletConfig.Spec.KubeletConfig != nil && kubeletConfig.Spec.KubeletConfig.Raw != nil {
+		specKubeletConfig, err := decodeKubeletConfig(kubeletConfig.Spec.KubeletConfig.Raw)
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("could not deserialize the new Kubelet config: %v", err)
+		}
+
+		if val, ok := specKubeletConfig.SystemReserved["memory"]; ok {
+			userDefinedSystemReserved["memory"] = val
+			delete(specKubeletConfig.SystemReserved, "memory")
+		}
+
+		if val, ok := specKubeletConfig.SystemReserved["cpu"]; ok {
+			userDefinedSystemReserved["cpu"] = val
+			delete(specKubeletConfig.SystemReserved, "cpu")
+		}
+
+		// FeatureGates must be set from the FeatureGate.
+		// Remove them here to prevent the specKubeletConfig merge overwriting them.
+		specKubeletConfig.FeatureGates = nil
+
+		// Merge the Old and New
+		err = mergo.Merge(originalKubeConfig, specKubeletConfig, mergo.WithOverride)
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("could not merge original config and new config: %v", err)
+		}
+	}
+
+	// Encode the new config into an Ignition File
+	kubeletIgnition, err := kubeletConfigToIgnFile(originalKubeConfig)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("could not encode JSON: %v", err)
+	}
+
+	if kubeletConfig.Spec.LogLevel != nil {
+		logLevelIgnition = createNewKubeletLogLevelIgnition(*kubeletConfig.Spec.LogLevel)
+	}
+	if kubeletConfig.Spec.AutoSizingReserved != nil && len(userDefinedSystemReserved) == 0 {
+		autoSizingReservedIgnition = createNewKubeletDynamicSystemReservedIgnition(kubeletConfig.Spec.AutoSizingReserved, userDefinedSystemReserved)
+	}
+	if len(userDefinedSystemReserved) > 0 {
+		autoSizingReservedIgnition = createNewKubeletDynamicSystemReservedIgnition(nil, userDefinedSystemReserved)
+	}
+
+	return kubeletIgnition, logLevelIgnition, autoSizingReservedIgnition, nil
+}

pkg/controller/kubelet-config/kubelet_config_bootstrap.go

@@ -0,0 +1,111 @@
+package kubeletconfig
+
+import (
+	"encoding/json"
+	"fmt"
+
+	configv1 "github.com/openshift/api/config/v1"
+	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
+	"github.com/openshift/machine-config-operator/pkg/version"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+// RunKubeletBootstrap generates MachineConfig objects for mcpPools that would have been generated by syncKubeletConfig
+func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletConfig, controllerConfig *mcfgv1.ControllerConfig, features *configv1.FeatureGate, mcpPools []*mcfgv1.MachineConfigPool) ([]*mcfgv1.MachineConfig, error) {
+	var res []*mcfgv1.MachineConfig
+	managedKeyExist := make(map[string]bool)
+	userDefinedSystemReserved := make(map[string]string)
+	// Validate the KubeletConfig CR if exists
+	for _, kubeletConfig := range kubeletConfigs {
+		if err := validateUserKubeletConfig(kubeletConfig); err != nil {
+			return nil, err
+		}
+	}
+
+	for _, kubeletConfig := range kubeletConfigs {
+		// use selector since label matching part of a KubeletConfig is not handled during the bootstrap
+		selector, err := metav1.LabelSelectorAsSelector(kubeletConfig.Spec.MachineConfigPoolSelector)
+		if err != nil {
+			return nil, fmt.Errorf("invalid label selector: %v", err)
+		}
+
+		for _, pool := range mcpPools {
+			// If a pool with a nil or empty selector creeps in, it should match nothing, not everything.
+			// skip the pool if no matched label for kubeletconfig
+			if selector.Empty() || !selector.Matches(labels.Set(pool.Labels)) {
+				continue
+			}
+			role := pool.Name
+
+			originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, features)
+			if err != nil {
+				return nil, err
+			}
+			if kubeletConfig.Spec.TLSSecurityProfile != nil {
+				// Inject TLS Options from Spec
+				observedMinTLSVersion, observedCipherSuites := getSecurityProfileCiphers(kubeletConfig.Spec.TLSSecurityProfile)
+				originalKubeConfig.TLSMinVersion = observedMinTLSVersion
+				originalKubeConfig.TLSCipherSuites = observedCipherSuites
+			}
+
+			kubeletIgnition, logLevelIgnition, autoSizingReservedIgnition, err := generateKubeletIgnFiles(kubeletConfig, originalKubeConfig, userDefinedSystemReserved)
+			if err != nil {
+				return nil, err
+			}
+
+			tempIgnConfig := ctrlcommon.NewIgnConfig()
+			if autoSizingReservedIgnition != nil {
+				tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *autoSizingReservedIgnition)
+			}
+			if logLevelIgnition != nil {
+				tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *logLevelIgnition)
+			}
+			if kubeletIgnition != nil {
+				tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *kubeletIgnition)
+			}
+
+			rawIgn, err := json.Marshal(tempIgnConfig)
+			if err != nil {
+				return nil, err
+			}
+			managedKey, err := generateBootstrapManagedKeyKubelet(pool, managedKeyExist)
+			if err != nil {
+				return nil, err
+			}
+			ignConfig := ctrlcommon.NewIgnConfig()
+			mc, err := ctrlcommon.MachineConfigFromIgnConfig(role, managedKey, ignConfig)
+			if err != nil {
+				return nil, fmt.Errorf("could not create MachineConfig from new Ignition config: %v", err)
+			}
+			mc.Spec.Config.Raw = rawIgn
+			mc.SetAnnotations(map[string]string{
+				ctrlcommon.GeneratedByControllerVersionAnnotationKey: version.Hash,
+			})
+			oref := metav1.OwnerReference{
+				APIVersion: controllerKind.GroupVersion().String(),
+				Kind:       controllerKind.Kind,
+			}
+			mc.SetOwnerReferences([]metav1.OwnerReference{oref})
+			res = append(res, mc)
+		}
+	}
+	return res, nil
+}
+
+// generateBootstrapManagedKeyKubelet generates the machine config name for a CR during bootstrap, returns error if there's more than 1 kubeletconfigs fir the same pool
+// Note: Only one kubeletconfig manifest per pool is allowed for bootstrap mode for the following reason:
+// if you provide multiple per pool, they would overwrite each other and not merge, potentially confusing customers post install;
+// we can simplify the logic for the bootstrap generation and avoid some edge cases.
+func generateBootstrapManagedKeyKubelet(pool *mcfgv1.MachineConfigPool, managedKeyExist map[string]bool) (string, error) {
+	if _, ok := managedKeyExist[pool.Name]; ok {
+		return "", fmt.Errorf("Error found multiple KubeletConfigs targeting MachineConfigPool %v. Please apply only one KubeletConfig manifest for each pool during installation", pool.Name)
+	}
+	managedKey, err := ctrlcommon.GetManagedKey(pool, nil, "99", "kubelet", "")
+	if err != nil {
+		return "", err
+	}
+	managedKeyExist[pool.Name] = true
+	return managedKey, nil
+}