diff --git a/pkg/controller/kubelet-config/helpers.go b/pkg/controller/kubelet-config/helpers.go index 1e3aab5bb0..667a0298fa 100644 --- a/pkg/controller/kubelet-config/helpers.go +++ b/pkg/controller/kubelet-config/helpers.go @@ -28,6 +28,7 @@ func createNewKubeletDynamicSystemReservedIgnition(autoSystemReserved *bool, use var autoNodeSizing string var systemReservedMemory string var systemReservedCPU string + var systemReservedEphemeralStorage string if autoSystemReserved == nil { autoNodeSizing = "false" @@ -35,19 +36,26 @@ func createNewKubeletDynamicSystemReservedIgnition(autoSystemReserved *bool, use autoNodeSizing = strconv.FormatBool(*autoSystemReserved) } - if val, ok := userDefinedSystemReserved["memory"]; ok { + if val, ok := userDefinedSystemReserved["memory"]; ok && val != "" { systemReservedMemory = val } else { systemReservedMemory = "1Gi" } - if val, ok := userDefinedSystemReserved["cpu"]; ok { + if val, ok := userDefinedSystemReserved["cpu"]; ok && val != "" { systemReservedCPU = val } else { systemReservedCPU = "500m" } - config := fmt.Sprintf("NODE_SIZING_ENABLED=%s\nSYSTEM_RESERVED_MEMORY=%s\nSYSTEM_RESERVED_CPU=%s\n", autoNodeSizing, systemReservedMemory, systemReservedCPU) + if val, ok := userDefinedSystemReserved["ephemeral-storage"]; ok && val != "" { + systemReservedEphemeralStorage = val + } else { + systemReservedEphemeralStorage = "1Gi" + } + + config := fmt.Sprintf("NODE_SIZING_ENABLED=%s\nSYSTEM_RESERVED_MEMORY=%s\nSYSTEM_RESERVED_CPU=%s\nSYSTEM_RESERVED_ES=%s\n", + autoNodeSizing, systemReservedMemory, systemReservedCPU, systemReservedEphemeralStorage) mode := 0644 overwrite := true diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index e69a18cecc..d84b4abf27 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -438,6 +438,7 @@ func (ctrl *Controller) addAnnotation(cfg *mcfgv1.KubeletConfig, annotationKey, // syncKubeletConfig will sync the kubeletconfig with the given key. // This function is not meant to be invoked concurrently with the same key. +// //nolint:gocyclo func (ctrl *Controller) syncKubeletConfig(key string) error { startTime := time.Now() @@ -578,6 +579,11 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { delete(specKubeletConfig.SystemReserved, "cpu") } + if val, ok := specKubeletConfig.SystemReserved["ephemeral-storage"]; ok { + userDefinedSystemReserved["ephemeral-storage"] = val + delete(specKubeletConfig.SystemReserved, "ephemeral-storage") + } + // FeatureGates must be set from the FeatureGate. // Remove them here to prevent the specKubeletConfig merge overwriting them. specKubeletConfig.FeatureGates = nil diff --git a/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml b/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml index f5c28c1ca9..d1c72a540f 100644 --- a/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml +++ b/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml @@ -4,4 +4,5 @@ contents: inline: | NODE_SIZING_ENABLED=false SYSTEM_RESERVED_MEMORY=1Gi - SYSTEM_RESERVED_CPU=500m \ No newline at end of file + SYSTEM_RESERVED_CPU=500m + SYSTEM_RESERVED_ES=1Gi \ No newline at end of file diff --git a/templates/common/_base/files/kubelet-auto-sizing.yaml b/templates/common/_base/files/kubelet-auto-sizing.yaml index 747180383d..2b257187cc 100644 --- a/templates/common/_base/files/kubelet-auto-sizing.yaml +++ b/templates/common/_base/files/kubelet-auto-sizing.yaml @@ -77,23 +77,46 @@ contents: function dynamic_pid_sizing { echo "Not implemented yet" } + function set_memory { + SYSTEM_RESERVED_MEMORY=$1 + if [ -z "${SYSTEM_RESERVED_MEMORY}" ]; then + SYSTEM_RESERVED_MEMORY="1Gi" + fi + echo "SYSTEM_RESERVED_MEMORY=${SYSTEM_RESERVED_MEMORY}" >> ${NODE_SIZES_ENV} + } + function set_cpu { + SYSTEM_RESERVED_CPU=$1 + if [ -z "${SYSTEM_RESERVED_CPU}" ]; then + SYSTEM_RESERVED_CPU="500m" + fi + echo "SYSTEM_RESERVED_CPU=${SYSTEM_RESERVED_CPU}" >> ${NODE_SIZES_ENV} + } + function set_es { + SYSTEM_RESERVED_ES=$1 + if [ -z "${SYSTEM_RESERVED_ES}" ]; then + SYSTEM_RESERVED_ES="1Gi" + fi + echo "SYSTEM_RESERVED_ES=${SYSTEM_RESERVED_ES}" >> ${NODE_SIZES_ENV} + } function dynamic_node_sizing { rm -f ${NODE_SIZES_ENV} dynamic_memory_sizing dynamic_cpu_sizing + set_es $1 #dynamic_ephemeral_sizing #dynamic_pid_sizing } function static_node_sizing { rm -f ${NODE_SIZES_ENV} - echo "SYSTEM_RESERVED_MEMORY=$1" >> ${NODE_SIZES_ENV} - echo "SYSTEM_RESERVED_CPU=$2" >> ${NODE_SIZES_ENV} + set_memory $1 + set_cpu $2 + set_es $3 } if [ $1 == "true" ]; then - dynamic_node_sizing + dynamic_node_sizing $4 elif [ $1 == "false" ]; then - static_node_sizing $2 $3 + static_node_sizing $2 $3 $4 else echo "Unrecongnized command line option. Valid options are \"true\" or \"false\"" fi diff --git a/templates/common/_base/units/kubelet-auto-node-size.service.yaml b/templates/common/_base/units/kubelet-auto-node-size.service.yaml index 52dce600df..96485f4390 100644 --- a/templates/common/_base/units/kubelet-auto-node-size.service.yaml +++ b/templates/common/_base/units/kubelet-auto-node-size.service.yaml @@ -11,6 +11,6 @@ contents: | Type=oneshot RemainAfterExit=yes EnvironmentFile=/etc/node-sizing-enabled.env - ExecStart=/bin/bash /usr/local/sbin/dynamic-system-reserved-calc.sh ${NODE_SIZING_ENABLED} ${SYSTEM_RESERVED_MEMORY} ${SYSTEM_RESERVED_CPU} + ExecStart=/bin/bash /usr/local/sbin/dynamic-system-reserved-calc.sh ${NODE_SIZING_ENABLED} ${SYSTEM_RESERVED_MEMORY} ${SYSTEM_RESERVED_CPU} ${SYSTEM_RESERVED_ES} [Install] RequiredBy=kubelet.service diff --git a/templates/master/01-master-kubelet/_base/files/kubelet.yaml b/templates/master/01-master-kubelet/_base/files/kubelet.yaml index 4181bbe2cf..6be322eb6d 100644 --- a/templates/master/01-master-kubelet/_base/files/kubelet.yaml +++ b/templates/master/01-master-kubelet/_base/files/kubelet.yaml @@ -22,8 +22,6 @@ contents: serializeImagePulls: false staticPodPath: /etc/kubernetes/manifests systemCgroups: /system.slice - systemReserved: - ephemeral-storage: 1Gi featureGates: APIPriorityAndFairness: true LegacyNodeRoleBehavior: false diff --git a/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml b/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml index 3d52dff415..4bb9f50d9b 100644 --- a/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml @@ -42,7 +42,7 @@ contents: | --hostname-override=${KUBELET_NODE_NAME} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml b/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml index 113a1c98f8..75d8227c00 100644 --- a/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml @@ -40,7 +40,7 @@ contents: | --hostname-override=${KUBELET_NODE_NAME} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml b/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml index 4181bbe2cf..6be322eb6d 100644 --- a/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml +++ b/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml @@ -22,8 +22,6 @@ contents: serializeImagePulls: false staticPodPath: /etc/kubernetes/manifests systemCgroups: /system.slice - systemReserved: - ephemeral-storage: 1Gi featureGates: APIPriorityAndFairness: true LegacyNodeRoleBehavior: false diff --git a/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml index 787337f1fc..eb214d2bed 100644 --- a/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml @@ -41,7 +41,7 @@ contents: | {{cloudConfigFlag . }} \ --hostname-override=${KUBELET_NODE_NAME} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml index ddb91e0c4a..731fc00d7e 100644 --- a/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml @@ -39,7 +39,7 @@ contents: | {{cloudConfigFlag . }} \ --hostname-override=${KUBELET_NODE_NAME} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always