From d09be517ee6cc7184e1077ffd09e9a9f0b8c8ae1 Mon Sep 17 00:00:00 2001 From: Harshal Patil Date: Thu, 13 Oct 2022 14:55:13 +0530 Subject: [PATCH] Add ephemeral storage to kubelet system reserved args Signed-off-by: Harshal Patil --- pkg/controller/kubelet-config/helpers.go | 15 ++++++++++++++- .../files/kubelet-auto-node-sizing-enabled.yaml | 3 ++- .../common/_base/files/kubelet-auto-sizing.yaml | 6 ++++-- .../units/kubelet-auto-node-size.service.yaml | 2 +- .../01-master-kubelet/_base/files/kubelet.yaml | 2 -- .../_base/units/kubelet.service.yaml | 2 +- .../alibabacloud/units/kubelet.service.yaml | 2 +- .../on-prem/units/kubelet.service.yaml | 2 +- .../01-worker-kubelet/_base/files/kubelet.yaml | 2 -- .../_base/units/kubelet.service.yaml | 2 +- .../alibabacloud/units/kubelet.service.yaml | 2 +- .../on-prem/units/kubelet.service.yaml | 2 +- 12 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pkg/controller/kubelet-config/helpers.go b/pkg/controller/kubelet-config/helpers.go index d65797aeb9..350fa13d89 100644 --- a/pkg/controller/kubelet-config/helpers.go +++ b/pkg/controller/kubelet-config/helpers.go @@ -29,6 +29,7 @@ func createNewKubeletDynamicSystemReservedIgnition(autoSystemReserved *bool, use var autoNodeSizing string var systemReservedMemory string var systemReservedCPU string + var systemReservedEphemeralStorage string if autoSystemReserved == nil { autoNodeSizing = "false" @@ -48,7 +49,14 @@ func createNewKubeletDynamicSystemReservedIgnition(autoSystemReserved *bool, use systemReservedCPU = "500m" } - config := fmt.Sprintf("NODE_SIZING_ENABLED=%s\nSYSTEM_RESERVED_MEMORY=%s\nSYSTEM_RESERVED_CPU=%s\n", autoNodeSizing, systemReservedMemory, systemReservedCPU) + if val, ok := userDefinedSystemReserved["ephemeral-storage"]; ok { + systemReservedEphemeralStorage = val + } else { + systemReservedEphemeralStorage = "1Gi" + } + + config := fmt.Sprintf("NODE_SIZING_ENABLED=%s\nSYSTEM_RESERVED_MEMORY=%s\nSYSTEM_RESERVED_CPU=%s\nSYSTEM_RESERVED_ES=%s\n", + autoNodeSizing, systemReservedMemory, systemReservedCPU, systemReservedEphemeralStorage) mode := 0644 overwrite := true @@ -366,6 +374,11 @@ func generateKubeletIgnFiles(kubeletConfig *mcfgv1.KubeletConfig, originalKubeCo delete(specKubeletConfig.SystemReserved, "cpu") } + if val, ok := specKubeletConfig.SystemReserved["ephemeral-storage"]; ok { + userDefinedSystemReserved["ephemeral-storage"] = val + delete(specKubeletConfig.SystemReserved, "ephemeral-storage") + } + // FeatureGates must be set from the FeatureGate. // Remove them here to prevent the specKubeletConfig merge overwriting them. specKubeletConfig.FeatureGates = nil diff --git a/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml b/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml index f5c28c1ca9..d1c72a540f 100644 --- a/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml +++ b/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml @@ -4,4 +4,5 @@ contents: inline: | NODE_SIZING_ENABLED=false SYSTEM_RESERVED_MEMORY=1Gi - SYSTEM_RESERVED_CPU=500m \ No newline at end of file + SYSTEM_RESERVED_CPU=500m + SYSTEM_RESERVED_ES=1Gi \ No newline at end of file diff --git a/templates/common/_base/files/kubelet-auto-sizing.yaml b/templates/common/_base/files/kubelet-auto-sizing.yaml index 747180383d..b65812aa47 100644 --- a/templates/common/_base/files/kubelet-auto-sizing.yaml +++ b/templates/common/_base/files/kubelet-auto-sizing.yaml @@ -81,6 +81,7 @@ contents: rm -f ${NODE_SIZES_ENV} dynamic_memory_sizing dynamic_cpu_sizing + echo "SYSTEM_RESERVED_ES=$1" >> ${NODE_SIZES_ENV} #dynamic_ephemeral_sizing #dynamic_pid_sizing } @@ -88,12 +89,13 @@ contents: rm -f ${NODE_SIZES_ENV} echo "SYSTEM_RESERVED_MEMORY=$1" >> ${NODE_SIZES_ENV} echo "SYSTEM_RESERVED_CPU=$2" >> ${NODE_SIZES_ENV} + echo "SYSTEM_RESERVED_ES=$3" >> ${NODE_SIZES_ENV} } if [ $1 == "true" ]; then - dynamic_node_sizing + dynamic_node_sizing $4 elif [ $1 == "false" ]; then - static_node_sizing $2 $3 + static_node_sizing $2 $3 $4 else echo "Unrecongnized command line option. Valid options are \"true\" or \"false\"" fi diff --git a/templates/common/_base/units/kubelet-auto-node-size.service.yaml b/templates/common/_base/units/kubelet-auto-node-size.service.yaml index 52dce600df..96485f4390 100644 --- a/templates/common/_base/units/kubelet-auto-node-size.service.yaml +++ b/templates/common/_base/units/kubelet-auto-node-size.service.yaml @@ -11,6 +11,6 @@ contents: | Type=oneshot RemainAfterExit=yes EnvironmentFile=/etc/node-sizing-enabled.env - ExecStart=/bin/bash /usr/local/sbin/dynamic-system-reserved-calc.sh ${NODE_SIZING_ENABLED} ${SYSTEM_RESERVED_MEMORY} ${SYSTEM_RESERVED_CPU} + ExecStart=/bin/bash /usr/local/sbin/dynamic-system-reserved-calc.sh ${NODE_SIZING_ENABLED} ${SYSTEM_RESERVED_MEMORY} ${SYSTEM_RESERVED_CPU} ${SYSTEM_RESERVED_ES} [Install] RequiredBy=kubelet.service diff --git a/templates/master/01-master-kubelet/_base/files/kubelet.yaml b/templates/master/01-master-kubelet/_base/files/kubelet.yaml index 556f458fc4..2f010c6b8f 100644 --- a/templates/master/01-master-kubelet/_base/files/kubelet.yaml +++ b/templates/master/01-master-kubelet/_base/files/kubelet.yaml @@ -22,8 +22,6 @@ contents: serializeImagePulls: false staticPodPath: /etc/kubernetes/manifests systemCgroups: /system.slice - systemReserved: - ephemeral-storage: 1Gi featureGates: APIPriorityAndFairness: true LegacyNodeRoleBehavior: false diff --git a/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml b/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml index 3d52dff415..4bb9f50d9b 100644 --- a/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml @@ -42,7 +42,7 @@ contents: | --hostname-override=${KUBELET_NODE_NAME} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml b/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml index 422e54a9df..ec0e7fa811 100644 --- a/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml @@ -42,7 +42,7 @@ contents: | --provider-id=alicloud://${KUBELET_NODE_NAME} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml b/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml index 113a1c98f8..75d8227c00 100644 --- a/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml @@ -40,7 +40,7 @@ contents: | --hostname-override=${KUBELET_NODE_NAME} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml b/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml index 556f458fc4..2f010c6b8f 100644 --- a/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml +++ b/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml @@ -22,8 +22,6 @@ contents: serializeImagePulls: false staticPodPath: /etc/kubernetes/manifests systemCgroups: /system.slice - systemReserved: - ephemeral-storage: 1Gi featureGates: APIPriorityAndFairness: true LegacyNodeRoleBehavior: false diff --git a/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml index 787337f1fc..eb214d2bed 100644 --- a/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml @@ -41,7 +41,7 @@ contents: | {{cloudConfigFlag . }} \ --hostname-override=${KUBELET_NODE_NAME} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml index f545225e0f..f54c76fe54 100644 --- a/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml @@ -41,7 +41,7 @@ contents: | {{cloudConfigFlag . }} \ --provider-id=alicloud://${KUBELET_NODE_NAME} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml index ddb91e0c4a..731fc00d7e 100644 --- a/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml @@ -39,7 +39,7 @@ contents: | {{cloudConfigFlag . }} \ --hostname-override=${KUBELET_NODE_NAME} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always