From 28f062ba1815c622ed17718baf3194ecd75d4193 Mon Sep 17 00:00:00 2001 From: Harshal Patil Date: Thu, 13 Oct 2022 14:55:13 +0530 Subject: [PATCH] Add ephemeral storage to kubelet system reserved args Signed-off-by: Harshal Patil --- pkg/controller/kubelet-config/helpers.go | 16 +++++++++++++++- .../files/kubelet-auto-node-sizing-enabled.yaml | 3 ++- .../common/_base/files/kubelet-auto-sizing.yaml | 6 ++++-- .../units/kubelet-auto-node-size.service.yaml | 2 +- .../01-master-kubelet/_base/files/kubelet.yaml | 2 -- .../_base/units/kubelet.service.yaml | 2 +- .../alibabacloud/units/kubelet.service.yaml | 2 +- .../on-prem/units/kubelet.service.yaml | 2 +- .../01-worker-kubelet/_base/files/kubelet.yaml | 2 -- .../_base/units/kubelet.service.yaml | 2 +- .../alibabacloud/units/kubelet.service.yaml | 2 +- .../on-prem/units/kubelet.service.yaml | 2 +- 12 files changed, 28 insertions(+), 15 deletions(-) diff --git a/pkg/controller/kubelet-config/helpers.go b/pkg/controller/kubelet-config/helpers.go index ec0a749d3b..25938028cc 100644 --- a/pkg/controller/kubelet-config/helpers.go +++ b/pkg/controller/kubelet-config/helpers.go @@ -35,6 +35,7 @@ func createNewKubeletDynamicSystemReservedIgnition(autoSystemReserved *bool, use var autoNodeSizing string var systemReservedMemory string var systemReservedCPU string + var systemReservedEphemeralStorage string if autoSystemReserved == nil { autoNodeSizing = "false" @@ -54,7 +55,15 @@ func createNewKubeletDynamicSystemReservedIgnition(autoSystemReserved *bool, use systemReservedCPU = "500m" } - config := fmt.Sprintf("NODE_SIZING_ENABLED=%s\nSYSTEM_RESERVED_MEMORY=%s\nSYSTEM_RESERVED_CPU=%s\n", autoNodeSizing, systemReservedMemory, systemReservedCPU) + if val, ok := userDefinedSystemReserved["ephemeral-storage"]; ok { + systemReservedEphemeralStorage = val + } else { + systemReservedEphemeralStorage = "1Gi" + } + + config := fmt.Sprintf("NODE_SIZING_ENABLED=%s\nSYSTEM_RESERVED_MEMORY=%s\nSYSTEM_RESERVED_CPU=%s\nSYSTEM_RESERVED_ES=%s\n", + autoNodeSizing, systemReservedMemory, systemReservedCPU, systemReservedEphemeralStorage) + r := ctrlcommon.NewIgnFileBytesOverwriting("/etc/node-sizing-enabled.env", []byte(config)) return &r } @@ -447,6 +456,11 @@ func generateKubeletIgnFiles(kubeletConfig *mcfgv1.KubeletConfig, originalKubeCo delete(specKubeletConfig.SystemReserved, "cpu") } + if val, ok := specKubeletConfig.SystemReserved["ephemeral-storage"]; ok { + userDefinedSystemReserved["ephemeral-storage"] = val + delete(specKubeletConfig.SystemReserved, "ephemeral-storage") + } + // FeatureGates must be set from the FeatureGate. // Remove them here to prevent the specKubeletConfig merge overwriting them. specKubeletConfig.FeatureGates = nil diff --git a/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml b/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml index f5c28c1ca9..d1c72a540f 100644 --- a/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml +++ b/templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml @@ -4,4 +4,5 @@ contents: inline: | NODE_SIZING_ENABLED=false SYSTEM_RESERVED_MEMORY=1Gi - SYSTEM_RESERVED_CPU=500m \ No newline at end of file + SYSTEM_RESERVED_CPU=500m + SYSTEM_RESERVED_ES=1Gi \ No newline at end of file diff --git a/templates/common/_base/files/kubelet-auto-sizing.yaml b/templates/common/_base/files/kubelet-auto-sizing.yaml index b158d87847..3b62bbbb18 100644 --- a/templates/common/_base/files/kubelet-auto-sizing.yaml +++ b/templates/common/_base/files/kubelet-auto-sizing.yaml @@ -82,6 +82,7 @@ contents: rm -f ${NODE_SIZES_ENV} dynamic_memory_sizing dynamic_cpu_sizing + echo "SYSTEM_RESERVED_ES=$1" >> ${NODE_SIZES_ENV} #dynamic_ephemeral_sizing #dynamic_pid_sizing } @@ -89,12 +90,13 @@ contents: rm -f ${NODE_SIZES_ENV} echo "SYSTEM_RESERVED_MEMORY=$1" >> ${NODE_SIZES_ENV} echo "SYSTEM_RESERVED_CPU=$2" >> ${NODE_SIZES_ENV} + echo "SYSTEM_RESERVED_ES=$3" >> ${NODE_SIZES_ENV} } if [ $1 == "true" ]; then - dynamic_node_sizing + dynamic_node_sizing $4 elif [ $1 == "false" ]; then - static_node_sizing $2 $3 + static_node_sizing $2 $3 $4 else echo "Unrecognized command line option. Valid options are \"true\" or \"false\"" fi diff --git a/templates/common/_base/units/kubelet-auto-node-size.service.yaml b/templates/common/_base/units/kubelet-auto-node-size.service.yaml index 52dce600df..96485f4390 100644 --- a/templates/common/_base/units/kubelet-auto-node-size.service.yaml +++ b/templates/common/_base/units/kubelet-auto-node-size.service.yaml @@ -11,6 +11,6 @@ contents: | Type=oneshot RemainAfterExit=yes EnvironmentFile=/etc/node-sizing-enabled.env - ExecStart=/bin/bash /usr/local/sbin/dynamic-system-reserved-calc.sh ${NODE_SIZING_ENABLED} ${SYSTEM_RESERVED_MEMORY} ${SYSTEM_RESERVED_CPU} + ExecStart=/bin/bash /usr/local/sbin/dynamic-system-reserved-calc.sh ${NODE_SIZING_ENABLED} ${SYSTEM_RESERVED_MEMORY} ${SYSTEM_RESERVED_CPU} ${SYSTEM_RESERVED_ES} [Install] RequiredBy=kubelet.service diff --git a/templates/master/01-master-kubelet/_base/files/kubelet.yaml b/templates/master/01-master-kubelet/_base/files/kubelet.yaml index a47d676ded..fa0d9546a0 100644 --- a/templates/master/01-master-kubelet/_base/files/kubelet.yaml +++ b/templates/master/01-master-kubelet/_base/files/kubelet.yaml @@ -23,8 +23,6 @@ contents: serializeImagePulls: false staticPodPath: /etc/kubernetes/manifests systemCgroups: /system.slice - systemReserved: - ephemeral-storage: 1Gi featureGates: APIPriorityAndFairness: true RotateKubeletServerCertificate: true diff --git a/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml b/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml index cecfe8d2e6..c68df30f19 100644 --- a/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/_base/units/kubelet.service.yaml @@ -43,7 +43,7 @@ contents: | --provider-id=${KUBELET_PROVIDERID} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml b/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml index 72f595c1a0..24ec97db51 100644 --- a/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/alibabacloud/units/kubelet.service.yaml @@ -42,7 +42,7 @@ contents: | --provider-id=alicloud://${KUBELET_NODE_NAME} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml b/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml index 86df14494a..878d0e7925 100644 --- a/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml +++ b/templates/master/01-master-kubelet/on-prem/units/kubelet.service.yaml @@ -40,7 +40,7 @@ contents: | --hostname-override=${KUBELET_NODE_NAME} \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml b/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml index a47d676ded..fa0d9546a0 100644 --- a/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml +++ b/templates/worker/01-worker-kubelet/_base/files/kubelet.yaml @@ -23,8 +23,6 @@ contents: serializeImagePulls: false staticPodPath: /etc/kubernetes/manifests systemCgroups: /system.slice - systemReserved: - ephemeral-storage: 1Gi featureGates: APIPriorityAndFairness: true RotateKubeletServerCertificate: true diff --git a/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml index 825a86e66b..92eea2a811 100644 --- a/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/_base/units/kubelet.service.yaml @@ -42,7 +42,7 @@ contents: | --hostname-override=${KUBELET_NODE_NAME} \ --provider-id=${KUBELET_PROVIDERID} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml index 7ae585e63d..b12848a11a 100644 --- a/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/alibabacloud/units/kubelet.service.yaml @@ -41,7 +41,7 @@ contents: | {{cloudConfigFlag . }} \ --provider-id=alicloud://${KUBELET_NODE_NAME} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always diff --git a/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml b/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml index 29d8d05715..fd00333640 100644 --- a/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml +++ b/templates/worker/01-worker-kubelet/on-prem/units/kubelet.service.yaml @@ -39,7 +39,7 @@ contents: | {{cloudConfigFlag . }} \ --hostname-override=${KUBELET_NODE_NAME} \ --pod-infra-container-image={{.Images.infraImageKey}} \ - --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY} \ + --system-reserved=cpu=${SYSTEM_RESERVED_CPU},memory=${SYSTEM_RESERVED_MEMORY},ephemeral-storage=${SYSTEM_RESERVED_ES} \ --v=${KUBELET_LOG_LEVEL} Restart=always