From 16eaa98154b78f65089148fe715e1a5530d0be55 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Mon, 26 Mar 2018 16:51:03 +0100 Subject: [PATCH 1/5] Set kubelet eviction-hard based on total memory The kubelet will default to memory.available<100Mi as the hard eviction threshold. This makes that figure 5% of the total memory or 100Mi. Whichever is greater. Read from the default memory facts to set the kubelet `eviction-hard` parameter. Default facts: https://puppet.com/docs/facter/3.10/core_facts.html#memory Implements https://github.com/jetstack/tarmak/issues/149 --- .../functions/five_percent_of_total_ram.rb | 16 ++++++++++++++++ puppet/modules/kubernetes/manifests/kubelet.pp | 2 ++ .../kubernetes/spec/classes/kubelet_spec.rb | 1 + puppet/modules/kubernetes/spec/spec_helper.rb | 5 +++++ .../kubernetes/templates/kubelet.service.erb | 1 + 5 files changed, 25 insertions(+) create mode 100644 puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb diff --git a/puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb b/puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb new file mode 100644 index 0000000000..8f4cff9774 --- /dev/null +++ b/puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb @@ -0,0 +1,16 @@ +Puppet::Functions.create_function(:five_percent_of_total_ram) do + dispatch :five_percent_of_total_ram do + param 'Integer', :total_bytes + end + + def five_percent_of_total_ram(total_bytes) + five_percent = (total_bytes * 0.05 / 1_000_000).round + default = 100 + + if five_percent < default + '100Mi' + else + five_percent.to_s + 'Mi' + end + end +end diff --git a/puppet/modules/kubernetes/manifests/kubelet.pp b/puppet/modules/kubernetes/manifests/kubelet.pp index 2ae8b00e9a..756fb14ebb 100644 --- a/puppet/modules/kubernetes/manifests/kubelet.pp +++ b/puppet/modules/kubernetes/manifests/kubelet.pp @@ -10,6 +10,8 @@ String $role = 'worker', String $container_runtime = 'docker', String $kubelet_dir = '/var/lib/kubelet', + String $hard_eviction_memory_threshold = + five_percent_of_total_ram(dig44($facts, ['memory', 'system', 'total_bytes'], 1)), Optional[String] $network_plugin = undef, Integer $network_plugin_mtu = 1460, Boolean $allow_privileged = true, diff --git a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb index 74469e11ba..b7b81c57b4 100644 --- a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb +++ b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb @@ -20,6 +20,7 @@ should_not contain_file(service_file).with_content(/--network-plugin/) should contain_file(service_file).with_content(/--container-runtime=docker/) should contain_file(service_file).with_content(%r{--kubeconfig=/etc/kubernetes/kubeconfig-kubelet}) + should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<200Mi}) end end diff --git a/puppet/modules/kubernetes/spec/spec_helper.rb b/puppet/modules/kubernetes/spec/spec_helper.rb index 26eeef08c9..4c666e9d8e 100644 --- a/puppet/modules/kubernetes/spec/spec_helper.rb +++ b/puppet/modules/kubernetes/spec/spec_helper.rb @@ -5,5 +5,10 @@ :path => '/bin:/sbin:/usr/bin:/usr/sbin:/opt/bin', :osfamily => 'RedHat', :kernelversion => '3.11.1', + :memory => { + :system => { + :total_bytes => 4_000_000_000, + } + } } end diff --git a/puppet/modules/kubernetes/templates/kubelet.service.erb b/puppet/modules/kubernetes/templates/kubelet.service.erb index d32b2e8b91..154c7ea988 100644 --- a/puppet/modules/kubernetes/templates/kubelet.service.erb +++ b/puppet/modules/kubernetes/templates/kubelet.service.erb @@ -103,6 +103,7 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \ "--tls-cert-file=<%= @cert_file %>" \ "--tls-private-key-file=<%= @key_file %>" \ <% end -%> + --eviction-hard=memory.available<<%= @hard_eviction_memory_threshold %> --logtostderr=true Restart=on-failure From f22b260517bb2b8839bb8b30ab8c2c24943ab90c Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Tue, 27 Mar 2018 12:40:18 +0100 Subject: [PATCH 2/5] Code review changes --- .../lib/puppet/functions/five_percent_of_total_ram.rb | 2 +- puppet/modules/kubernetes/spec/classes/kubelet_spec.rb | 2 +- puppet/modules/kubernetes/templates/kubelet.service.erb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb b/puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb index 8f4cff9774..9cc7c92b87 100644 --- a/puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb +++ b/puppet/modules/kubernetes/lib/puppet/functions/five_percent_of_total_ram.rb @@ -4,7 +4,7 @@ end def five_percent_of_total_ram(total_bytes) - five_percent = (total_bytes * 0.05 / 1_000_000).round + five_percent = (total_bytes * 0.05 / 1024**2).round default = 100 if five_percent < default diff --git a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb index b7b81c57b4..e101ac8dab 100644 --- a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb +++ b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb @@ -20,7 +20,7 @@ should_not contain_file(service_file).with_content(/--network-plugin/) should contain_file(service_file).with_content(/--container-runtime=docker/) should contain_file(service_file).with_content(%r{--kubeconfig=/etc/kubernetes/kubeconfig-kubelet}) - should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<200Mi}) + should contain_file(service_file).with_content(%r{--eviction-hard="memory.available<191Mi"}) end end diff --git a/puppet/modules/kubernetes/templates/kubelet.service.erb b/puppet/modules/kubernetes/templates/kubelet.service.erb index 154c7ea988..1a46276441 100644 --- a/puppet/modules/kubernetes/templates/kubelet.service.erb +++ b/puppet/modules/kubernetes/templates/kubelet.service.erb @@ -103,7 +103,7 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \ "--tls-cert-file=<%= @cert_file %>" \ "--tls-private-key-file=<%= @key_file %>" \ <% end -%> - --eviction-hard=memory.available<<%= @hard_eviction_memory_threshold %> + --eviction-hard="memory.available<<%= @hard_eviction_memory_threshold %>" --logtostderr=true Restart=on-failure From 9df71dd1fdbaa4c5b10a002f6f2fa93889f55ea0 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Fri, 6 Apr 2018 14:42:20 +0100 Subject: [PATCH 3/5] Format command correctly with backslash --- puppet/modules/kubernetes/templates/kubelet.service.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/kubernetes/templates/kubelet.service.erb b/puppet/modules/kubernetes/templates/kubelet.service.erb index 1a46276441..09dc08869b 100644 --- a/puppet/modules/kubernetes/templates/kubelet.service.erb +++ b/puppet/modules/kubernetes/templates/kubelet.service.erb @@ -103,7 +103,7 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \ "--tls-cert-file=<%= @cert_file %>" \ "--tls-private-key-file=<%= @key_file %>" \ <% end -%> - --eviction-hard="memory.available<<%= @hard_eviction_memory_threshold %>" + --eviction-hard="memory.available<<%= @hard_eviction_memory_threshold %>" \ --logtostderr=true Restart=on-failure From 445c8f3652924d6c4a641763591e93c035f83e5b Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Fri, 6 Apr 2018 17:24:20 +0100 Subject: [PATCH 4/5] Quote whole eviction-hard argument --- puppet/modules/kubernetes/templates/kubelet.service.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/kubernetes/templates/kubelet.service.erb b/puppet/modules/kubernetes/templates/kubelet.service.erb index 09dc08869b..6d12406db8 100644 --- a/puppet/modules/kubernetes/templates/kubelet.service.erb +++ b/puppet/modules/kubernetes/templates/kubelet.service.erb @@ -103,7 +103,7 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \ "--tls-cert-file=<%= @cert_file %>" \ "--tls-private-key-file=<%= @key_file %>" \ <% end -%> - --eviction-hard="memory.available<<%= @hard_eviction_memory_threshold %>" \ + "--eviction-hard=memory.available<<%= @hard_eviction_memory_threshold %>" \ --logtostderr=true Restart=on-failure From b62c827e1d2390a08079cd3f1f2f3e8a7b90e34b Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Tue, 10 Apr 2018 15:43:04 +0100 Subject: [PATCH 5/5] Fix quoted argument in spec --- puppet/modules/kubernetes/spec/classes/kubelet_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb index e101ac8dab..a8dc3ca2a8 100644 --- a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb +++ b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb @@ -20,7 +20,7 @@ should_not contain_file(service_file).with_content(/--network-plugin/) should contain_file(service_file).with_content(/--container-runtime=docker/) should contain_file(service_file).with_content(%r{--kubeconfig=/etc/kubernetes/kubeconfig-kubelet}) - should contain_file(service_file).with_content(%r{--eviction-hard="memory.available<191Mi"}) + should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<191Mi}) end end