diff --git a/chef/cookbooks/postgresql/recipes/config_pgtune.rb b/chef/cookbooks/postgresql/recipes/config_pgtune.rb index 0f7cf2957e..690a23e5e3 100644 --- a/chef/cookbooks/postgresql/recipes/config_pgtune.rb +++ b/chef/cookbooks/postgresql/recipes/config_pgtune.rb @@ -1,5 +1,5 @@ # -# Cookbook Name:: postgresql +# Cookbook:: postgresql # Recipe:: config_pgtune # Author:: David Crane () # @@ -31,7 +31,7 @@ # general range for a dedicated Postgresql system. # # This recipe takes three optional parameters that may be passed in as -# node['postgresql']['config_pgtune'] attributes: +# node["postgresql"]["config_pgtune"] attributes: # * db_type -- Specifies database type as one of: dw, oltp, # web, mixed, desktop. If not specified, the default is mixed. # * max_connections -- Specifies number of maximum connections @@ -40,7 +40,7 @@ # it will be detected from the Ohai automatic attributes. # # Using those inputs, this recipe will compute and set the following -# node.default['postgresql']['config'] attributes: +# node.default["postgresql"]["config"] attributes: # * max_connections # * shared_buffers # * effective_cache_size @@ -58,7 +58,7 @@ ####### # These are the workload characteristics of the five database types -# that can be specified as node['postgresql']['config_pgtune']['db_type']: +# that can be specified as node["postgresql"]["config_pgtune"]["db_type"]: # # dw -- Data Warehouse # * Typically I/O- or RAM-bound @@ -86,36 +86,38 @@ # Parse out db_type option, or use default. db_type = "mixed" -if (node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("db_type")) +if node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("db_type") db_type = node["postgresql"]["config_pgtune"]["db_type"] - if (!(["dw","oltp","web","mixed","desktop"].include?(db_type))) - Chef::Application.fatal!([ - "Bad value (#{db_type})", - "for node['postgresql']['config_pgtune']['db_type'] attribute.", - "Valid values are one of dw, oltp, web, mixed, desktop." - ].join(" ")) + unless %w(dw oltp web mixed desktop).include?(db_type) + Chef::Log.fatal([ + "Bad value (#{db_type})", + "for node['postgresql']['config_pgtune']['db_type'] attribute.", + "Valid values are one of dw, oltp, web, mixed, desktop.", + ].join(" ")) + raise end end # Parse out max_connections option, or use a value based on db_type. con = -{ "web" => 200, - "oltp" => 300, - "dw" => 20, - "mixed" => 80, - "desktop" => 5 -}.fetch(db_type) - -if (node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("max_connections")) - max_connections = node["postgresql"]["config_pgtune"]["max_connections"] - if (max_connections.match(/\A[1-9]\d*\Z/) == nil) - Chef::Application.fatal!([ - "Bad value (#{max_connections})", - "for node['postgresql']['config_pgtune']['max_connections'] attribute.", - "Valid values are non-zero integers only." - ].join(" ")) + { "web" => 200, + "oltp" => 300, + "dw" => 20, + "mixed" => 80, + "desktop" => 5, + }.fetch(db_type) + +if node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("max_connections") + max_connections = node["postgresql"]["config_pgtune"]["max_connections"].to_i + if max_connections <= 0 + Chef::Log.fatal([ + "Bad value (#{max_connections})", + "for node['postgresql']['config_pgtune']['max_connections'] attribute.", + "Valid values are non-zero integers only.", + ].join(" ")) + raise end - con = max_connections.to_i + con = max_connections end # Parse out total_memory option, or use value detected by Ohai. @@ -123,14 +125,14 @@ # Override max_connections with a node attribute if DevOps desires. # For example, on a system *not* dedicated to Postgresql. -if (node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("total_memory")) +if node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("total_memory") total_memory = node["postgresql"]["config_pgtune"]["total_memory"] - if (total_memory.match(/\A[1-9]\d*kB\Z/) == nil) + if total_memory.match(/\A[1-9]\d*kB\Z/).nil? Chef::Application.fatal!([ - "Bad value (#{total_memory})", - "for node['postgresql']['config_pgtune']['total_memory'] attribute.", - "Valid values are non-zero integers followed by kB (e.g., 49416564kB)." - ].join(" ")) + "Bad value (#{total_memory})", + "for node['postgresql']['config_pgtune']['total_memory'] attribute.", + "Valid values are non-zero integers followed by kB (e.g., 49416564kB).", + ].join(" ")) end end @@ -150,77 +152,73 @@ # for low memory systems. In that case, the calculation is skipped, # leaving the built-in Postgresql settings, which are actually # intended for those low memory systems. -if (mem >= 256) +if mem >= 256 # (2) shared_buffers # Sets the number of shared memory buffers used by the server. shared_buffers = - { "web" => mem/4, - "oltp" => mem/4, - "dw" => mem/4, - "mixed" => mem/4, - "desktop" => mem/16 - }.fetch(db_type) + { "web" => mem / 4, + "oltp" => mem / 4, + "dw" => mem / 4, + "mixed" => mem / 4, + "desktop" => mem / 16, + }.fetch(db_type) # Robert Haas has advised to cap the size of shared_buffers based on # the memory architecture: 2GB on 32-bit and 8GB on 64-bit machines. # http://rhaas.blogspot.com/2012/03/tuning-sharedbuffers-and-walbuffers.html case node["kernel"]["machine"] when "i386" # 32-bit machines - if shared_buffers > 2*1024 - shared_buffers = 2*1024 - end + shared_buffers = 2 * 1024 if shared_buffers > 2 * 1024 when "x86_64" # 64-bit machines - if shared_buffers > 8*1024 - shared_buffers = 8*1024 - end + shared_buffers = 8 * 1024 if shared_buffers > 8 * 1024 end - node.default["postgresql"]["config"]["shared_buffers"] = binaryround(shared_buffers*1024*1024) + node.default["postgresql"]["config"]["shared_buffers"] = binaryround(shared_buffers * 1024 * 1024) # (3) effective_cache_size # Sets the planner's assumption about the size of the disk cache. # That is, the portion of the kernel's disk cache that will be # used for PostgreSQL data files. effective_cache_size = - { "web" => mem * 3 / 4, - "oltp" => mem * 3 / 4, - "dw" => mem * 3 / 4, - "mixed" => mem * 3 / 4, - "desktop" => mem / 4 - }.fetch(db_type) + { "web" => mem * 3 / 4, + "oltp" => mem * 3 / 4, + "dw" => mem * 3 / 4, + "mixed" => mem * 3 / 4, + "desktop" => mem / 4, + }.fetch(db_type) - node.default["postgresql"]["config"]["effective_cache_size"] = binaryround(effective_cache_size*1024*1024) + node.default["postgresql"]["config"]["effective_cache_size"] = binaryround(effective_cache_size * 1024 * 1024) # (4) work_mem # Sets the maximum memory to be used for query workspaces. + mem_con_v = (mem.to_f / con).ceil + work_mem = - { "web" => mem / con, - "oltp" => mem / con, - "dw" => mem / con / 2, - "mixed" => mem / con / 2, - "desktop" => mem / con / 6 - }.fetch(db_type) + { "web" => mem_con_v, + "oltp" => mem_con_v, + "dw" => mem_con_v / 2, + "mixed" => mem_con_v / 2, + "desktop" => mem_con_v / 6, + }.fetch(db_type) - node.default["postgresql"]["config"]["work_mem"] = binaryround(work_mem*1024*1024) + node.default["postgresql"]["config"]["work_mem"] = binaryround(work_mem * 1024 * 1024) # (5) maintenance_work_mem # Sets the maximum memory to be used for maintenance operations. # This includes operations such as VACUUM and CREATE INDEX. maintenance_work_mem = - { "web" => mem / 16, - "oltp" => mem / 16, - "dw" => mem / 8, - "mixed" => mem / 16, - "desktop" => mem / 16 - }.fetch(db_type) + { "web" => mem / 16, + "oltp" => mem / 16, + "dw" => mem / 8, + "mixed" => mem / 16, + "desktop" => mem / 16, + }.fetch(db_type) # Cap maintenence RAM at 1GB on servers with lots of memory - if (maintenance_work_mem > 1*1024) - maintenance_work_mem = 1*1024 - end + maintenance_work_mem = 1 * 1024 if maintenance_work_mem > 1 * 1024 - node.default["postgresql"]["config"]["maintenance_work_mem"] = binaryround(maintenance_work_mem*1024*1024) + node.default["postgresql"]["config"]["maintenance_work_mem"] = binaryround(maintenance_work_mem * 1024 * 1024) end @@ -231,25 +229,29 @@ # (6) checkpoint_segments # Sets the maximum distance in log segments between automatic WAL checkpoints. checkpoint_segments = -{ "web" => 8, - "oltp" => 16, - "dw" => 64, - "mixed" => 16, - "desktop" => 3 -}.fetch(db_type) + { "web" => 8, + "oltp" => 16, + "dw" => 64, + "mixed" => 16, + "desktop" => 3, + }.fetch(db_type) -node.default["postgresql"]["config"]["checkpoint_segments"] = checkpoint_segments +if node["postgresql"]["version"].to_f >= 9.5 + node.default["postgresql"]["config"]["max_wal_size"] = ((3 * checkpoint_segments) * 16).to_s + "MB" +else + node.default["postgresql"]["config"]["checkpoint_segments"] = checkpoint_segments +end # (7) checkpoint_completion_target # Time spent flushing dirty buffers during checkpoint, as fraction # of checkpoint interval. checkpoint_completion_target = -{ "web" => "0.7", - "oltp" => "0.9", - "dw" => "0.9", - "mixed" => "0.9", - "desktop" => "0.5" -}.fetch(db_type) + { "web" => "0.7", + "oltp" => "0.9", + "dw" => "0.9", + "mixed" => "0.9", + "desktop" => "0.5", + }.fetch(db_type) node.default["postgresql"]["config"]["checkpoint_completion_target"] = checkpoint_completion_target @@ -260,7 +262,7 @@ if node["postgresql"]["version"].to_f < 9.1 wal_buffers = 512 * checkpoint_segments # The pgtune seems to use 1kB units for wal_buffers - node.default["postgresql"]["config"]["wal_buffers"] = binaryround(wal_buffers*1024) + node.default["postgresql"]["config"]["wal_buffers"] = binaryround(wal_buffers * 1024) else node.default["postgresql"]["config"]["wal_buffers"] = "-1" end @@ -270,11 +272,11 @@ # that have not had a column-specific target set via # ALTER TABLE SET STATISTICS. default_statistics_target = -{ "web" => 100, - "oltp" => 100, - "dw" => 500, - "mixed" => 100, - "desktop" => 100 -}.fetch(db_type) + { "web" => 100, + "oltp" => 100, + "dw" => 500, + "mixed" => 100, + "desktop" => 100, + }.fetch(db_type) node.default["postgresql"]["config"]["default_statistics_target"] = default_statistics_target diff --git a/chef/cookbooks/postgresql/recipes/server.rb b/chef/cookbooks/postgresql/recipes/server.rb index 947acc4a78..d90bc117dd 100644 --- a/chef/cookbooks/postgresql/recipes/server.rb +++ b/chef/cookbooks/postgresql/recipes/server.rb @@ -24,6 +24,7 @@ ::Chef::Recipe.send(:include, Opscode::OpenSSL::Password) include_recipe "postgresql::client" +include_recipe "postgresql::config_pgtune" # For Crowbar, we need to set the address to bind - default to admin node. newaddr = CrowbarDatabaseHelper.get_listen_address(node) diff --git a/chef/data_bags/crowbar/migrate/database/010_pgtune.rb b/chef/data_bags/crowbar/migrate/database/010_pgtune.rb new file mode 100644 index 0000000000..9c4612316d --- /dev/null +++ b/chef/data_bags/crowbar/migrate/database/010_pgtune.rb @@ -0,0 +1,19 @@ +def upgrade(ta, td, a, d) + a["postgresql"]["config_pgtune"] ||= {} + unless a["postgresql"]["config_pgtune"].key?("max_connections") + a["postgresql"]["config_pgtune"]["max_connections"] = a["postgresql"]["config"]["max_connections"] + end + a["postgresql"]["config"].delete("max_connections") + + return a, d +end + +def downgrade(ta, td, a, d) + if ta["postgresql"]["config"].key?("max_connections") + a["postgresql"]["config"]["max_connections"] = a["postgresql"]["config_pgtune"]["max_connections"] + a["postgresql"]["config_pgtune"].delete("max_connections") + a["postgresql"].delete("config_pgtune") if a["postgresql"]["config_pgtune"].empty? + end + + return a, d +end diff --git a/chef/data_bags/crowbar/template-database.json b/chef/data_bags/crowbar/template-database.json index 17621a5879..e19cbaf1fe 100644 --- a/chef/data_bags/crowbar/template-database.json +++ b/chef/data_bags/crowbar/template-database.json @@ -8,8 +8,10 @@ "datadir": "/var/lib/mysql" }, "postgresql": { + "config_pgtune": { + "max_connections": 1000 + }, "config": { - "max_connections": 1000, "log_filename": "postgresql.log-%Y%m%d%H%M", "log_truncate_on_rotation": false } @@ -33,7 +35,7 @@ "database": { "crowbar-revision": 0, "crowbar-applied": false, - "schema-revision": 4, + "schema-revision": 10, "element_states": { "database-server": [ "readying", "ready", "applying" ] }, diff --git a/chef/data_bags/crowbar/template-database.schema b/chef/data_bags/crowbar/template-database.schema index cb451cf798..1a3c352a62 100644 --- a/chef/data_bags/crowbar/template-database.schema +++ b/chef/data_bags/crowbar/template-database.schema @@ -28,11 +28,17 @@ "type": "map", "required": false, "mapping": { + "config_pgtune": { + "type": "map", + "required": false, + "mapping": { + "max_connections": { "type": "int" } + } + }, "config": { "type": "map", "required": false, "mapping": { - "max_connections": { "type": "int" }, "log_truncate_on_rotation": { "type": "bool" }, "log_filename": {"type": "str" } } diff --git a/crowbar_framework/app/views/barclamp/database/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/database/_edit_attributes.html.haml index 70f0fd2059..16c163cb15 100644 --- a/crowbar_framework/app/views/barclamp/database/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/database/_edit_attributes.html.haml @@ -17,7 +17,7 @@ %legend = t('.postgresql_attributes') - = integer_field %w(postgresql config max_connections) + = integer_field %w(postgresql config_pgtune max_connections) -# As HA is only supported for postgresql, we put this section in #postgresql_container %fieldset#ha-setup{ "data-show-for-clusters-only" => "true", "data-elements-path" => "database-server" } diff --git a/crowbar_framework/config/locales/database/en.yml b/crowbar_framework/config/locales/database/en.yml index d0c52c3327..3b1a6c986f 100644 --- a/crowbar_framework/config/locales/database/en.yml +++ b/crowbar_framework/config/locales/database/en.yml @@ -27,7 +27,7 @@ en: datadir: 'Datadir' postgresql_attributes: 'PostgreSQL Options' postgresql: - config: + config_pgtune: max_connections: 'Global Connection Limit (max_connections)' ha_header: 'High Availability' ha: