From f8262cf04473c3c43c3b37089fdb693451e0f14c Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 10:26:34 +0200 Subject: [PATCH 01/19] rabbitmq: Remove outdated rabbitmq clustering code (cherry picked from commit 33f0f04604532f56c655681d7e14b9c605c42ea6) --- chef/cookbooks/rabbitmq/README.md | 6 ---- chef/cookbooks/rabbitmq/attributes/default.rb | 4 --- chef/cookbooks/rabbitmq/metadata.rb | 17 --------- chef/cookbooks/rabbitmq/recipes/cluster.rb | 35 ------------------- .../templates/default/doterlang.cookie.erb | 1 - .../default/rabbitmq_cluster.config.erb | 5 --- 6 files changed, 68 deletions(-) delete mode 100644 chef/cookbooks/rabbitmq/recipes/cluster.rb delete mode 100644 chef/cookbooks/rabbitmq/templates/default/doterlang.cookie.erb delete mode 100644 chef/cookbooks/rabbitmq/templates/default/rabbitmq_cluster.config.erb diff --git a/chef/cookbooks/rabbitmq/README.md b/chef/cookbooks/rabbitmq/README.md index 9d47d5a007..d6420807e1 100644 --- a/chef/cookbooks/rabbitmq/README.md +++ b/chef/cookbooks/rabbitmq/README.md @@ -8,10 +8,6 @@ default ------- Installs `rabbitmq-server` from RabbitMQ.com's APT repository. The distribution-provided version was quite old and newer features were needed. -cluster -------- -Configures nodes to be members of a RabbitMQ cluster, but does not actually join them. - Resources/Providers =================== There are 2 LWRPs for interacting with RabbitMQ. @@ -59,8 +55,6 @@ end Limitations =========== -It is quite useful as is, but clustering configuration does not currently do the dance to join the cluster members to each other. - The rabbitmq::chef recipe was only used for the chef-server cookbook and has been moved to chef-server::rabbitmq. License and Author diff --git a/chef/cookbooks/rabbitmq/attributes/default.rb b/chef/cookbooks/rabbitmq/attributes/default.rb index 0bbd8412c3..685263ab68 100644 --- a/chef/cookbooks/rabbitmq/attributes/default.rb +++ b/chef/cookbooks/rabbitmq/attributes/default.rb @@ -35,10 +35,6 @@ default[:rabbitmq][:configfile] = nil default[:rabbitmq][:logdir] = nil default[:rabbitmq][:mnesiadir] = nil -#clustering -default[:rabbitmq][:cluster] = "no" -default[:rabbitmq][:cluster_config] = "/etc/rabbitmq/rabbitmq_cluster.config" -default[:rabbitmq][:cluster_disk_nodes] = [] # ha default[:rabbitmq][:ha][:enabled] = false diff --git a/chef/cookbooks/rabbitmq/metadata.rb b/chef/cookbooks/rabbitmq/metadata.rb index 964eda4b1a..bb09ccc51d 100644 --- a/chef/cookbooks/rabbitmq/metadata.rb +++ b/chef/cookbooks/rabbitmq/metadata.rb @@ -4,7 +4,6 @@ description "Installs and configures RabbitMQ server" version "1.2.0" recipe "rabbitmq", "Install and configure RabbitMQ" -recipe "rabbitmq::cluster", "Set up RabbitMQ clustering." depends "crowbar-openstack" depends "crowbar-pacemaker" @@ -45,22 +44,6 @@ display_name: "RabbitMQ Mnesia database directory", description: "Path to the directory for Mnesia database files." -attribute "rabbitmq/cluster", - display_name: "RabbitMQ clustering", - description: "Whether to activate clustering.", - default: "no" - -attribute "rabbitmq/cluster_config", - display_name: "RabbitMQ clustering configuration file", - description: "Path to the clustering configuration file, if cluster is yes.", - default: "/etc/rabbitmq/rabbitmq_cluster.config" - -attribute "rabbitmq/cluster_disk_nodes", - display_name: "RabbitMQ cluster disk nodes", - description: "Array of member Erlang nodenames for the disk-based storage nodes in the cluster.", - default: [], - type: "array" - attribute "rabbitmq/erlang_cookie", display_name: "RabbitMQ Erlang cookie", description: "Access cookie for clustering nodes. There is no default." diff --git a/chef/cookbooks/rabbitmq/recipes/cluster.rb b/chef/cookbooks/rabbitmq/recipes/cluster.rb deleted file mode 100644 index dd96f8af01..0000000000 --- a/chef/cookbooks/rabbitmq/recipes/cluster.rb +++ /dev/null @@ -1,35 +0,0 @@ -# -# Cookbook Name:: rabbitmq -# Recipe:: cluster -# -# Copyright 2009, Benjamin Black -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -include_recipe "rabbitmq::default" - -template "/var/lib/rabbitmq/.erlang.cookie" do - source "doterlang.cookie.erb" - owner "rabbitmq" - group "rabbitmq" - mode 0400 -end - -template "/etc/rabbitmq/rabbitmq_cluster.config" do - source "rabbitmq_cluster.config.erb" - owner "root" - group "root" - mode 0644 - notifies :restart, resources(service: "rabbitmq-server"), :immediately -end - diff --git a/chef/cookbooks/rabbitmq/templates/default/doterlang.cookie.erb b/chef/cookbooks/rabbitmq/templates/default/doterlang.cookie.erb deleted file mode 100644 index 80be847009..0000000000 --- a/chef/cookbooks/rabbitmq/templates/default/doterlang.cookie.erb +++ /dev/null @@ -1 +0,0 @@ -<%= node[:rabbitmq][:erlang_cookie] %> \ No newline at end of file diff --git a/chef/cookbooks/rabbitmq/templates/default/rabbitmq_cluster.config.erb b/chef/cookbooks/rabbitmq/templates/default/rabbitmq_cluster.config.erb deleted file mode 100644 index 8d806dd07f..0000000000 --- a/chef/cookbooks/rabbitmq/templates/default/rabbitmq_cluster.config.erb +++ /dev/null @@ -1,5 +0,0 @@ -%%% -%% Generated by Chef for <%= node[:fqdn] %> -%%% - -[<%= node[:rabbitmq][:cluster_disk_nodes].map{|n| "\'#{n}\'"}.join(',') %>]. From dfd14be871a9f0260a518409d26adb775edbbbd5 Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 13:36:54 +0200 Subject: [PATCH 02/19] rabbitmq: Enable deploying rabbitmq with clustering when doing HA Clustering enables us to not require shared storage, and also to have active/active HA for rabbitmq. The configuration of OpenStack services needs to be adapted so that the services are aware of the multiple rabbitmq servers, and the services will then connect to one server, and if that connection fails, will try to connect to another one. (cherry picked from commit 72f5b349bd9499a9b2f24fb3467dbd3d24f3e6e1) Conflicts resolved: chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb chef/data_bags/crowbar/template-rabbitmq.json chef/data_bags/crowbar/template-rabbitmq.schema --- chef/cookbooks/rabbitmq/attributes/default.rb | 2 + chef/cookbooks/rabbitmq/libraries/crowbar.rb | 4 +- chef/cookbooks/rabbitmq/recipes/default.rb | 24 ++++++++++ chef/cookbooks/rabbitmq/recipes/ha_cluster.rb | 44 +++++++++++++++++++ chef/cookbooks/rabbitmq/recipes/rabbit.rb | 35 ++++++++++++--- .../templates/default/rabbitmq.config.erb | 6 +++ .../crowbar/migrate/rabbitmq/012_cluster.rb | 12 +++++ chef/data_bags/crowbar/template-rabbitmq.json | 3 +- .../crowbar/template-rabbitmq.schema | 1 + .../app/models/rabbitmq_service.rb | 22 ++++++++-- .../rabbitmq/_edit_attributes.html.haml | 21 +++++---- .../config/locales/rabbitmq/en.yml | 1 + 12 files changed, 154 insertions(+), 21 deletions(-) create mode 100644 chef/cookbooks/rabbitmq/recipes/ha_cluster.rb create mode 100644 chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb diff --git a/chef/cookbooks/rabbitmq/attributes/default.rb b/chef/cookbooks/rabbitmq/attributes/default.rb index 685263ab68..47e434b8c0 100644 --- a/chef/cookbooks/rabbitmq/attributes/default.rb +++ b/chef/cookbooks/rabbitmq/attributes/default.rb @@ -36,6 +36,8 @@ default[:rabbitmq][:logdir] = nil default[:rabbitmq][:mnesiadir] = nil +default[:rabbitmq][:cluster] = false + # ha default[:rabbitmq][:ha][:enabled] = false default[:rabbitmq][:ha][:storage][:mode] = nil diff --git a/chef/cookbooks/rabbitmq/libraries/crowbar.rb b/chef/cookbooks/rabbitmq/libraries/crowbar.rb index 35b30ffb4b..f23b27bf1d 100644 --- a/chef/cookbooks/rabbitmq/libraries/crowbar.rb +++ b/chef/cookbooks/rabbitmq/libraries/crowbar.rb @@ -8,7 +8,7 @@ def self.get_ha_vhostname(node) end def self.get_listen_address(node) - if node[:rabbitmq][:ha][:enabled] + if node[:rabbitmq][:ha][:enabled] && !node[:rabbitmq][:cluster] vhostname = get_ha_vhostname(node) net_db = Chef::DataBagItem.load("crowbar", "admin_network").raw_data net_db["allocated_by_name"]["#{vhostname}.#{node[:domain]}"]["address"] @@ -18,7 +18,7 @@ def self.get_listen_address(node) end def self.get_public_listen_address(node) - if node[:rabbitmq][:ha][:enabled] + if node[:rabbitmq][:ha][:enabled] && !node[:rabbitmq][:cluster] vhostname = get_ha_vhostname(node) net_db = Chef::DataBagItem.load("crowbar", "public_network").raw_data net_db["allocated_by_name"]["#{vhostname}.#{node[:domain]}"]["address"] diff --git a/chef/cookbooks/rabbitmq/recipes/default.rb b/chef/cookbooks/rabbitmq/recipes/default.rb index d306df2d6b..14bb56788e 100644 --- a/chef/cookbooks/rabbitmq/recipes/default.rb +++ b/chef/cookbooks/rabbitmq/recipes/default.rb @@ -18,6 +18,10 @@ # limitations under the License. # +ha_enabled = node[:rabbitmq][:ha][:enabled] +# we only do cluster if we do HA +cluster_enabled = node[:rabbitmq][:cluster] && ha_enabled + package "rabbitmq-server" package "rabbitmq-server-plugins" if node[:platform_family] == "suse" @@ -36,11 +40,31 @@ notifies :restart, "service[rabbitmq-server]" end +if cluster_enabled + file "/var/lib/rabbitmq/.erlang.cookie" do + content node[:rabbitmq][:erlang_cookie] + owner "rabbitmq" + group "rabbitmq" + mode 0400 + notifies :restart, "service[rabbitmq-server]" + end + + others = CrowbarPacemakerHelper.cluster_nodes(node, "rabbitmq-server").select do |cluster_node| + cluster_node[:hostname] != node[:hostname] + end + other_cluster_nodes = others.map { |n| "rabbit@#{n[:hostname]}" }.join(",") +else + other_cluster_nodes = nil +end + template "/etc/rabbitmq/rabbitmq.config" do source "rabbitmq.config.erb" owner "root" group "root" mode 0644 + variables( + cluster_nodes: other_cluster_nodes + ) notifies :restart, "service[rabbitmq-server]" end diff --git a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb new file mode 100644 index 0000000000..036dfcf72c --- /dev/null +++ b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb @@ -0,0 +1,44 @@ +# Copyright 2017 SUSE +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +service_name = "rabbitmq" + +agent_name = "ocf:rabbitmq:rabbitmq-server" +rabbitmq_op = {} +rabbitmq_op["monitor"] = {} +rabbitmq_op["monitor"]["interval"] = "10s" + +crowbar_pacemaker_sync_mark "wait-rabbitmq_ha_resources" + +transaction_objects = [] + +objects = openstack_pacemaker_controller_clone_for_transaction service_name do + agent agent_name + # nodename is empty so that we explicitly depend on the config files + params ({ + "nodename" => "" + }) + op rabbitmq_op +end +transaction_objects.push(objects) + +pacemaker_transaction "rabbitmq service" do + cib_objects transaction_objects + # note that this will also automatically start the resources + action :commit_new + only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } +end + +crowbar_pacemaker_sync_mark "create-rabbitmq_ha_resources" diff --git a/chef/cookbooks/rabbitmq/recipes/rabbit.rb b/chef/cookbooks/rabbitmq/recipes/rabbit.rb index 08148f5885..f9b4123068 100644 --- a/chef/cookbooks/rabbitmq/recipes/rabbit.rb +++ b/chef/cookbooks/rabbitmq/recipes/rabbit.rb @@ -19,6 +19,8 @@ # ha_enabled = node[:rabbitmq][:ha][:enabled] +# we only do cluster if we do HA +cluster_enabled = node[:rabbitmq][:cluster] && ha_enabled node.set[:rabbitmq][:address] = CrowbarRabbitmqHelper.get_listen_address(node) node.set[:rabbitmq][:management_address] = node[:rabbitmq][:address] @@ -28,7 +30,9 @@ end node.set[:rabbitmq][:addresses] = addresses -if ha_enabled +if cluster_enabled + node.set[:rabbitmq][:nodename] = "rabbit@#{node[:hostname]}" +elsif ha_enabled node.set[:rabbitmq][:nodename] = "rabbit@#{CrowbarRabbitmqHelper.get_ha_vhostname(node)}" end @@ -36,11 +40,17 @@ if ha_enabled log "HA support for rabbitmq is enabled" - include_recipe "rabbitmq::ha" - # All the rabbitmqctl commands are local, and can only be run if rabbitmq is - # local - service_name = "rabbitmq" - only_if_command = "crm resource show #{service_name} | grep -q \" #{node.hostname} *$\"" + if cluster_enabled + include_recipe "rabbitmq::ha_cluster" + # only run the rabbitmqctl commands on the founder node + only_if_command = CrowbarPacemakerHelper.is_cluster_founder?(node) ? "true" : "false" + else + include_recipe "rabbitmq::ha" + # All the rabbitmqctl commands are local, and can only be run if rabbitmq is + # local + service_name = "rabbitmq" + only_if_command = "crm resource show #{service_name} | grep -q \" #{node.hostname} *$\"" + end else log "HA support for rabbitmq is disabled" end @@ -84,6 +94,19 @@ only_if only_if_command if ha_enabled end +if cluster_enabled + set_policy_command = "rabbitmqctl set_policy -p #{node[:rabbitmq][:vhost]} " \ + " ha-all '^(?!amq\.).*' '{\"ha-mode\": \"all\"}'" + check_policy_command = "rabbitmqctl list_policies -p #{node[:rabbitmq][:vhost]} | " \ + " grep -q '^#{node[:rabbitmq][:vhost]}\\s*ha-all\\s'" + + execute set_policy_command do + not_if check_policy_command + action :run + only_if only_if_command if ha_enabled + end +end + if node[:rabbitmq][:trove][:enabled] rabbitmq_vhost node[:rabbitmq][:trove][:vhost] do action :add diff --git a/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb b/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb index 97f29c40d8..8bc1a1807e 100644 --- a/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb +++ b/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb @@ -4,6 +4,12 @@ {tcp_listeners, [ <%= node[:rabbitmq][:addresses].map { |address| "{\"#{address}\", #{node[:rabbitmq][:port]}}" }.join(", ") %> ]}, +<% unless @cluster_nodes.nil? -%> + {cluster_nodes, {[ + <%= @cluster_nodes %> + ], disc} + }, +<% end -%> {disk_free_limit, 50000000} ] }, diff --git a/chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb b/chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb new file mode 100644 index 0000000000..a211d177a3 --- /dev/null +++ b/chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb @@ -0,0 +1,12 @@ +def upgrade(ta, td, a, d) + unless a.key?("cluster") + # don't convert anything existing to cluster + a["cluster"] = false + end + return a, d +end + +def downgrade(ta, td, a, d) + a.delete("cluster") unless ta.key?("cluster") + return a, d +end diff --git a/chef/data_bags/crowbar/template-rabbitmq.json b/chef/data_bags/crowbar/template-rabbitmq.json index 3ffc8c155c..9dedf8f337 100644 --- a/chef/data_bags/crowbar/template-rabbitmq.json +++ b/chef/data_bags/crowbar/template-rabbitmq.json @@ -8,6 +8,7 @@ "password": "", "user": "nova", "vhost": "/nova", + "cluster": true, "ha": { "storage": { "mode": "shared", @@ -33,7 +34,7 @@ "rabbitmq": { "crowbar-revision": 0, "crowbar-applied": false, - "schema-revision": 11, + "schema-revision": 12, "element_states": { "rabbitmq-server": [ "readying", "ready", "applying" ] }, diff --git a/chef/data_bags/crowbar/template-rabbitmq.schema b/chef/data_bags/crowbar/template-rabbitmq.schema index f11e5e2592..70f8b16168 100644 --- a/chef/data_bags/crowbar/template-rabbitmq.schema +++ b/chef/data_bags/crowbar/template-rabbitmq.schema @@ -17,6 +17,7 @@ "password": { "type": "str", "required": true }, "user": { "type": "str", "required": true }, "vhost": { "type": "str", "required": true }, + "cluster": { "type": "bool", "required": true }, "ha" : { "type": "map", "required": true, diff --git a/crowbar_framework/app/models/rabbitmq_service.rb b/crowbar_framework/app/models/rabbitmq_service.rb index e4bd7d2263..f9bdc8856d 100644 --- a/crowbar_framework/app/models/rabbitmq_service.rb +++ b/crowbar_framework/app/models/rabbitmq_service.rb @@ -85,7 +85,7 @@ def apply_role_pre_chef_call(old_role, role, all_nodes) end end - if rabbitmq_ha_enabled + if rabbitmq_ha_enabled && !role.default_attributes["rabbitmq"]["cluster"] unless rabbitmq_elements.length == 1 && PacemakerServiceObject.is_cluster?(rabbitmq_elements[0]) raise "Internal error: HA enabled, but element is not a cluster" end @@ -100,6 +100,20 @@ def apply_role_pre_chef_call(old_role, role, all_nodes) ensure_dns_uptodate end + if role.default_attributes["rabbitmq"]["cluster"] + role.default_attributes["rabbitmq"]["erlang_cookie"] = \ + (old_role && old_role.default_attributes["rabbitmq"]["erlang_cookie"]) || random_password + end + + unless rabbitmq_ha_enabled + # cluster mode requires HA (for now); don't do a validation check as we + # still want to have the setting default to true in case people want to + # turn HA on, and in this case, result in clustering by default + role.default_attributes["rabbitmq"]["cluster"] = false + end + + role.save + @logger.debug("Rabbitmq apply_role_pre_chef_call: leaving") end @@ -108,9 +122,11 @@ def validate_proposal_after_save proposal attributes = proposal["attributes"][@bc_name] - # HA validation servers = proposal["deployment"][@bc_name]["elements"]["rabbitmq-server"] - unless servers.nil? || servers.first.nil? || !is_cluster?(servers.first) + ha_enabled = !(servers.nil? || servers.first.nil? || !is_cluster?(servers.first)) + + # Shared storage validation for HA + if ha_enabled && !attributes["cluster"] storage_mode = attributes["ha"]["storage"]["mode"] validation_error I18n.t( "barclamp.#{@bc_name}.validation.unknown_mode", storage_mode: storage_mode diff --git a/crowbar_framework/app/views/barclamp/rabbitmq/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/rabbitmq/_edit_attributes.html.haml index cfdae04321..11ee878b95 100644 --- a/crowbar_framework/app/views/barclamp/rabbitmq/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/rabbitmq/_edit_attributes.html.haml @@ -11,14 +11,17 @@ %legend = t('.ha_header') - = select_field %w(ha storage mode), :collection => :ha_storage_mode_for_rabbitmq, "data-showit" => ["drbd", "shared"].join(";"), "data-showit-target" => "#drbd_storage_container;#shared_storage_container", "data-showit-direct" => "true" + = boolean_field %w(cluster), "data-hideit" => "true", "data-hideit-target" => "#ha_storage_container", "data-hideit-direct" => "true" - #drbd_storage_container - .alert.alert-info - = t('.ha.storage.drbd_info') - = integer_field %w(ha storage drbd size) + #ha_storage_container + = select_field %w(ha storage mode), :collection => :ha_storage_mode_for_rabbitmq, "data-showit" => ["drbd", "shared"].join(";"), "data-showit-target" => "#drbd_storage_container;#shared_storage_container", "data-showit-direct" => "true" - #shared_storage_container - = string_field %w(ha storage shared device) - = string_field %w(ha storage shared fstype) - = string_field %w(ha storage shared options) + #drbd_storage_container + .alert.alert-info + = t('.ha.storage.drbd_info') + = integer_field %w(ha storage drbd size) + + #shared_storage_container + = string_field %w(ha storage shared device) + = string_field %w(ha storage shared fstype) + = string_field %w(ha storage shared options) diff --git a/crowbar_framework/config/locales/rabbitmq/en.yml b/crowbar_framework/config/locales/rabbitmq/en.yml index f173b40084..19e6e3df40 100644 --- a/crowbar_framework/config/locales/rabbitmq/en.yml +++ b/crowbar_framework/config/locales/rabbitmq/en.yml @@ -23,6 +23,7 @@ en: user: 'User' port: 'Port' ha_header: 'High Availability' + cluster: 'Use RabbitMQ Clustering' ha: storage: mode: 'Storage Mode' From ca89d86b7cc9c491cf10cb70246f8ceed7c3f18d Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 13:48:54 +0200 Subject: [PATCH 03/19] rabbitmq: Move to rabbitmq-server-ha OCF RA for cluster mode This simplifies our code, but also, this resource agent has actually more features and is designed for rabbitmq clustering, including automatic rejoin of nodes in the cluster. In short, it's really the resource agent we want. (cherry picked from commit 11fa090a3a4c7f7c603291cc0639730367930ba6) Conflicts resolved: chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb --- chef/cookbooks/rabbitmq/recipes/default.rb | 24 ------- chef/cookbooks/rabbitmq/recipes/ha_cluster.rb | 66 +++++++++++++++++-- chef/cookbooks/rabbitmq/recipes/rabbit.rb | 5 +- .../templates/default/rabbitmq.config.erb | 2 - 4 files changed, 63 insertions(+), 34 deletions(-) diff --git a/chef/cookbooks/rabbitmq/recipes/default.rb b/chef/cookbooks/rabbitmq/recipes/default.rb index 14bb56788e..d306df2d6b 100644 --- a/chef/cookbooks/rabbitmq/recipes/default.rb +++ b/chef/cookbooks/rabbitmq/recipes/default.rb @@ -18,10 +18,6 @@ # limitations under the License. # -ha_enabled = node[:rabbitmq][:ha][:enabled] -# we only do cluster if we do HA -cluster_enabled = node[:rabbitmq][:cluster] && ha_enabled - package "rabbitmq-server" package "rabbitmq-server-plugins" if node[:platform_family] == "suse" @@ -40,31 +36,11 @@ notifies :restart, "service[rabbitmq-server]" end -if cluster_enabled - file "/var/lib/rabbitmq/.erlang.cookie" do - content node[:rabbitmq][:erlang_cookie] - owner "rabbitmq" - group "rabbitmq" - mode 0400 - notifies :restart, "service[rabbitmq-server]" - end - - others = CrowbarPacemakerHelper.cluster_nodes(node, "rabbitmq-server").select do |cluster_node| - cluster_node[:hostname] != node[:hostname] - end - other_cluster_nodes = others.map { |n| "rabbit@#{n[:hostname]}" }.join(",") -else - other_cluster_nodes = nil -end - template "/etc/rabbitmq/rabbitmq.config" do source "rabbitmq.config.erb" owner "root" group "root" mode 0644 - variables( - cluster_nodes: other_cluster_nodes - ) notifies :restart, "service[rabbitmq-server]" end diff --git a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb index 036dfcf72c..e1d60f6edb 100644 --- a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb +++ b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb @@ -13,9 +13,7 @@ # limitations under the License. # -service_name = "rabbitmq" - -agent_name = "ocf:rabbitmq:rabbitmq-server" +agent_name = "ocf:rabbitmq:rabbitmq-server-ha" rabbitmq_op = {} rabbitmq_op["monitor"] = {} rabbitmq_op["monitor"]["interval"] = "10s" @@ -24,15 +22,38 @@ transaction_objects = [] -objects = openstack_pacemaker_controller_clone_for_transaction service_name do +service_name = "rabbitmq" +pacemaker_primitive service_name do agent agent_name # nodename is empty so that we explicitly depend on the config files params ({ - "nodename" => "" + "erlang_cookie" => node[:rabbitmq][:erlang_cookie] }) op rabbitmq_op + action :update + only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } end -transaction_objects.push(objects) +transaction_objects.push("pacemaker_primitive[#{service_name}]") + +# no location on the role here: the ms resource will have this constraint + +ms_name = "ms-#{service_name}" +pacemaker_ms ms_name do + rsc service_name + meta ({ + "master-max" => "1", + "master-node-max" => "1", + "ordered" => "false", + "interleave" => "false", + "notify" => "true" + }) + action :update + only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } +end +transaction_objects.push("pacemaker_ms[#{ms_name}]") + +ms_location_name = openstack_pacemaker_controller_only_location_for ms_name +transaction_objects.push("pacemaker_location[#{ms_location_name}]") pacemaker_transaction "rabbitmq service" do cib_objects transaction_objects @@ -42,3 +63,36 @@ end crowbar_pacemaker_sync_mark "create-rabbitmq_ha_resources" + +# wait for service to have a master, and to be active +ruby_block "wait for #{ms_name} to be started" do + block do + require "timeout" + begin + Timeout.timeout(30) do + # Check that the service has a master + cmd = "crm resource show #{ms_name} 2> /dev/null | grep -q \"is running on.*Master\"" + while ! ::Kernel.system(cmd) + Chef::Log.debug("#{service_name} still without master") + sleep(2) + end + # Check that the master is this node + cmd = "crm resource show #{service_name} | grep -q \" #{node.hostname} *Master$\"" + if ::Kernel.system(cmd) + # The sed command grabs everything between '{running_applications' + # and ']}', and what we want is that the rabbit application is + # running + cmd = "rabbitmqctl -q status 2> /dev/null| sed -n '/{running_applications/,/\]}/p' | grep -q '{rabbit,'" + while ! ::Kernel.system(cmd) + Chef::Log.debug("#{service_name} still not answering") + sleep(2) + end + end + end + rescue Timeout::Error + message = "The #{service_name} pacemaker resource is not started. Please manually check for an error." + Chef::Log.fatal(message) + raise message + end + end # block +end # ruby_block diff --git a/chef/cookbooks/rabbitmq/recipes/rabbit.rb b/chef/cookbooks/rabbitmq/recipes/rabbit.rb index f9b4123068..51bf7ef235 100644 --- a/chef/cookbooks/rabbitmq/recipes/rabbit.rb +++ b/chef/cookbooks/rabbitmq/recipes/rabbit.rb @@ -42,8 +42,9 @@ log "HA support for rabbitmq is enabled" if cluster_enabled include_recipe "rabbitmq::ha_cluster" - # only run the rabbitmqctl commands on the founder node - only_if_command = CrowbarPacemakerHelper.is_cluster_founder?(node) ? "true" : "false" + # only run the rabbitmqctl commands on the master node + ms_name = "ms-rabbitmq" + only_if_command = "crm resource show #{ms_name} | grep -q \" #{node.hostname} *Master$\"" else include_recipe "rabbitmq::ha" # All the rabbitmqctl commands are local, and can only be run if rabbitmq is diff --git a/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb b/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb index 8bc1a1807e..1544b87158 100644 --- a/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb +++ b/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb @@ -4,12 +4,10 @@ {tcp_listeners, [ <%= node[:rabbitmq][:addresses].map { |address| "{\"#{address}\", #{node[:rabbitmq][:port]}}" }.join(", ") %> ]}, -<% unless @cluster_nodes.nil? -%> {cluster_nodes, {[ <%= @cluster_nodes %> ], disc} }, -<% end -%> {disk_free_limit, 50000000} ] }, From daae22949d18f209d108e25ef241c959e48952e0 Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 18:14:48 +0200 Subject: [PATCH 04/19] rabbitmq: Set cluster_name for rabbitmq cluster While not mandatory, this makes it less confusing to have a cluster name that is not based on the hostname of one cluster node. (cherry picked from commit 5fff263c150933d507f8ca88663a44488af0dba0) --- chef/cookbooks/rabbitmq/attributes/default.rb | 1 + chef/cookbooks/rabbitmq/recipes/rabbit.rb | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/chef/cookbooks/rabbitmq/attributes/default.rb b/chef/cookbooks/rabbitmq/attributes/default.rb index 47e434b8c0..c832fba626 100644 --- a/chef/cookbooks/rabbitmq/attributes/default.rb +++ b/chef/cookbooks/rabbitmq/attributes/default.rb @@ -37,6 +37,7 @@ default[:rabbitmq][:mnesiadir] = nil default[:rabbitmq][:cluster] = false +default[:rabbitmq][:clustername] = "rabbit@#{node[:hostname]}" # ha default[:rabbitmq][:ha][:enabled] = false diff --git a/chef/cookbooks/rabbitmq/recipes/rabbit.rb b/chef/cookbooks/rabbitmq/recipes/rabbit.rb index 51bf7ef235..9cf4145054 100644 --- a/chef/cookbooks/rabbitmq/recipes/rabbit.rb +++ b/chef/cookbooks/rabbitmq/recipes/rabbit.rb @@ -32,6 +32,7 @@ if cluster_enabled node.set[:rabbitmq][:nodename] = "rabbit@#{node[:hostname]}" + node.set[:rabbitmq][:clustername] = "rabbit@#{CrowbarRabbitmqHelper.get_ha_vhostname(node)}" elsif ha_enabled node.set[:rabbitmq][:nodename] = "rabbit@#{CrowbarRabbitmqHelper.get_ha_vhostname(node)}" end @@ -106,6 +107,14 @@ action :run only_if only_if_command if ha_enabled end + + check_cluster_name_command = "rabbitmqctl cluster_status | " \ + "grep -q '{cluster_name,<<\"#{node[:rabbitmq][:clustername]}\">>}'" + execute "rabbitmqctl set_cluster_name #{node[:rabbitmq][:clustername]}" do + not_if check_cluster_name_command + action :run + only_if only_if_command if ha_enabled + end end if node[:rabbitmq][:trove][:enabled] From 97f8321eb0db4d771cf64736812228c9ea5c6ace Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 14:11:26 +0200 Subject: [PATCH 05/19] barbican, keystone: Move to transport_url setting for rabbitmq (cherry picked from commit c7a1ce850fa85d380d0a91c1a6855a561c463f6c) Conflicts resolved: chef/cookbooks/barbican/templates/default/barbican.conf.erb chef/cookbooks/keystone/templates/default/keystone.conf.erb --- chef/cookbooks/keystone/templates/default/keystone.conf.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/keystone/templates/default/keystone.conf.erb b/chef/cookbooks/keystone/templates/default/keystone.conf.erb index 0d0df70d5d..ab5520b39e 100644 --- a/chef/cookbooks/keystone/templates/default/keystone.conf.erb +++ b/chef/cookbooks/keystone/templates/default/keystone.conf.erb @@ -137,6 +137,7 @@ log_file = keystone.log # Deprecated group/name - [DEFAULT]/logdir #log_dir = log_dir = /var/log/keystone +transport_url = <%= @rabbit_settings[:url] %> # Use syslog for logging. Existing syslog format is DEPRECATED and will be # changed later to honor RFC5424. (boolean value) From c46191edf5ce2519f70266f66bd26ff06342b102 Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 14:47:21 +0200 Subject: [PATCH 06/19] crowbar-openstack: Make rabbitmq helper cluster-aware (cherry picked from commit 5502fc6f452a5b5c2b0b929d986c4c606a8ddcab) Conflicts resolved: chef/cookbooks/crowbar-openstack/libraries/helpers.rb --- .../crowbar-openstack/libraries/helpers.rb | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index c748b2aa56..c857df9f51 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -109,27 +109,59 @@ def self.rabbitmq_settings(node, barclamp) end if @rabbitmq_settings && @rabbitmq_settings.include?(instance) - Chef::Log.info("RabbitMQ server found at #{@rabbitmq_settings[instance][:address]} [cached]") + Chef::Log.info("RabbitMQ settings found [cached]") else @rabbitmq_settings ||= Hash.new - rabbit = get_node(node, "rabbitmq-server", "rabbitmq", instance) + rabbits = get_nodes(node, "rabbitmq-server", "rabbitmq", instance) - if rabbit.nil? + if rabbits.empty? Chef::Log.warn("No RabbitMQ server found!") else - @rabbitmq_settings[instance] = { - address: rabbit[:rabbitmq][:address], - port: rabbit[:rabbitmq][:port], - user: rabbit[:rabbitmq][:user], - password: rabbit[:rabbitmq][:password], - vhost: rabbit[:rabbitmq][:vhost], - url: "rabbit://#{rabbit[:rabbitmq][:user]}:" \ + one_rabbit = rabbits.first + client_ca_certs = if one_rabbit[:rabbitmq][:ssl][:enabled] && \ + !one_rabbit[:rabbitmq][:ssl][:insecure] + one_rabbit[:rabbitmq][:ssl][:client_ca_certs] + end + + if rabbits.first[:rabbitmq][:cluster] + rabbit_hosts = rabbits.map do |rabbit| + port = if rabbit[:rabbitmq][:ssl][:enabled] + rabbit[:rabbitmq][:ssl][:port] + else + rabbit[:rabbitmq][:port] + end + + "#{rabbit[:rabbitmq][:user]}:" \ "#{rabbit[:rabbitmq][:password]}@" \ - "#{rabbit[:rabbitmq][:address]}:#{rabbit[:rabbitmq][:port]}/" \ - "#{rabbit[:rabbitmq][:vhost]}" - } + "#{rabbit[:rabbitmq][:address]}:#{port}" + end + + @rabbitmq_settings[instance] = { + use_ssl: one_rabbit[:rabbitmq][:ssl][:enabled], + client_ca_certs: client_ca_certs, + url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ + "#{rabbits.first[:rabbitmq][:vhost]}" + } + Chef::Log.info("RabbitMQ cluster found") + else + rabbit = one_rabbit + port = if rabbit[:rabbitmq][:ssl][:enabled] + rabbit[:rabbitmq][:ssl][:port] + else + rabbit[:rabbitmq][:port] + end + + @rabbitmq_settings[instance] = { + use_ssl: rabbit[:rabbitmq][:ssl][:enabled], + client_ca_certs: client_ca_certs, + url: "rabbit://#{rabbit[:rabbitmq][:user]}:" \ + "#{rabbit[:rabbitmq][:password]}@" \ + "#{rabbit[:rabbitmq][:address]}:#{port}/" \ + "#{rabbit[:rabbitmq][:vhost]}" + } - Chef::Log.info("RabbitMQ server found at #{@rabbitmq_settings[instance][:address]}") + Chef::Log.info("RabbitMQ server found") + end end end @@ -153,4 +185,9 @@ def self.get_node(node, role, barclamp, instance) result end + + def self.get_nodes(node, role, barclamp, instance) + nodes, _, _ = Chef::Search::Query.new.search(:node, "roles:#{role} AND #{barclamp}_config_environment:#{barclamp}-config-#{instance}") + nodes + end end From 2a61410051366d232aeb4369dbf997400864246e Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 14:50:16 +0200 Subject: [PATCH 07/19] crowbar-openstack, trove: Simplify rabbitmq settings for trove Just do all the magic in the crowbar-openstack helper, since it knows about clustering and everything else already. (cherry picked from commit 1745518e19a377486adebdca92c45022b3be0bf8) Conflicts resolved: chef/cookbooks/trove/libraries/helpers.rb chef/cookbooks/trove/recipes/api.rb chef/cookbooks/trove/recipes/conductor.rb chef/cookbooks/trove/recipes/taskmanager.rb --- .../crowbar-openstack/libraries/helpers.rb | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index c857df9f51..3815b2cf85 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -136,11 +136,25 @@ def self.rabbitmq_settings(node, barclamp) "#{rabbit[:rabbitmq][:address]}:#{port}" end + trove_rabbit_hosts = rabbits.map do |rabbit| + port = if rabbit[:rabbitmq][:ssl][:enabled] + rabbit[:rabbitmq][:ssl][:port] + else + rabbit[:rabbitmq][:port] + end + + "#{rabbit[:rabbitmq][:trove][:user]}:" \ + "#{rabbit[:rabbitmq][:trove][:password]}@" \ + "#{rabbit[:rabbitmq][:address]}:#{port}" + end + @rabbitmq_settings[instance] = { use_ssl: one_rabbit[:rabbitmq][:ssl][:enabled], client_ca_certs: client_ca_certs, url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ - "#{rabbits.first[:rabbitmq][:vhost]}" + "#{rabbits.first[:rabbitmq][:vhost]}", + trove_url: "rabbit://#{trove_rabbit_hosts.sort.join(",")}/" \ + "#{rabbits.first[:rabbitmq][:trove][:vhost]}" } Chef::Log.info("RabbitMQ cluster found") else @@ -157,7 +171,11 @@ def self.rabbitmq_settings(node, barclamp) url: "rabbit://#{rabbit[:rabbitmq][:user]}:" \ "#{rabbit[:rabbitmq][:password]}@" \ "#{rabbit[:rabbitmq][:address]}:#{port}/" \ - "#{rabbit[:rabbitmq][:vhost]}" + "#{rabbit[:rabbitmq][:vhost]}", + trove_url: "rabbit://#{rabbit[:rabbitmq][:trove][:user]}:" \ + "#{rabbit[:rabbitmq][:trove][:password]}@" \ + "#{rabbit[:rabbitmq][:address]}:#{port}/" \ + "#{rabbit[:rabbitmq][:trove][:vhost]}" } Chef::Log.info("RabbitMQ server found") From 8d1178fc03dd8ce52ac3727d1eb5f560f587933f Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 15:09:56 +0200 Subject: [PATCH 08/19] openstack: Set amqp_durable_queues and rabbit_ha_queues options (cherry picked from commit 8bd51a6d695f755d695b8a54016ad9ec39aaa3e0) Conflicts resolved: chef/cookbooks/aodh/templates/default/aodh.conf.erb chef/cookbooks/barbican/templates/default/barbican.conf.erb chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb chef/cookbooks/cinder/templates/default/cinder.conf.erb chef/cookbooks/ec2-api/templates/default/ec2api.conf.erb chef/cookbooks/glance/templates/default/glance-api.conf.erb chef/cookbooks/heat/recipes/server.rb chef/cookbooks/heat/templates/default/heat.conf.erb chef/cookbooks/keystone/templates/default/keystone.conf.erb chef/cookbooks/magnum/templates/default/magnum.conf.erb chef/cookbooks/manila/templates/default/manila.conf.erb chef/cookbooks/neutron/templates/default/neutron.conf.erb chef/cookbooks/nova/templates/default/nova.conf.erb chef/cookbooks/sahara/templates/default/sahara.conf.erb chef/cookbooks/trove/recipes/api.rb chef/cookbooks/trove/recipes/conductor.rb chef/cookbooks/trove/recipes/taskmanager.rb chef/cookbooks/trove/templates/default/trove-conductor.conf.erb chef/cookbooks/trove/templates/default/trove-taskmanager.conf.erb chef/cookbooks/trove/templates/default/trove.conf.erb --- .../ceilometer/templates/default/ceilometer.conf.erb | 4 ++-- chef/cookbooks/cinder/templates/default/cinder.conf.erb | 4 ++-- chef/cookbooks/crowbar-openstack/libraries/helpers.rb | 4 ++++ chef/cookbooks/glance/templates/default/glance-api.conf.erb | 4 ++-- .../glance/templates/default/glance-registry.conf.erb | 4 ++-- chef/cookbooks/heat/templates/default/heat.conf.erb | 4 ++-- chef/cookbooks/keystone/templates/default/keystone.conf.erb | 4 ++-- chef/cookbooks/manila/templates/default/manila.conf.erb | 4 ++-- chef/cookbooks/neutron/templates/default/neutron.conf.erb | 4 ++-- chef/cookbooks/nova/templates/default/nova.conf.erb | 4 ++-- 10 files changed, 22 insertions(+), 18 deletions(-) diff --git a/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb b/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb index 05c9228b80..0b56709f59 100644 --- a/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb +++ b/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb @@ -1168,7 +1168,7 @@ lock_path = /var/run/ceilometer # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1269,7 +1269,7 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/cinder/templates/default/cinder.conf.erb b/chef/cookbooks/cinder/templates/default/cinder.conf.erb index 987ee2d965..c7bf37b158 100644 --- a/chef/cookbooks/cinder/templates/default/cinder.conf.erb +++ b/chef/cookbooks/cinder/templates/default/cinder.conf.erb @@ -3464,7 +3464,7 @@ lock_path = /var/run/cinder # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -3562,7 +3562,7 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index 3815b2cf85..064128d747 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -149,6 +149,8 @@ def self.rabbitmq_settings(node, barclamp) end @rabbitmq_settings[instance] = { + ha_queues: true, + durable_queues: true, use_ssl: one_rabbit[:rabbitmq][:ssl][:enabled], client_ca_certs: client_ca_certs, url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ @@ -166,6 +168,8 @@ def self.rabbitmq_settings(node, barclamp) end @rabbitmq_settings[instance] = { + ha_queues: false, + durable_queues: false, use_ssl: rabbit[:rabbitmq][:ssl][:enabled], client_ca_certs: client_ca_certs, url: "rabbit://#{rabbit[:rabbitmq][:user]}:" \ diff --git a/chef/cookbooks/glance/templates/default/glance-api.conf.erb b/chef/cookbooks/glance/templates/default/glance-api.conf.erb index 239f64e744..ab7b206d0c 100644 --- a/chef/cookbooks/glance/templates/default/glance-api.conf.erb +++ b/chef/cookbooks/glance/templates/default/glance-api.conf.erb @@ -1389,7 +1389,7 @@ lock_path = /var/run/glance # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1493,7 +1493,7 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/glance/templates/default/glance-registry.conf.erb b/chef/cookbooks/glance/templates/default/glance-registry.conf.erb index 4d32722c56..1ea86f515f 100644 --- a/chef/cookbooks/glance/templates/default/glance-registry.conf.erb +++ b/chef/cookbooks/glance/templates/default/glance-registry.conf.erb @@ -1123,7 +1123,7 @@ admin_tenant_name = <%= @keystone_settings['service_tenant'] %> # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1227,7 +1227,7 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/heat/templates/default/heat.conf.erb b/chef/cookbooks/heat/templates/default/heat.conf.erb index f09dc63850..d254f6b9b7 100644 --- a/chef/cookbooks/heat/templates/default/heat.conf.erb +++ b/chef/cookbooks/heat/templates/default/heat.conf.erb @@ -1396,7 +1396,7 @@ admin_tenant_name=<%= @keystone_settings['service_tenant'] %> # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1492,7 +1492,7 @@ rabbit_virtual_host=<%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer diff --git a/chef/cookbooks/keystone/templates/default/keystone.conf.erb b/chef/cookbooks/keystone/templates/default/keystone.conf.erb index ab5520b39e..cd87e18fbf 100644 --- a/chef/cookbooks/keystone/templates/default/keystone.conf.erb +++ b/chef/cookbooks/keystone/templates/default/keystone.conf.erb @@ -1595,7 +1595,7 @@ use_pool = <%= node[:keystone][:ldap][:use_pool] %> # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1691,7 +1691,7 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer diff --git a/chef/cookbooks/manila/templates/default/manila.conf.erb b/chef/cookbooks/manila/templates/default/manila.conf.erb index d57eb3e9cf..a5a069bd86 100644 --- a/chef/cookbooks/manila/templates/default/manila.conf.erb +++ b/chef/cookbooks/manila/templates/default/manila.conf.erb @@ -1878,7 +1878,7 @@ lock_path=/var/run/manila # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1981,7 +1981,7 @@ rabbit_virtual_host=<%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/neutron/templates/default/neutron.conf.erb b/chef/cookbooks/neutron/templates/default/neutron.conf.erb index 28207f2ca2..1a6dfbaa28 100644 --- a/chef/cookbooks/neutron/templates/default/neutron.conf.erb +++ b/chef/cookbooks/neutron/templates/default/neutron.conf.erb @@ -1014,7 +1014,7 @@ lock_path = /var/run/neutron # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -# amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1102,7 +1102,7 @@ rabbit_virtual_host=<%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -# rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Deprecated, use rpc_backend=kombu+memory or rpc_backend=fake (boolean value) # Deprecated group/name - [DEFAULT]/fake_rabbit diff --git a/chef/cookbooks/nova/templates/default/nova.conf.erb b/chef/cookbooks/nova/templates/default/nova.conf.erb index 0c908f75fa..9ac765e848 100644 --- a/chef/cookbooks/nova/templates/default/nova.conf.erb +++ b/chef/cookbooks/nova/templates/default/nova.conf.erb @@ -3191,7 +3191,7 @@ lock_path = /var/run/nova # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues = false +amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -3287,7 +3287,7 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false +rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer From d75caebaf1c2d13dd4c33f4ab889bf47c1b301e4 Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 29 Mar 2017 17:16:32 +0200 Subject: [PATCH 09/19] openstack: Fix pacemaker ordering contraints for rabbitmq When clustering is used for rabbitmq, we need to use ms-rabbitmq and not rabbitmq in ordering constraints. (cherry picked from commit 2d5f89a7b3edb3aca376144da34c5c25b3bbac35) Conflicts resolved: chef/cookbooks/aodh/recipes/aodh_ha.rb chef/cookbooks/barbican/recipes/ha.rb chef/cookbooks/ceilometer/recipes/central_ha.rb chef/cookbooks/ceilometer/recipes/server_ha.rb chef/cookbooks/cinder/recipes/controller_ha.rb chef/cookbooks/ec2-api/recipes/ec2api_ha.rb chef/cookbooks/glance/recipes/ha.rb chef/cookbooks/heat/recipes/ha.rb chef/cookbooks/keystone/recipes/ha.rb chef/cookbooks/magnum/recipes/ha.rb chef/cookbooks/manila/recipes/controller_ha.rb chef/cookbooks/neutron/recipes/network_agents_ha.rb chef/cookbooks/neutron/recipes/server_ha.rb chef/cookbooks/nova/recipes/compute_ha.rb chef/cookbooks/nova/recipes/controller_ha.rb chef/cookbooks/sahara/recipes/ha.rb --- chef/cookbooks/ceilometer/recipes/central_ha.rb | 3 ++- chef/cookbooks/ceilometer/recipes/server_ha.rb | 3 ++- chef/cookbooks/cinder/recipes/controller_ha.rb | 3 ++- chef/cookbooks/crowbar-openstack/libraries/helpers.rb | 6 ++++-- chef/cookbooks/glance/recipes/ha.rb | 3 ++- chef/cookbooks/heat/recipes/ha.rb | 3 ++- chef/cookbooks/keystone/recipes/ha.rb | 3 ++- chef/cookbooks/manila/recipes/controller_ha.rb | 3 ++- chef/cookbooks/neutron/recipes/network_agents_ha.rb | 4 +++- chef/cookbooks/neutron/recipes/server_ha.rb | 3 ++- chef/cookbooks/nova/recipes/compute_ha.rb | 3 ++- chef/cookbooks/nova/recipes/controller_ha.rb | 3 ++- 12 files changed, 27 insertions(+), 13 deletions(-) diff --git a/chef/cookbooks/ceilometer/recipes/central_ha.rb b/chef/cookbooks/ceilometer/recipes/central_ha.rb index a6fd851e8d..0ba2451046 100644 --- a/chef/cookbooks/ceilometer/recipes/central_ha.rb +++ b/chef/cookbooks/ceilometer/recipes/central_ha.rb @@ -21,6 +21,7 @@ # Avoid races when creating pacemaker resources crowbar_pacemaker_sync_mark "wait-ceilometer_central_ha_resources" +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] service_name = "ceilometer-central" @@ -43,7 +44,7 @@ end crowbar_pacemaker_order_only_existing "o-#{service_name}" do - ordering ["rabbitmq", "cl-keystone", service_name] + ordering ["#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", service_name] score "Optional" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/ceilometer/recipes/server_ha.rb b/chef/cookbooks/ceilometer/recipes/server_ha.rb index 7416891ebf..1361c016f5 100644 --- a/chef/cookbooks/ceilometer/recipes/server_ha.rb +++ b/chef/cookbooks/ceilometer/recipes/server_ha.rb @@ -29,6 +29,7 @@ # Avoid races when creating pacemaker resources crowbar_pacemaker_sync_mark "wait-ceilometer_server_ha_resources" +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] primitives = [] @@ -66,7 +67,7 @@ end transaction_objects << "pacemaker_clone[#{clone_name}]" -order_only_existing = ["rabbitmq", "cl-keystone", clone_name] +order_only_existing = ["#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", clone_name] if node[:ceilometer][:use_mongodb] pacemaker_order "o-ceilometer-mongo" do diff --git a/chef/cookbooks/cinder/recipes/controller_ha.rb b/chef/cookbooks/cinder/recipes/controller_ha.rb index 70ca1bc31c..e6ef344401 100644 --- a/chef/cookbooks/cinder/recipes/controller_ha.rb +++ b/chef/cookbooks/cinder/recipes/controller_ha.rb @@ -41,6 +41,7 @@ # Avoid races when creating pacemaker resources crowbar_pacemaker_sync_mark "wait-cinder_ha_resources" +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] api_primitive = "cinder-api" @@ -92,7 +93,7 @@ end crowbar_pacemaker_order_only_existing "o-#{clone_name}" do - ordering ["postgresql", "rabbitmq", "cl-keystone", clone_name] + ordering ["postgresql", "#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", clone_name] score "Optional" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index 064128d747..a7cd7d6ed4 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -156,7 +156,8 @@ def self.rabbitmq_settings(node, barclamp) url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ "#{rabbits.first[:rabbitmq][:vhost]}", trove_url: "rabbit://#{trove_rabbit_hosts.sort.join(",")}/" \ - "#{rabbits.first[:rabbitmq][:trove][:vhost]}" + "#{rabbits.first[:rabbitmq][:trove][:vhost]}", + pacemaker_resource: "ms-rabbitmq" } Chef::Log.info("RabbitMQ cluster found") else @@ -179,7 +180,8 @@ def self.rabbitmq_settings(node, barclamp) trove_url: "rabbit://#{rabbit[:rabbitmq][:trove][:user]}:" \ "#{rabbit[:rabbitmq][:trove][:password]}@" \ "#{rabbit[:rabbitmq][:address]}:#{port}/" \ - "#{rabbit[:rabbitmq][:trove][:vhost]}" + "#{rabbit[:rabbitmq][:trove][:vhost]}", + pacemaker_resource: "rabbitmq" } Chef::Log.info("RabbitMQ server found") diff --git a/chef/cookbooks/glance/recipes/ha.rb b/chef/cookbooks/glance/recipes/ha.rb index 529568cee1..1fce139020 100644 --- a/chef/cookbooks/glance/recipes/ha.rb +++ b/chef/cookbooks/glance/recipes/ha.rb @@ -47,6 +47,7 @@ crowbar_pacemaker_sync_mark "wait-glance_ha_resources" primitives = [] +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] ["registry", "api"].each do |service| @@ -94,7 +95,7 @@ end crowbar_pacemaker_order_only_existing "o-#{clone_name}" do - ordering ["postgresql", "rabbitmq", "cl-keystone", clone_name] + ordering ["postgresql", "#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", clone_name] score "Optional" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/heat/recipes/ha.rb b/chef/cookbooks/heat/recipes/ha.rb index d8508765c6..7cb4e6c85e 100644 --- a/chef/cookbooks/heat/recipes/ha.rb +++ b/chef/cookbooks/heat/recipes/ha.rb @@ -46,6 +46,7 @@ crowbar_pacemaker_sync_mark "wait-heat_ha_resources" primitives = [] +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] ["engine", "api", "api_cfn", "api_cloudwatch"].each do |service| @@ -92,7 +93,7 @@ end crowbar_pacemaker_order_only_existing "o-#{clone_name}" do - ordering ["postgresql", "rabbitmq", "cl-keystone", "cl-g-nova-controller", clone_name] + ordering ["postgresql", "#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", "cl-g-nova-controller", clone_name] score "Optional" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/keystone/recipes/ha.rb b/chef/cookbooks/keystone/recipes/ha.rb index 37cb0b3293..49434aab92 100644 --- a/chef/cookbooks/keystone/recipes/ha.rb +++ b/chef/cookbooks/keystone/recipes/ha.rb @@ -44,6 +44,7 @@ # then we will avoid races there too as pacemaker will start the service). crowbar_pacemaker_sync_mark "wait-keystone_ha_resources" + rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] service_name = "keystone" @@ -85,7 +86,7 @@ end crowbar_pacemaker_order_only_existing "o-#{clone_name}" do - ordering ["postgresql", "rabbitmq", clone_name] + ordering ["postgresql", "#{rabbit_settings[:pacemaker_resource]}", clone_name] score "Optional" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/manila/recipes/controller_ha.rb b/chef/cookbooks/manila/recipes/controller_ha.rb index 80e3be0153..b8c683aee0 100644 --- a/chef/cookbooks/manila/recipes/controller_ha.rb +++ b/chef/cookbooks/manila/recipes/controller_ha.rb @@ -47,6 +47,7 @@ # Avoid races when creating pacemaker resources crowbar_pacemaker_sync_mark "wait-manila_ha_resources" +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] api_primitive = "manila-api" @@ -98,7 +99,7 @@ end crowbar_pacemaker_order_only_existing "o-#{clone_name}" do - ordering ["postgresql", "rabbitmq", "cl-keystone", "cl-glance", "cl-cinder", + ordering ["postgresql", "#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", "cl-glance", "cl-cinder", "cl-neutron", "cl-nova", clone_name] score "Optional" action :create diff --git a/chef/cookbooks/neutron/recipes/network_agents_ha.rb b/chef/cookbooks/neutron/recipes/network_agents_ha.rb index 1959f62523..388529eba8 100644 --- a/chef/cookbooks/neutron/recipes/network_agents_ha.rb +++ b/chef/cookbooks/neutron/recipes/network_agents_ha.rb @@ -232,6 +232,8 @@ only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } end + rabbit_settings = fetch_rabbitmq_settings + crowbar_pacemaker_order_only_existing "o-#{ha_tool_primitive_name}" do # While neutron-ha-tool technically doesn't directly depend on postgresql or # rabbitmq, if these bits are not running, then neutron-server can run but @@ -239,7 +241,7 @@ # constraint on these services, but it's optional, not mandatory (because it # doesn't need to be restarted when postgresql or rabbitmq are restarted). # So explicitly depend on postgresql and rabbitmq (if they are in the cluster). - ordering "( postgresql rabbitmq g-haproxy cl-neutron-server #{agents_clone_name} ) #{ha_tool_primitive_name}" + ordering "( postgresql #{rabbit_settings[:pacemaker_resource]} g-haproxy cl-neutron-server #{agents_clone_name} ) #{ha_tool_primitive_name}" score "Mandatory" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/neutron/recipes/server_ha.rb b/chef/cookbooks/neutron/recipes/server_ha.rb index 6f43bb0f59..f09e408250 100644 --- a/chef/cookbooks/neutron/recipes/server_ha.rb +++ b/chef/cookbooks/neutron/recipes/server_ha.rb @@ -29,6 +29,7 @@ # Avoid races when creating pacemaker resources crowbar_pacemaker_sync_mark "wait-neutron_ha_resources" +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] primitive_name = "neutron-server" @@ -63,7 +64,7 @@ end crowbar_pacemaker_order_only_existing "o-#{clone_name}" do - ordering ["postgresql", "rabbitmq", "cl-keystone", clone_name] + ordering ["postgresql", "#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", clone_name] score "Optional" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/nova/recipes/compute_ha.rb b/chef/cookbooks/nova/recipes/compute_ha.rb index 097c7a4aa2..724c3d8ae0 100644 --- a/chef/cookbooks/nova/recipes/compute_ha.rb +++ b/chef/cookbooks/nova/recipes/compute_ha.rb @@ -290,6 +290,7 @@ end end +rabbit_settings = fetch_rabbitmq_settings crowbar_pacemaker_order_only_existing "o-#{evacuate_primitive}" do # We need services required to boot an instance; most of these services are # obviously required. Some additional notes: @@ -297,7 +298,7 @@ # - cinder is used in case of boot from volume # - neutron agents are used even with DVR, if only to have a DHCP server for # the instance to get an IP address - ordering "( postgresql rabbitmq cl-keystone cl-swift-proxy cl-g-glance cl-g-cinder-controller cl-neutron-server cl-g-neutron-agents cl-g-nova-controller ) #{evacuate_primitive}" + ordering "( postgresql #{rabbit_settings[:pacemaker_resource]} cl-keystone cl-swift-proxy cl-g-glance cl-g-cinder-controller cl-neutron-server cl-g-neutron-agents cl-g-nova-controller ) #{evacuate_primitive}" score "Mandatory" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } diff --git a/chef/cookbooks/nova/recipes/controller_ha.rb b/chef/cookbooks/nova/recipes/controller_ha.rb index 3db73649da..9ed88a727c 100644 --- a/chef/cookbooks/nova/recipes/controller_ha.rb +++ b/chef/cookbooks/nova/recipes/controller_ha.rb @@ -83,6 +83,7 @@ # Avoid races when creating pacemaker resources crowbar_pacemaker_sync_mark "wait-nova_ha_resources" +rabbit_settings = fetch_rabbitmq_settings transaction_objects = [] primitives = [] @@ -143,7 +144,7 @@ end crowbar_pacemaker_order_only_existing "o-#{clone_name}" do - ordering ["postgresql", "rabbitmq", "cl-keystone", clone_name] + ordering ["postgresql", "#{rabbit_settings[:pacemaker_resource]}", "cl-keystone", clone_name] score "Optional" action :create only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) } From 716abe7605d236e17d999c15e15699bde03748a1 Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Thu, 30 Mar 2017 14:23:44 +0200 Subject: [PATCH 10/19] rabbitmq: Rework how we wait for clustered rabbitmq to be up Due to how the OCF resource agent is working, with the rabbit app being started, and then stopped, and started again post-promotion, our old way of waiting for rabbitmq was racy. Now we wait for the pacemaker resource to be up, and check that the rabbit app is up too. (cherry picked from commit 06d545f7fd07f22e0425c1521cdf11c3a8d6eb55) --- chef/cookbooks/rabbitmq/recipes/ha_cluster.rb | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb index e1d60f6edb..4233aab60e 100644 --- a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb +++ b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb @@ -13,6 +13,8 @@ # limitations under the License. # +pid_file = "/var/run/rabbitmq/pid" + agent_name = "ocf:rabbitmq:rabbitmq-server-ha" rabbitmq_op = {} rabbitmq_op["monitor"] = {} @@ -27,7 +29,8 @@ agent agent_name # nodename is empty so that we explicitly depend on the config files params ({ - "erlang_cookie" => node[:rabbitmq][:erlang_cookie] + "erlang_cookie" => node[:rabbitmq][:erlang_cookie], + "pid_file" => pid_file }) op rabbitmq_op action :update @@ -69,28 +72,15 @@ block do require "timeout" begin - Timeout.timeout(30) do - # Check that the service has a master - cmd = "crm resource show #{ms_name} 2> /dev/null | grep -q \"is running on.*Master\"" - while ! ::Kernel.system(cmd) - Chef::Log.debug("#{service_name} still without master") - sleep(2) - end - # Check that the master is this node - cmd = "crm resource show #{service_name} | grep -q \" #{node.hostname} *Master$\"" - if ::Kernel.system(cmd) - # The sed command grabs everything between '{running_applications' - # and ']}', and what we want is that the rabbit application is - # running - cmd = "rabbitmqctl -q status 2> /dev/null| sed -n '/{running_applications/,/\]}/p' | grep -q '{rabbit,'" - while ! ::Kernel.system(cmd) - Chef::Log.debug("#{service_name} still not answering") - sleep(2) - end - end + # 30s (our usual timeout) is a bit short with this OCF RA which + # stops/starts the rabbit app multiple times due to the master-slave + # config + Timeout.timeout(60) do + ::Kernel.system("crm_resource --wait --resource #{ms_name}") + ::Kernel.system("rabbitmqctl wait #{pid_file}") end rescue Timeout::Error - message = "The #{service_name} pacemaker resource is not started. Please manually check for an error." + message = "RabbitMQ is not started. Please manually check for an error." Chef::Log.fatal(message) raise message end From 46020eaf5f7704cd1392c80e798de7950477d4df Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Thu, 30 Mar 2017 14:25:11 +0200 Subject: [PATCH 11/19] rabbitmq: Add additional parameters for rabbitmq pacemaker resource The rabbitmq-server-ha OCF resource agent is designed to work with rabbitmq 3.6.x by default, we need to turn off some features for the version we ship (3.4.x). (cherry picked from commit 629d084ff154b49a6265dafbe56de797192c3a32) --- chef/cookbooks/rabbitmq/recipes/ha_cluster.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb index 4233aab60e..77bd8f2d2c 100644 --- a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb +++ b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb @@ -30,7 +30,9 @@ # nodename is empty so that we explicitly depend on the config files params ({ "erlang_cookie" => node[:rabbitmq][:erlang_cookie], - "pid_file" => pid_file + "pid_file" => pid_file, + "rmq_feature_health_check" => false, + "rmq_feature_local_list_queues" => false }) op rabbitmq_op action :update From 3f856484efbb01c6f29f7dd572d7b0e4cedbf70d Mon Sep 17 00:00:00 2001 From: Vincent Untz Date: Wed, 5 Apr 2017 11:10:22 +0200 Subject: [PATCH 12/19] rabbitmq: Define vhost to watch for pacemaker resource This is a new parameter for the rabbitmq-server-ha OCF resource agent, introduced in https://github.com/rabbitmq/rabbitmq-server-release/pull/24 It's required to make sure that we monitor the right vhost when ensuring synchronization. (cherry picked from commit 373561c9d2d616313c93b0fe39f99dd81b586a98) --- chef/cookbooks/rabbitmq/recipes/ha_cluster.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb index 77bd8f2d2c..0247f6e287 100644 --- a/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb +++ b/chef/cookbooks/rabbitmq/recipes/ha_cluster.rb @@ -32,7 +32,8 @@ "erlang_cookie" => node[:rabbitmq][:erlang_cookie], "pid_file" => pid_file, "rmq_feature_health_check" => false, - "rmq_feature_local_list_queues" => false + "rmq_feature_local_list_queues" => false, + "default_vhost" => node[:rabbitmq][:vhost] }) op rabbitmq_op action :update From 3e76981a3e633fce950cc7e0b3cbfa23c23fa65f Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Tue, 30 May 2017 17:33:02 +0200 Subject: [PATCH 13/19] rabbit: avoid migrations on backport In order to avoid migrations in backported patch, removing earlier introduced migration scripts. Given that the default value for cluster is false, this will leave rabbit on it's original, non-clustered configuration. --- .../crowbar/migrate/rabbitmq/012_cluster.rb | 12 ------------ chef/data_bags/crowbar/template-rabbitmq.json | 2 +- chef/data_bags/crowbar/template-rabbitmq.schema | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb diff --git a/chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb b/chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb deleted file mode 100644 index a211d177a3..0000000000 --- a/chef/data_bags/crowbar/migrate/rabbitmq/012_cluster.rb +++ /dev/null @@ -1,12 +0,0 @@ -def upgrade(ta, td, a, d) - unless a.key?("cluster") - # don't convert anything existing to cluster - a["cluster"] = false - end - return a, d -end - -def downgrade(ta, td, a, d) - a.delete("cluster") unless ta.key?("cluster") - return a, d -end diff --git a/chef/data_bags/crowbar/template-rabbitmq.json b/chef/data_bags/crowbar/template-rabbitmq.json index 9dedf8f337..0c58aa3523 100644 --- a/chef/data_bags/crowbar/template-rabbitmq.json +++ b/chef/data_bags/crowbar/template-rabbitmq.json @@ -34,7 +34,7 @@ "rabbitmq": { "crowbar-revision": 0, "crowbar-applied": false, - "schema-revision": 12, + "schema-revision": 11, "element_states": { "rabbitmq-server": [ "readying", "ready", "applying" ] }, diff --git a/chef/data_bags/crowbar/template-rabbitmq.schema b/chef/data_bags/crowbar/template-rabbitmq.schema index 70f8b16168..36b70a47d0 100644 --- a/chef/data_bags/crowbar/template-rabbitmq.schema +++ b/chef/data_bags/crowbar/template-rabbitmq.schema @@ -17,7 +17,7 @@ "password": { "type": "str", "required": true }, "user": { "type": "str", "required": true }, "vhost": { "type": "str", "required": true }, - "cluster": { "type": "bool", "required": true }, + "cluster": { "type": "bool", "required": false }, "ha" : { "type": "map", "required": true, From 67b2b95dac739c28f182d2b112b273955c730f4b Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Wed, 31 May 2017 15:41:41 +0200 Subject: [PATCH 14/19] Remove ssl settings While backporting the patch, accidentally some rabbit-ssl related code sneaked in. Removing that. --- .../crowbar-openstack/libraries/helpers.rb | 38 +------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index a7cd7d6ed4..80b8fee4e5 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -118,69 +118,35 @@ def self.rabbitmq_settings(node, barclamp) Chef::Log.warn("No RabbitMQ server found!") else one_rabbit = rabbits.first - client_ca_certs = if one_rabbit[:rabbitmq][:ssl][:enabled] && \ - !one_rabbit[:rabbitmq][:ssl][:insecure] - one_rabbit[:rabbitmq][:ssl][:client_ca_certs] - end if rabbits.first[:rabbitmq][:cluster] rabbit_hosts = rabbits.map do |rabbit| - port = if rabbit[:rabbitmq][:ssl][:enabled] - rabbit[:rabbitmq][:ssl][:port] - else - rabbit[:rabbitmq][:port] - end + port = rabbit[:rabbitmq][:port] "#{rabbit[:rabbitmq][:user]}:" \ "#{rabbit[:rabbitmq][:password]}@" \ "#{rabbit[:rabbitmq][:address]}:#{port}" end - trove_rabbit_hosts = rabbits.map do |rabbit| - port = if rabbit[:rabbitmq][:ssl][:enabled] - rabbit[:rabbitmq][:ssl][:port] - else - rabbit[:rabbitmq][:port] - end - - "#{rabbit[:rabbitmq][:trove][:user]}:" \ - "#{rabbit[:rabbitmq][:trove][:password]}@" \ - "#{rabbit[:rabbitmq][:address]}:#{port}" - end - @rabbitmq_settings[instance] = { ha_queues: true, durable_queues: true, - use_ssl: one_rabbit[:rabbitmq][:ssl][:enabled], - client_ca_certs: client_ca_certs, url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ "#{rabbits.first[:rabbitmq][:vhost]}", - trove_url: "rabbit://#{trove_rabbit_hosts.sort.join(",")}/" \ - "#{rabbits.first[:rabbitmq][:trove][:vhost]}", pacemaker_resource: "ms-rabbitmq" } Chef::Log.info("RabbitMQ cluster found") else rabbit = one_rabbit - port = if rabbit[:rabbitmq][:ssl][:enabled] - rabbit[:rabbitmq][:ssl][:port] - else - rabbit[:rabbitmq][:port] - end + port = rabbit[:rabbitmq][:port] @rabbitmq_settings[instance] = { ha_queues: false, durable_queues: false, - use_ssl: rabbit[:rabbitmq][:ssl][:enabled], - client_ca_certs: client_ca_certs, url: "rabbit://#{rabbit[:rabbitmq][:user]}:" \ "#{rabbit[:rabbitmq][:password]}@" \ "#{rabbit[:rabbitmq][:address]}:#{port}/" \ "#{rabbit[:rabbitmq][:vhost]}", - trove_url: "rabbit://#{rabbit[:rabbitmq][:trove][:user]}:" \ - "#{rabbit[:rabbitmq][:trove][:password]}@" \ - "#{rabbit[:rabbitmq][:address]}:#{port}/" \ - "#{rabbit[:rabbitmq][:trove][:vhost]}", pacemaker_resource: "rabbitmq" } From 66c8bf810ef1aa971f4e98a8f83faa1bf70a66bc Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Wed, 31 May 2017 16:57:44 +0200 Subject: [PATCH 15/19] Add transport_url for missing components --- .../cookbooks/ceilometer/templates/default/ceilometer.conf.erb | 2 +- chef/cookbooks/cinder/templates/default/cinder.conf.erb | 2 +- chef/cookbooks/glance/templates/default/glance-api.conf.erb | 2 +- .../glance/templates/default/glance-registry.conf.erb | 2 +- chef/cookbooks/heat/templates/default/heat.conf.erb | 2 +- chef/cookbooks/keystone/templates/default/keystone.conf.erb | 3 +-- chef/cookbooks/manila/templates/default/manila.conf.erb | 2 +- chef/cookbooks/neutron/templates/default/neutron.conf.erb | 2 +- chef/cookbooks/nova/templates/default/nova.conf.erb | 2 +- 9 files changed, 9 insertions(+), 10 deletions(-) diff --git a/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb b/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb index 0b56709f59..bcab39a829 100644 --- a/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb +++ b/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb @@ -277,7 +277,7 @@ use_stderr = false # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) diff --git a/chef/cookbooks/cinder/templates/default/cinder.conf.erb b/chef/cookbooks/cinder/templates/default/cinder.conf.erb index c7bf37b158..59f9978f04 100644 --- a/chef/cookbooks/cinder/templates/default/cinder.conf.erb +++ b/chef/cookbooks/cinder/templates/default/cinder.conf.erb @@ -2689,7 +2689,7 @@ rpc_response_timeout = <%= node[:cinder][:rpc_response_timeout] %> # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) diff --git a/chef/cookbooks/glance/templates/default/glance-api.conf.erb b/chef/cookbooks/glance/templates/default/glance-api.conf.erb index ab7b206d0c..a60de5add1 100644 --- a/chef/cookbooks/glance/templates/default/glance-api.conf.erb +++ b/chef/cookbooks/glance/templates/default/glance-api.conf.erb @@ -522,7 +522,7 @@ notification_driver = messaging # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) diff --git a/chef/cookbooks/glance/templates/default/glance-registry.conf.erb b/chef/cookbooks/glance/templates/default/glance-registry.conf.erb index 1ea86f515f..46a32d3359 100644 --- a/chef/cookbooks/glance/templates/default/glance-registry.conf.erb +++ b/chef/cookbooks/glance/templates/default/glance-registry.conf.erb @@ -336,7 +336,7 @@ notification_driver = messaging # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) diff --git a/chef/cookbooks/heat/templates/default/heat.conf.erb b/chef/cookbooks/heat/templates/default/heat.conf.erb index d254f6b9b7..a4813c614a 100644 --- a/chef/cookbooks/heat/templates/default/heat.conf.erb +++ b/chef/cookbooks/heat/templates/default/heat.conf.erb @@ -395,7 +395,7 @@ use_stderr=false # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) diff --git a/chef/cookbooks/keystone/templates/default/keystone.conf.erb b/chef/cookbooks/keystone/templates/default/keystone.conf.erb index cd87e18fbf..8ef7339434 100644 --- a/chef/cookbooks/keystone/templates/default/keystone.conf.erb +++ b/chef/cookbooks/keystone/templates/default/keystone.conf.erb @@ -137,7 +137,6 @@ log_file = keystone.log # Deprecated group/name - [DEFAULT]/logdir #log_dir = log_dir = /var/log/keystone -transport_url = <%= @rabbit_settings[:url] %> # Use syslog for logging. Existing syslog format is DEPRECATED and will be # changed later to honor RFC5424. (boolean value) @@ -247,7 +246,7 @@ use_syslog = <%= @use_syslog ? "True" : "False" %> # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) diff --git a/chef/cookbooks/manila/templates/default/manila.conf.erb b/chef/cookbooks/manila/templates/default/manila.conf.erb index a5a069bd86..78f5ee7392 100644 --- a/chef/cookbooks/manila/templates/default/manila.conf.erb +++ b/chef/cookbooks/manila/templates/default/manila.conf.erb @@ -1315,7 +1315,7 @@ use_stderr = false # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) diff --git a/chef/cookbooks/neutron/templates/default/neutron.conf.erb b/chef/cookbooks/neutron/templates/default/neutron.conf.erb index 1a6dfbaa28..a075e7052d 100644 --- a/chef/cookbooks/neutron/templates/default/neutron.conf.erb +++ b/chef/cookbooks/neutron/templates/default/neutron.conf.erb @@ -613,7 +613,7 @@ notification_driver = neutron.openstack.common.notifier.rpc_notifier # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend # option and driver specific configuration. (string value) -# transport_url= +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other # drivers include qpid and zmq. (string value) diff --git a/chef/cookbooks/nova/templates/default/nova.conf.erb b/chef/cookbooks/nova/templates/default/nova.conf.erb index 9ac765e848..1d24feff78 100644 --- a/chef/cookbooks/nova/templates/default/nova.conf.erb +++ b/chef/cookbooks/nova/templates/default/nova.conf.erb @@ -1500,7 +1500,7 @@ notification_driver = messaging # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) -#transport_url = +transport_url = <%= @rabbit_settings[:url] %> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) From 06e39f3b4036307fa1714030899c7b893f89e1ad Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Thu, 1 Jun 2017 10:38:03 +0200 Subject: [PATCH 16/19] expose the same variables for templates Earlier changes modified the structure of `rabbitmq_settings` hash, which meant that even if people just deployed new cookbooks, the next periodic chef run would modify config files and thus trigger service restarts. This change adds back the same structures and also adds a flag to the `rabbitmq_settings` structure to indicate it's legaciness. --- .../ceilometer/templates/default/ceilometer.conf.erb | 12 ++++++++++++ .../cinder/templates/default/cinder.conf.erb | 12 ++++++++++++ .../cookbooks/crowbar-openstack/libraries/helpers.rb | 7 +++++++ .../glance/templates/default/glance-api.conf.erb | 12 ++++++++++++ .../templates/default/glance-registry.conf.erb | 12 ++++++++++++ chef/cookbooks/heat/templates/default/heat.conf.erb | 12 ++++++++++++ .../keystone/templates/default/keystone.conf.erb | 12 ++++++++++++ .../manila/templates/default/manila.conf.erb | 12 ++++++++++++ .../neutron/templates/default/neutron.conf.erb | 12 ++++++++++++ chef/cookbooks/nova/templates/default/nova.conf.erb | 12 ++++++++++++ 10 files changed, 115 insertions(+) diff --git a/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb b/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb index bcab39a829..34b3101d46 100644 --- a/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb +++ b/chef/cookbooks/ceilometer/templates/default/ceilometer.conf.erb @@ -277,7 +277,11 @@ use_stderr = false # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) @@ -1168,7 +1172,11 @@ lock_path = /var/run/ceilometer # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1269,7 +1277,11 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/cinder/templates/default/cinder.conf.erb b/chef/cookbooks/cinder/templates/default/cinder.conf.erb index 59f9978f04..f4bf7a198f 100644 --- a/chef/cookbooks/cinder/templates/default/cinder.conf.erb +++ b/chef/cookbooks/cinder/templates/default/cinder.conf.erb @@ -2689,7 +2689,11 @@ rpc_response_timeout = <%= node[:cinder][:rpc_response_timeout] %> # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) @@ -3464,7 +3468,11 @@ lock_path = /var/run/cinder # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -3562,7 +3570,11 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index 80b8fee4e5..82babb9653 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -131,6 +131,7 @@ def self.rabbitmq_settings(node, barclamp) @rabbitmq_settings[instance] = { ha_queues: true, durable_queues: true, + use_legacy_configuration: false, url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ "#{rabbits.first[:rabbitmq][:vhost]}", pacemaker_resource: "ms-rabbitmq" @@ -143,6 +144,12 @@ def self.rabbitmq_settings(node, barclamp) @rabbitmq_settings[instance] = { ha_queues: false, durable_queues: false, + use_legacy_configuration: true, + address: rabbit[:rabbitmq][:address], + port: rabbit[:rabbitmq][:port], + user: rabbit[:rabbitmq][:user], + password: rabbit[:rabbitmq][:password], + vhost: rabbit[:rabbitmq][:vhost], url: "rabbit://#{rabbit[:rabbitmq][:user]}:" \ "#{rabbit[:rabbitmq][:password]}@" \ "#{rabbit[:rabbitmq][:address]}:#{port}/" \ diff --git a/chef/cookbooks/glance/templates/default/glance-api.conf.erb b/chef/cookbooks/glance/templates/default/glance-api.conf.erb index a60de5add1..ea78afe4d7 100644 --- a/chef/cookbooks/glance/templates/default/glance-api.conf.erb +++ b/chef/cookbooks/glance/templates/default/glance-api.conf.erb @@ -522,7 +522,11 @@ notification_driver = messaging # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) @@ -1389,7 +1393,11 @@ lock_path = /var/run/glance # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1493,7 +1501,11 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/glance/templates/default/glance-registry.conf.erb b/chef/cookbooks/glance/templates/default/glance-registry.conf.erb index 46a32d3359..122c7b24f7 100644 --- a/chef/cookbooks/glance/templates/default/glance-registry.conf.erb +++ b/chef/cookbooks/glance/templates/default/glance-registry.conf.erb @@ -336,7 +336,11 @@ notification_driver = messaging # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) @@ -1123,7 +1127,11 @@ admin_tenant_name = <%= @keystone_settings['service_tenant'] %> # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1227,7 +1235,11 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/heat/templates/default/heat.conf.erb b/chef/cookbooks/heat/templates/default/heat.conf.erb index a4813c614a..10f481476e 100644 --- a/chef/cookbooks/heat/templates/default/heat.conf.erb +++ b/chef/cookbooks/heat/templates/default/heat.conf.erb @@ -395,7 +395,11 @@ use_stderr=false # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) @@ -1396,7 +1400,11 @@ admin_tenant_name=<%= @keystone_settings['service_tenant'] %> # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1492,7 +1500,11 @@ rabbit_virtual_host=<%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer diff --git a/chef/cookbooks/keystone/templates/default/keystone.conf.erb b/chef/cookbooks/keystone/templates/default/keystone.conf.erb index 8ef7339434..e3c3b3a201 100644 --- a/chef/cookbooks/keystone/templates/default/keystone.conf.erb +++ b/chef/cookbooks/keystone/templates/default/keystone.conf.erb @@ -246,7 +246,11 @@ use_syslog = <%= @use_syslog ? "True" : "False" %> # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) @@ -1594,7 +1598,11 @@ use_pool = <%= node[:keystone][:ldap][:use_pool] %> # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1690,7 +1698,11 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer diff --git a/chef/cookbooks/manila/templates/default/manila.conf.erb b/chef/cookbooks/manila/templates/default/manila.conf.erb index 78f5ee7392..aa32556326 100644 --- a/chef/cookbooks/manila/templates/default/manila.conf.erb +++ b/chef/cookbooks/manila/templates/default/manila.conf.erb @@ -1315,7 +1315,11 @@ use_stderr = false # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend option # and driver specific configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers # include qpid and zmq. (string value) @@ -1878,7 +1882,11 @@ lock_path=/var/run/manila # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1981,7 +1989,11 @@ rabbit_virtual_host=<%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this # option, you must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down # if heartbeat's keep-alive fails (0 disable the heartbeat). diff --git a/chef/cookbooks/neutron/templates/default/neutron.conf.erb b/chef/cookbooks/neutron/templates/default/neutron.conf.erb index a075e7052d..74a05b60d1 100644 --- a/chef/cookbooks/neutron/templates/default/neutron.conf.erb +++ b/chef/cookbooks/neutron/templates/default/neutron.conf.erb @@ -613,7 +613,11 @@ notification_driver = neutron.openstack.common.notifier.rpc_notifier # A URL representing the messaging driver to use and its full # configuration. If not set, we fall back to the rpc_backend # option and driver specific configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other # drivers include qpid and zmq. (string value) @@ -1014,7 +1018,11 @@ lock_path = /var/run/neutron # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -1102,7 +1110,11 @@ rabbit_virtual_host=<%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Deprecated, use rpc_backend=kombu+memory or rpc_backend=fake (boolean value) # Deprecated group/name - [DEFAULT]/fake_rabbit diff --git a/chef/cookbooks/nova/templates/default/nova.conf.erb b/chef/cookbooks/nova/templates/default/nova.conf.erb index 1d24feff78..7735969aff 100644 --- a/chef/cookbooks/nova/templates/default/nova.conf.erb +++ b/chef/cookbooks/nova/templates/default/nova.conf.erb @@ -1500,7 +1500,11 @@ notification_driver = messaging # A URL representing the messaging driver to use and its full configuration. If # not set, we fall back to the rpc_backend option and driver specific # configuration. (string value) +<% if @rabbit_settings['use_legacy_configuration'] -%> +#transport_url = +<% else -%> transport_url = <%= @rabbit_settings[:url] %> +<% end -%> # The messaging driver to use, defaults to rabbit. Other drivers include qpid # and zmq. (string value) @@ -3191,7 +3195,11 @@ lock_path = /var/run/nova # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#amqp_durable_queues = false +<% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> +<% end -%> # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete @@ -3287,7 +3295,11 @@ rabbit_virtual_host = <%= @rabbit_settings[:vhost] %> # Use HA queues in RabbitMQ (x-ha-policy: all). If you change this option, you # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues +<% if @rabbit_settings['use_legacy_configuration'] -%> +#rabbit_ha_queues = false +<% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> +<% end -%> # Number of seconds after which the Rabbit broker is considered down if # heartbeat's keep-alive fails (0 disable the heartbeat). EXPERIMENTAL (integer From 85cddabaf6b7a4877195799331c2af8a0357b30e Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Tue, 6 Jun 2017 10:29:36 +0200 Subject: [PATCH 17/19] refactor: Use one_rabbit variable replace occurences of rabbits.first --- chef/cookbooks/crowbar-openstack/libraries/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index 82babb9653..9bee9d74af 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -119,7 +119,7 @@ def self.rabbitmq_settings(node, barclamp) else one_rabbit = rabbits.first - if rabbits.first[:rabbitmq][:cluster] + if one_rabbit[:rabbitmq][:cluster] rabbit_hosts = rabbits.map do |rabbit| port = rabbit[:rabbitmq][:port] @@ -133,7 +133,7 @@ def self.rabbitmq_settings(node, barclamp) durable_queues: true, use_legacy_configuration: false, url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ - "#{rabbits.first[:rabbitmq][:vhost]}", + "#{one_rabbit[:rabbitmq][:vhost]}", pacemaker_resource: "ms-rabbitmq" } Chef::Log.info("RabbitMQ cluster found") From 6d4bd35d4dee0b70e14785882b90b0dcee804687 Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Tue, 6 Jun 2017 10:50:07 +0200 Subject: [PATCH 18/19] Format neutron config file In order to avoid file changes and thus service restarts, re format the code so it look like the original one. --- chef/cookbooks/neutron/templates/default/neutron.conf.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/neutron/templates/default/neutron.conf.erb b/chef/cookbooks/neutron/templates/default/neutron.conf.erb index 74a05b60d1..f72306d406 100644 --- a/chef/cookbooks/neutron/templates/default/neutron.conf.erb +++ b/chef/cookbooks/neutron/templates/default/neutron.conf.erb @@ -614,7 +614,7 @@ notification_driver = neutron.openstack.common.notifier.rpc_notifier # configuration. If not set, we fall back to the rpc_backend # option and driver specific configuration. (string value) <% if @rabbit_settings['use_legacy_configuration'] -%> -#transport_url = +# transport_url= <% else -%> transport_url = <%= @rabbit_settings[:url] %> <% end -%> @@ -1019,7 +1019,7 @@ lock_path = /var/run/neutron # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_durable_queues <% if @rabbit_settings['use_legacy_configuration'] -%> -#amqp_durable_queues = false +# amqp_durable_queues = false <% else -%> amqp_durable_queues = <%= @rabbit_settings[:durable_queues] %> <% end -%> @@ -1111,7 +1111,7 @@ rabbit_virtual_host=<%= @rabbit_settings[:vhost] %> # must wipe the RabbitMQ database. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_ha_queues <% if @rabbit_settings['use_legacy_configuration'] -%> -#rabbit_ha_queues = false +# rabbit_ha_queues = false <% else -%> rabbit_ha_queues = <%= @rabbit_settings[:ha_queues] %> <% end -%> From a93b38e90c9f711d7ad3d68710ec08e23d02cfc6 Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Fri, 16 Jun 2017 16:30:06 +0200 Subject: [PATCH 19/19] Fix: add nodes to rabbit conf The variable @cluster_nodes was not exposed to the template. Amended the template, so configuration will not change in case we install the package on top of a non-clustered setup (that's why the clustered variable was introduced), and export the list of hosts. --- .../crowbar-openstack/libraries/helpers.rb | 11 ++++++++++- chef/cookbooks/rabbitmq/recipes/default.rb | 13 +++++++++++++ .../rabbitmq/templates/default/rabbitmq.config.erb | 2 ++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb index 9bee9d74af..bc19182bd3 100644 --- a/chef/cookbooks/crowbar-openstack/libraries/helpers.rb +++ b/chef/cookbooks/crowbar-openstack/libraries/helpers.rb @@ -128,13 +128,21 @@ def self.rabbitmq_settings(node, barclamp) "#{rabbit[:rabbitmq][:address]}:#{port}" end + rabbit_node_names = rabbits.map do |rabbit_node| + "\'rabbit@#{rabbit_node.name}\'" + end + + cluster_nodes = rabbit_node_names.join(",") + @rabbitmq_settings[instance] = { + clustered: true, ha_queues: true, durable_queues: true, use_legacy_configuration: false, url: "rabbit://#{rabbit_hosts.sort.join(",")}/" \ "#{one_rabbit[:rabbitmq][:vhost]}", - pacemaker_resource: "ms-rabbitmq" + pacemaker_resource: "ms-rabbitmq", + cluster_nodes: cluster_nodes } Chef::Log.info("RabbitMQ cluster found") else @@ -142,6 +150,7 @@ def self.rabbitmq_settings(node, barclamp) port = rabbit[:rabbitmq][:port] @rabbitmq_settings[instance] = { + clustered: false, ha_queues: false, durable_queues: false, use_legacy_configuration: true, diff --git a/chef/cookbooks/rabbitmq/recipes/default.rb b/chef/cookbooks/rabbitmq/recipes/default.rb index d306df2d6b..6e78819de2 100644 --- a/chef/cookbooks/rabbitmq/recipes/default.rb +++ b/chef/cookbooks/rabbitmq/recipes/default.rb @@ -36,12 +36,25 @@ notifies :restart, "service[rabbitmq-server]" end +rabbit_settings = fetch_rabbitmq_settings +if rabbit_settings[:clustered] + add_cluster_section = true + cluster_nodes = rabbit_settings[:cluster_nodes] +else + add_cluster_section = false + cluster_nodes = "" +end + template "/etc/rabbitmq/rabbitmq.config" do source "rabbitmq.config.erb" owner "root" group "root" mode 0644 notifies :restart, "service[rabbitmq-server]" + variables( + add_cluster_section: add_cluster_section, + cluster_nodes: cluster_nodes + ) end case node[:platform_family] diff --git a/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb b/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb index 1544b87158..72349d964a 100644 --- a/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb +++ b/chef/cookbooks/rabbitmq/templates/default/rabbitmq.config.erb @@ -4,10 +4,12 @@ {tcp_listeners, [ <%= node[:rabbitmq][:addresses].map { |address| "{\"#{address}\", #{node[:rabbitmq][:port]}}" }.join(", ") %> ]}, +<% if @add_cluster_section -%> {cluster_nodes, {[ <%= @cluster_nodes %> ], disc} }, +<% end -%> {disk_free_limit, 50000000} ] },