From be40a880401f198d0ae0589f557dc611dc11a3d1 Mon Sep 17 00:00:00 2001 From: Suzana Stavreva Date: Thu, 16 Mar 2017 11:27:35 +0100 Subject: [PATCH 001/181] Barclamp skeleton --- bin/crowbar_oscm | 22 +++ chef/cookbooks/oscm/README.md | 1 + chef/cookbooks/oscm/metadata.rb | 13 ++ chef/cookbooks/oscm/recipes/main.rb | 0 .../oscm/recipes/role_oscm_server.rb | 3 + chef/data_bags/crowbar/template-oscm.json | 64 ++++++++ chef/data_bags/crowbar/template-oscm.schema | 147 ++++++++++++++++++ chef/roles/oscm-server.rb | 5 + .../app/controllers/oscm_controller.rb | 25 +++ .../app/helpers/barclamp/oscm_helper.rb | 29 ++++ crowbar_framework/app/models/oscm_service.rb | 78 ++++++++++ .../barclamp/oscm/_edit_attributes.html.haml | 71 +++++++++ crowbar_framework/config/locales/oscm/en.yml | 56 +++++++ oscm.yml | 30 ++++ 14 files changed, 544 insertions(+) create mode 100644 bin/crowbar_oscm create mode 100644 chef/cookbooks/oscm/README.md create mode 100644 chef/cookbooks/oscm/metadata.rb create mode 100644 chef/cookbooks/oscm/recipes/main.rb create mode 100644 chef/cookbooks/oscm/recipes/role_oscm_server.rb create mode 100644 chef/data_bags/crowbar/template-oscm.json create mode 100644 chef/data_bags/crowbar/template-oscm.schema create mode 100644 chef/roles/oscm-server.rb create mode 100644 crowbar_framework/app/controllers/oscm_controller.rb create mode 100644 crowbar_framework/app/helpers/barclamp/oscm_helper.rb create mode 100644 crowbar_framework/app/models/oscm_service.rb create mode 100644 crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml create mode 100644 crowbar_framework/config/locales/oscm/en.yml create mode 100644 oscm.yml diff --git a/bin/crowbar_oscm b/bin/crowbar_oscm new file mode 100644 index 0000000000..7e69f96668 --- /dev/null +++ b/bin/crowbar_oscm @@ -0,0 +1,22 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require File.join(File.expand_path(File.dirname(__FILE__)), "barclamp_lib") +@barclamp = "oscm" +@timeout = 3600 + +main + diff --git a/chef/cookbooks/oscm/README.md b/chef/cookbooks/oscm/README.md new file mode 100644 index 0000000000..a413e4b084 --- /dev/null +++ b/chef/cookbooks/oscm/README.md @@ -0,0 +1 @@ +Chef Cookbook to install and configure OSCM \ No newline at end of file diff --git a/chef/cookbooks/oscm/metadata.rb b/chef/cookbooks/oscm/metadata.rb new file mode 100644 index 0000000000..9fe6a199e6 --- /dev/null +++ b/chef/cookbooks/oscm/metadata.rb @@ -0,0 +1,13 @@ +name "oscm" +maintainer "EST" +maintainer_email "arkadiusz.kowalczyk@ts.fujistu.com" +license "Apache 2.0" +description "Installs/Configures OSCM" +long_description IO.read(File.join(File.dirname(__FILE__), "README.md")) +version "0.1" + +depends "database" +depends "keystone" +depends "crowbar-openstack" +depends "crowbar-pacemaker" +depends "utils" diff --git a/chef/cookbooks/oscm/recipes/main.rb b/chef/cookbooks/oscm/recipes/main.rb new file mode 100644 index 0000000000..e69de29bb2 diff --git a/chef/cookbooks/oscm/recipes/role_oscm_server.rb b/chef/cookbooks/oscm/recipes/role_oscm_server.rb new file mode 100644 index 0000000000..6894c3632b --- /dev/null +++ b/chef/cookbooks/oscm/recipes/role_oscm_server.rb @@ -0,0 +1,3 @@ +if CrowbarRoleRecipe.node_state_valid_for_role?(node, "oscm", "oscm-server") + include_recipe "oscm::main" +end diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json new file mode 100644 index 0000000000..b7d27d76e4 --- /dev/null +++ b/chef/data_bags/crowbar/template-oscm.json @@ -0,0 +1,64 @@ +{ + "id": "template-oscm", + "description": "OSCM", + "attributes": { + "oscm" : { + "timeout": 300, + "debug" : false, + "heat_template" : { + "image" : "sles12-docker", + "flavor" : "oscm.small", + "keypair": "oscm", + "user": "oscm", + "project":"oscm" + }, + "mail": { + "auth": "true", + "host": "", + "port": 25, + "user": "", + "password": "", + "from": "" + }, + "docker": { + "host": "", + "port": 0 + }, + "proxy": { + "http_host": "", + "http_port": 0, + "https_host": "", + "https_port": 0 + }, + "ssl": { + "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", + "keyfile": "/etc/oscm/ssl/private/signing_key.pem", + "generate_certs": false, + "insecure": false, + "ca_certs": "/etc/oscm/ssl/certs/ca.pem" + } + } + }, + "deployment": { + "oscm" : { + "crowbar-revision": 0, + "crowbar-applied": false, + "schema-revision": 100, + "element_states": { + "oscm-server": [ "readying", "ready", "applying" ] + }, + "element_order": [ + [ "oscm-server" ] + ], + "elements": {}, + "element_run_list_order": { + "oscm-server": 110 + }, + "config": { + "mode": "full", + "transitions": false, + "transition_list": [] + } + } + } +} diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema new file mode 100644 index 0000000000..c16817d249 --- /dev/null +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -0,0 +1,147 @@ +{ + "type": "map", + "required": true, + "mapping": { + "id": { "type": "str", "required": true}, + "description": { "type": "str", "required": true }, + "attributes": { + "type": "map", + "required": true, + "mapping": { + "oscm": { + "type": "map", + "required": true, + "mapping": { + "timeout": { "type": "int", "required": true }, + "debug": { "type": "bool", "required": true }, + "heat_template" : { + "type" : "map", + "required" : true, + "mapping": { + "image": { "type": "str", "required": true }, + "flavor": { "type": "str", "required": true }, + "keypair": { "type": "str", "required": true }, + "user": { "type": "str", "required": true }, + "project": { "type": "str", "required": true } + } + }, + "mail" : { + "type" : "map", + "required" : true, + "mapping": { + "auth": { "type": "bool", "required": true }, + "host": { "type": "str", "required": true }, + "port": { "type": "int", "required": true }, + "user": { "type": "str", "required": true }, + "password": { "type": "str", "required": true }, + "from": { "type": "str", "required": true } + } + }, + "docker" : { + "type" : "map", + "required" : true, + "mapping": { + "host": { "type": "str", "required": true }, + "port": { "type": "int", "required": true } + } + }, + "proxy" : { + "type" : "map", + "required" : true, + "mapping": { + "http_host": { "type": "str", "required": true }, + "http_port": { "type": "int", "required": true }, + "https_host": { "type": "str", "required": true }, + "https_port": { "type": "int", "required": true } + } + }, + "ssl": { + "type": "map", + "required": true, + "mapping": { + "certfile": { "type" : "str", "required" : true }, + "keyfile": { "type" : "str", "required" : true }, + "generate_certs": { "type" : "bool", "required" : true }, + "insecure": { "type" : "bool", "required" : true }, + "ca_certs": { "type" : "str", "required" : true } + } + } + } + } + } + }, + + "deployment": { + "type": "map", + "required": true, + "mapping": { + "oscm": { + "type": "map", + "required": true, + "mapping": { + "crowbar-revision": { "type": "int", "required": true }, + "crowbar-committing": { "type": "bool" }, + "crowbar-applied": { "type": "bool" }, + "crowbar-status": { "type": "str" }, + "crowbar-failed": { "type": "str" }, + "crowbar-queued": { "type": "bool" }, + "schema-revision": { "type": "int" }, + "element_states": { + "type": "map", + "mapping": { + = : { + "type": "seq", + "required": true, + "sequence": [ { "type": "str" } ] + } + } + }, + "elements": { + "type": "map", + "required": true, + "mapping": { + = : { + "type": "seq", + "required": true, + "sequence": [ { "type": "str" } ] + } + } + }, + "element_order": { + "type": "seq", + "required": true, + "sequence": [ { + "type": "seq", + "sequence": [ { "type": "str" } ] + } ] + }, + "element_run_list_order": { + "type": "map", + "required": false, + "mapping": { + = : { + "type": "int", + "required": true + } + } + }, + "config": { + "type": "map", + "required": true, + "mapping": { + "environment": { "type": "str", "required": true }, + "mode": { "type": "str", "required": true }, + "transitions": { "type": "bool", "required": true }, + "transition_list": { + "type": "seq", + "required": true, + "sequence": [ { "type": "str" } ] + } + } + } + } + } + } + } + } +} diff --git a/chef/roles/oscm-server.rb b/chef/roles/oscm-server.rb new file mode 100644 index 0000000000..ca5a7df52b --- /dev/null +++ b/chef/roles/oscm-server.rb @@ -0,0 +1,5 @@ +name "oscm-server" +description "OSCM Server Role" +run_list("recipe[oscm::role_oscm_server]") +default_attributes +override_attributes \ No newline at end of file diff --git a/crowbar_framework/app/controllers/oscm_controller.rb b/crowbar_framework/app/controllers/oscm_controller.rb new file mode 100644 index 0000000000..d9cdb5e90f --- /dev/null +++ b/crowbar_framework/app/controllers/oscm_controller.rb @@ -0,0 +1,25 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class OscmController < BarclampController + # Controller for Oscm barclamp + + protected + + def initialize_service + @service_object = OscmService.new logger + end +end \ No newline at end of file diff --git a/crowbar_framework/app/helpers/barclamp/oscm_helper.rb b/crowbar_framework/app/helpers/barclamp/oscm_helper.rb new file mode 100644 index 0000000000..a593352cdd --- /dev/null +++ b/crowbar_framework/app/helpers/barclamp/oscm_helper.rb @@ -0,0 +1,29 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Barclamp + module OscmHelper + def api_protocols_for_oscm(selected) + options_for_select( + [ + ["HTTP", "http"], + ["HTTPS", "https"] + ], + selected.to_s + ) + end + end +end diff --git a/crowbar_framework/app/models/oscm_service.rb b/crowbar_framework/app/models/oscm_service.rb new file mode 100644 index 0000000000..e6d9afe474 --- /dev/null +++ b/crowbar_framework/app/models/oscm_service.rb @@ -0,0 +1,78 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class OscmService < PacemakerServiceObject + def initialize(thelogger) + @bc_name = "oscm" + @logger = thelogger + end + + class << self + # Turn off multi proposal support till it really works and people ask for it. + def self.allow_multiple_proposals? + false + end + + def role_constraints + { + "oscm-server" => { + "unique" => false, + "count" => 1, + "admin" => false, + "cluster" => false, + "exclude_platform" => { + "suse" => "< 12.1", + "windows" => "/.*/" + } + }, + } + end + end + + + def proposal_dependencies(role) + answer = [] + ["heat"].each do |dep| + answer << { "barclamp" => dep, "inst" => role.default_attributes[@bc_name]["#{dep}_instance"] } + end + answer + end + + def create_proposal + @logger.debug("Oscm create_proposal: entering") + base = super + + nodes = NodeObject.all + server_nodes = nodes.select { |n| n.intended_role == "controller" } + server_nodes = [nodes.first] if server_nodes.empty? + + base["deployment"][@bc_name]["elements"] = { + "oscm-server" => [server_nodes.first.name] + } unless server_nodes.nil? + + base["attributes"][@bc_name]["heat_instance"] = find_dep_proposal("heat") + + @logger.debug("Oscm create_proposal: exiting") + base + end + + def validate_proposal_after_save(proposal) + validate_one_for_role proposal, "oscm-server" + + super + end + +end diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml new file mode 100644 index 0000000000..4ee22bd1e0 --- /dev/null +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -0,0 +1,71 @@ += attributes_for @proposal do + .panel-sub + = header show_raw_deployment?, true + + .panel-body + + %fieldset + %legend + = t(".mail_settings") + + = string_field %w(mail host) + = string_field %w(mail port) + = string_fiels %w(mail from) + + = boolean_field %w(mail auth), + "data-showit" => "true", + "data-showit-target" => #mail_container, + "data-showit-direct" => "true" + + #mail_container + = string_field %w(mail user) + = password_field %w(mail password) + + %fieldset + %legend + = t(".docker_settings") + + = boolean_field %w(docker use_registry), + "data-showit" => "true", + "data-showit-target" => #docker_container, + "data-showit-direct" => "true" + + #docker_container + = string_field %w(docker host) + = string_field %w(docker port) + + %fieldset + %legend + = t(".proxy_settings") + + = boolean_field %w(proxy use_proxy), + "data-showit" => "true", + "data-showit-target" => #proxy_container, + "data-showit-direct" => "true" + + #proxy_container + = string_field %w(proxy http_host) + = string_field %w(proxy http_port) + = string_field %w(proxy https_host) + = string_field %w(proxy https_port) + + %fieldset + %legend + = t(".ssl_header") + + = select_field %w(api protocol), + :collection => :api_protocols_for_oscm, + "data-sslprefix" => "ssl", + "data-sslcert" => "/etc/oscm/ssl/certs/signing_cert.pem", + "data-sslkey" => "/etc/oscm/ssl/private/signing_key.pem" + + #ssl_container + = boolean_field %w(ssl generate_certs) + = string_field %w(ssl certfile) + = string_field %w(ssl keyfile) + = boolean_field %w(ssl insecure) + = string_field %w(ssl ca_certs) + + + + \ No newline at end of file diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml new file mode 100644 index 0000000000..d01a31c23c --- /dev/null +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -0,0 +1,56 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +en: + barclamp: + oscm: + edit_attributes: + heat_instance: 'Heat' + + mail_settings: 'Mail Settings' + mail: + auth: 'Authentication Required' + host: 'Mail Host' + port: 'SMTP Port' + user: 'User' + password: 'Password' + from: 'Oscm Email Address' + + docker_settings: 'Docker Registry' + docker: + use_registry: 'Install from Local Registry' + host: 'Registry Host' + port: 'Registry Port' + + proxy_settings: 'Proxy Settings' + proxy: + use_proxy: "Use Proxy" + http_host: "HTTP Host" + http_port: "HTTP Port" + https_host: "HTTPS Host" + https_port: "HTTPS Port" + + ssl_header: 'SSL Support' + ssl: + generate_certs: 'Generate (self-signed) certificates (implies insecure)' + certfile: 'SSL Certificate File' + keyfile: 'SSL (Private) Key File' + insecure: 'SSL Certificate is insecure (for instance, self-signed)' + ca_certs: 'SSL CA Certificates File' + + + + \ No newline at end of file diff --git a/oscm.yml b/oscm.yml new file mode 100644 index 0000000000..8f3431c997 --- /dev/null +++ b/oscm.yml @@ -0,0 +1,30 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +barclamp: + name: 'oscm' + display: 'OSCM' + description: 'Open Service Catalog Manager' + version: 1 + member: + - 'openstack' + requires: + - 'heat' + +crowbar: + run_order: 100 + chef_order: 100 + From 33f627dfe6ca7020f726b41471de870b186ef994 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 21 Mar 2017 11:28:16 +0100 Subject: [PATCH 002/181] Fix oscm json and schema --- chef/cookbooks/oscm/metadata.rb | 6 +----- chef/data_bags/crowbar/template-oscm.json | 3 ++- chef/data_bags/crowbar/template-oscm.schema | 1 + crowbar_framework/config/locales/oscm/en.yml | 8 +++----- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/chef/cookbooks/oscm/metadata.rb b/chef/cookbooks/oscm/metadata.rb index 9fe6a199e6..fdb8f4ffda 100644 --- a/chef/cookbooks/oscm/metadata.rb +++ b/chef/cookbooks/oscm/metadata.rb @@ -6,8 +6,4 @@ long_description IO.read(File.join(File.dirname(__FILE__), "README.md")) version "0.1" -depends "database" -depends "keystone" -depends "crowbar-openstack" -depends "crowbar-pacemaker" -depends "utils" +depends "heat" diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index b7d27d76e4..4168a97ca1 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -5,6 +5,7 @@ "oscm" : { "timeout": 300, "debug" : false, + "heat_instance": "none", "heat_template" : { "image" : "sles12-docker", "flavor" : "oscm.small", @@ -13,7 +14,7 @@ "project":"oscm" }, "mail": { - "auth": "true", + "auth": true, "host": "", "port": 25, "user": "", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index c16817d249..35b622e55f 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -14,6 +14,7 @@ "mapping": { "timeout": { "type": "int", "required": true }, "debug": { "type": "bool", "required": true }, + "heat_instance": { "type": "str", "required": true }, "heat_template" : { "type" : "map", "required" : true, diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index d01a31c23c..e7f1e4c5ad 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -34,7 +34,7 @@ en: use_registry: 'Install from Local Registry' host: 'Registry Host' port: 'Registry Port' - + proxy_settings: 'Proxy Settings' proxy: use_proxy: "Use Proxy" @@ -42,7 +42,7 @@ en: http_port: "HTTP Port" https_host: "HTTPS Host" https_port: "HTTPS Port" - + ssl_header: 'SSL Support' ssl: generate_certs: 'Generate (self-signed) certificates (implies insecure)' @@ -50,7 +50,5 @@ en: keyfile: 'SSL (Private) Key File' insecure: 'SSL Certificate is insecure (for instance, self-signed)' ca_certs: 'SSL CA Certificates File' - - - + \ No newline at end of file From 5eeabe91785a83efe6e8dbe3e5c42a03c148a94f Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 21 Mar 2017 13:17:51 +0100 Subject: [PATCH 003/181] Fix oscm view --- .../app/views/barclamp/oscm/_edit_attributes.html.haml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 4ee22bd1e0..0c860d74cb 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -10,11 +10,11 @@ = string_field %w(mail host) = string_field %w(mail port) - = string_fiels %w(mail from) + = string_field %w(mail from) = boolean_field %w(mail auth), "data-showit" => "true", - "data-showit-target" => #mail_container, + "data-showit-target" => "#mail_container", "data-showit-direct" => "true" #mail_container @@ -27,7 +27,7 @@ = boolean_field %w(docker use_registry), "data-showit" => "true", - "data-showit-target" => #docker_container, + "data-showit-target" => "#docker_container", "data-showit-direct" => "true" #docker_container @@ -40,7 +40,7 @@ = boolean_field %w(proxy use_proxy), "data-showit" => "true", - "data-showit-target" => #proxy_container, + "data-showit-target" => "#proxy_container", "data-showit-direct" => "true" #proxy_container From fc78d61ed8c1bb73801156057bb96893fcdfcd55 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 21 Mar 2017 16:18:17 +0100 Subject: [PATCH 004/181] Remove white space --- .../app/views/barclamp/oscm/_edit_attributes.html.haml | 10 +++------- crowbar_framework/config/locales/oscm/en.yml | 5 ----- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 0c860d74cb..c235989ab7 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -1,17 +1,16 @@ = attributes_for @proposal do .panel-sub = header show_raw_deployment?, true - - .panel-body + .panel-body %fieldset %legend = t(".mail_settings") - + = string_field %w(mail host) = string_field %w(mail port) = string_field %w(mail from) - + = boolean_field %w(mail auth), "data-showit" => "true", "data-showit-target" => "#mail_container", @@ -65,7 +64,4 @@ = string_field %w(ssl keyfile) = boolean_field %w(ssl insecure) = string_field %w(ssl ca_certs) - - - \ No newline at end of file diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index e7f1e4c5ad..d8b99a8457 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -19,7 +19,6 @@ en: oscm: edit_attributes: heat_instance: 'Heat' - mail_settings: 'Mail Settings' mail: auth: 'Authentication Required' @@ -28,13 +27,11 @@ en: user: 'User' password: 'Password' from: 'Oscm Email Address' - docker_settings: 'Docker Registry' docker: use_registry: 'Install from Local Registry' host: 'Registry Host' port: 'Registry Port' - proxy_settings: 'Proxy Settings' proxy: use_proxy: "Use Proxy" @@ -42,7 +39,6 @@ en: http_port: "HTTP Port" https_host: "HTTPS Host" https_port: "HTTPS Port" - ssl_header: 'SSL Support' ssl: generate_certs: 'Generate (self-signed) certificates (implies insecure)' @@ -50,5 +46,4 @@ en: keyfile: 'SSL (Private) Key File' insecure: 'SSL Certificate is insecure (for instance, self-signed)' ca_certs: 'SSL CA Certificates File' - \ No newline at end of file From 5cbe076152b849e79d26035d4426d2a528dbbff1 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 23 Mar 2017 11:29:27 +0100 Subject: [PATCH 005/181] Add missing api protocol from view in json and locales --- chef/data_bags/crowbar/template-oscm.json | 3 +++ chef/data_bags/crowbar/template-oscm.schema | 15 +++++++++++---- crowbar_framework/config/locales/oscm/en.yml | 12 +++++++----- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 4168a97ca1..6321911abd 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -37,6 +37,9 @@ "generate_certs": false, "insecure": false, "ca_certs": "/etc/oscm/ssl/certs/ca.pem" + }, + "api": { + "protocol": "http" } } }, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 35b622e55f..8089923066 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -56,16 +56,23 @@ "https_port": { "type": "int", "required": true } } }, - "ssl": { - "type": "map", - "required": true, - "mapping": { + "ssl": { + "type": "map", + "required": true, + "mapping": { "certfile": { "type" : "str", "required" : true }, "keyfile": { "type" : "str", "required" : true }, "generate_certs": { "type" : "bool", "required" : true }, "insecure": { "type" : "bool", "required" : true }, "ca_certs": { "type" : "str", "required" : true } } + }, + "api": { + "type": "map", + "required": true, + "mapping": { + "protocol": { "type" : "str", "required" : true }, + } } } } diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index d8b99a8457..8603f83fd4 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -34,11 +34,13 @@ en: port: 'Registry Port' proxy_settings: 'Proxy Settings' proxy: - use_proxy: "Use Proxy" - http_host: "HTTP Host" - http_port: "HTTP Port" - https_host: "HTTPS Host" - https_port: "HTTPS Port" + use_proxy: 'Use Proxy' + http_host: 'HTTP Host' + http_port: 'HTTP Port' + https_host: 'HTTPS Host' + https_port: 'HTTPS Port' + api: + protocol: 'Protocol' ssl_header: 'SSL Support' ssl: generate_certs: 'Generate (self-signed) certificates (implies insecure)' From 2f0cc210846978287502b87a12303e9657f584f2 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 23 Mar 2017 12:37:18 +0100 Subject: [PATCH 006/181] Fix syntax --- chef/data_bags/crowbar/template-oscm.schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 8089923066..fe6af7875a 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -71,7 +71,7 @@ "type": "map", "required": true, "mapping": { - "protocol": { "type" : "str", "required" : true }, + "protocol": { "type" : "str", "required" : true } } } } From e303dcf6edb891705d6c082fcb6a75a16f0cbff2 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 23 Mar 2017 14:48:20 +0100 Subject: [PATCH 007/181] Add boolean variables --- chef/data_bags/crowbar/template-oscm.json | 2 ++ chef/data_bags/crowbar/template-oscm.schema | 2 ++ .../app/views/barclamp/oscm/_edit_attributes.html.haml | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 6321911abd..c0cde8b4e1 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -22,10 +22,12 @@ "from": "" }, "docker": { + "use_registry": false "host": "", "port": 0 }, "proxy": { + "use_proxy": false, "http_host": "", "http_port": 0, "https_host": "", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index fe6af7875a..8d32bdc3cb 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -42,6 +42,7 @@ "type" : "map", "required" : true, "mapping": { + "use_registry": { "type": "bool", "required": true } "host": { "type": "str", "required": true }, "port": { "type": "int", "required": true } } @@ -50,6 +51,7 @@ "type" : "map", "required" : true, "mapping": { + "use_proxy": { "type": "bool", "required": true } "http_host": { "type": "str", "required": true }, "http_port": { "type": "int", "required": true }, "https_host": { "type": "str", "required": true }, diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index c235989ab7..63b6e63c44 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -25,7 +25,7 @@ = t(".docker_settings") = boolean_field %w(docker use_registry), - "data-showit" => "true", + "data-showit" => "false", "data-showit-target" => "#docker_container", "data-showit-direct" => "true" @@ -38,7 +38,7 @@ = t(".proxy_settings") = boolean_field %w(proxy use_proxy), - "data-showit" => "true", + "data-showit" => "false", "data-showit-target" => "#proxy_container", "data-showit-direct" => "true" From 2f58fc12202fe5e4615c0f010a149a7c80803429 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 23 Mar 2017 15:21:28 +0100 Subject: [PATCH 008/181] Fix json --- chef/data_bags/crowbar/template-oscm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index c0cde8b4e1..de573a8571 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -22,7 +22,7 @@ "from": "" }, "docker": { - "use_registry": false + "use_registry": false, "host": "", "port": 0 }, From fe66d740a4992b03e3eea0689e61689cdca8d933 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 23 Mar 2017 15:24:49 +0100 Subject: [PATCH 009/181] Fix json schema --- chef/data_bags/crowbar/template-oscm.schema | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 8d32bdc3cb..6ddb43c434 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -42,7 +42,7 @@ "type" : "map", "required" : true, "mapping": { - "use_registry": { "type": "bool", "required": true } + "use_registry": { "type": "bool", "required": true }, "host": { "type": "str", "required": true }, "port": { "type": "int", "required": true } } @@ -51,7 +51,7 @@ "type" : "map", "required" : true, "mapping": { - "use_proxy": { "type": "bool", "required": true } + "use_proxy": { "type": "bool", "required": true }, "http_host": { "type": "str", "required": true }, "http_port": { "type": "int", "required": true }, "https_host": { "type": "str", "required": true }, From ab2c0c7fff02b4a7f3fd8b9b69370ffaf7124d19 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 23 Mar 2017 15:59:34 +0100 Subject: [PATCH 010/181] Add UI attributes for proxy and docker --- chef/data_bags/crowbar/template-oscm.json | 8 ++++++-- chef/data_bags/crowbar/template-oscm.schema | 8 ++++++-- .../barclamp/oscm/_edit_attributes.html.haml | 20 +++++++++++++++---- crowbar_framework/config/locales/oscm/en.yml | 4 ++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index de573a8571..1775b6149a 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -24,14 +24,18 @@ "docker": { "use_registry": false, "host": "", - "port": 0 + "port": 0, + "auth": false, + "user": "", + "password": "" }, "proxy": { "use_proxy": false, "http_host": "", "http_port": 0, "https_host": "", - "https_port": 0 + "https_port": 0, + "no_proxy": "localhost,127.0.0.1" }, "ssl": { "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 6ddb43c434..38df12e353 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -44,7 +44,10 @@ "mapping": { "use_registry": { "type": "bool", "required": true }, "host": { "type": "str", "required": true }, - "port": { "type": "int", "required": true } + "port": { "type": "int", "required": true }, + "auth": { "type": "bool", "required": true }, + "user": { "type": "str", "required": true }, + "password": { "type": "str", "required": true } } }, "proxy" : { @@ -55,7 +58,8 @@ "http_host": { "type": "str", "required": true }, "http_port": { "type": "int", "required": true }, "https_host": { "type": "str", "required": true }, - "https_port": { "type": "int", "required": true } + "https_port": { "type": "int", "required": true }, + "no_proxy": { "type": "str", "required": true } } }, "ssl": { diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 63b6e63c44..50ff5cdd45 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -8,7 +8,7 @@ = t(".mail_settings") = string_field %w(mail host) - = string_field %w(mail port) + = integer_field %w(mail port) = string_field %w(mail from) = boolean_field %w(mail auth), @@ -31,7 +31,18 @@ #docker_container = string_field %w(docker host) - = string_field %w(docker port) + = integer_field %w(docker port) + = boolean_field %w(docker auth) + "data-showit" => "false", + "data-showit-target" => "#dockerauth_container", + "data-showit-direct" => "true" + + #dockerauth_container + = string_field %w(docker user) + = password_field %w(docker password) + + + %fieldset %legend @@ -44,9 +55,10 @@ #proxy_container = string_field %w(proxy http_host) - = string_field %w(proxy http_port) + = integer_field %w(proxy http_port) = string_field %w(proxy https_host) - = string_field %w(proxy https_port) + = integer_field %w(proxy https_port) + = string_field %w(proxy no_proxy) %fieldset %legend diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 8603f83fd4..21499cb91b 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -29,9 +29,12 @@ en: from: 'Oscm Email Address' docker_settings: 'Docker Registry' docker: + auth: 'Authentication Required' use_registry: 'Install from Local Registry' host: 'Registry Host' port: 'Registry Port' + user: 'User' + password: 'Password' proxy_settings: 'Proxy Settings' proxy: use_proxy: 'Use Proxy' @@ -39,6 +42,7 @@ en: http_port: 'HTTP Port' https_host: 'HTTPS Host' https_port: 'HTTPS Port' + no_proxy: 'No Proxy' api: protocol: 'Protocol' ssl_header: 'SSL Support' From 7e8313052d17eb66a3cac8086d7e512b83c94c2c Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 23 Mar 2017 16:44:12 +0100 Subject: [PATCH 011/181] Correct the indentation --- .../barclamp/oscm/_edit_attributes.html.haml | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 50ff5cdd45..ccea81e695 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -17,48 +17,45 @@ "data-showit-direct" => "true" #mail_container - = string_field %w(mail user) - = password_field %w(mail password) + = string_field %w(mail user) + = password_field %w(mail password) %fieldset %legend = t(".docker_settings") = boolean_field %w(docker use_registry), - "data-showit" => "false", + "data-showit" => "true", "data-showit-target" => "#docker_container", "data-showit-direct" => "true" #docker_container - = string_field %w(docker host) - = integer_field %w(docker port) - = boolean_field %w(docker auth) - "data-showit" => "false", - "data-showit-target" => "#dockerauth_container", - "data-showit-direct" => "true" - - #dockerauth_container - = string_field %w(docker user) - = password_field %w(docker password) - - + = string_field %w(docker host) + = integer_field %w(docker port) + = boolean_field %w(docker auth), + "data-showit" => "true", + "data-showit-target" => "#dockerauth_container", + "data-showit-direct" => "true" + #dockerauth_container + = string_field %w(docker user) + = password_field %w(docker password) %fieldset %legend = t(".proxy_settings") = boolean_field %w(proxy use_proxy), - "data-showit" => "false", + "data-showit" => "true", "data-showit-target" => "#proxy_container", "data-showit-direct" => "true" #proxy_container - = string_field %w(proxy http_host) - = integer_field %w(proxy http_port) - = string_field %w(proxy https_host) - = integer_field %w(proxy https_port) - = string_field %w(proxy no_proxy) + = string_field %w(proxy http_host) + = integer_field %w(proxy http_port) + = string_field %w(proxy https_host) + = integer_field %w(proxy https_port) + = string_field %w(proxy no_proxy) %fieldset %legend @@ -76,4 +73,4 @@ = string_field %w(ssl keyfile) = boolean_field %w(ssl insecure) = string_field %w(ssl ca_certs) - \ No newline at end of file + From 3547a3ff96bb7269d5bfc460e9e9b34ae240e9d4 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 27 Mar 2017 09:19:44 +0200 Subject: [PATCH 012/181] Add proxy authentication on UI --- chef/data_bags/crowbar/template-oscm.json | 5 ++++- chef/data_bags/crowbar/template-oscm.schema | 5 ++++- .../app/views/barclamp/oscm/_edit_attributes.html.haml | 9 +++++++++ crowbar_framework/config/locales/oscm/en.yml | 4 ++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 1775b6149a..df11b074b1 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -35,7 +35,10 @@ "http_port": 0, "https_host": "", "https_port": 0, - "no_proxy": "localhost,127.0.0.1" + "no_proxy": "localhost,127.0.0.1", + "auth": false, + "user": "", + "password": "" }, "ssl": { "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 38df12e353..a1b7d60f7d 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -59,7 +59,10 @@ "http_port": { "type": "int", "required": true }, "https_host": { "type": "str", "required": true }, "https_port": { "type": "int", "required": true }, - "no_proxy": { "type": "str", "required": true } + "no_proxy": { "type": "str", "required": true }, + "auth": { "type": "bool", "required": true }, + "user": { "type": "str", "required": true }, + "password": { "type": "str", "required": true } } }, "ssl": { diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index ccea81e695..cd66755cd3 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -55,7 +55,16 @@ = integer_field %w(proxy http_port) = string_field %w(proxy https_host) = integer_field %w(proxy https_port) + %span.help-block + = t('.proxy_hint') = string_field %w(proxy no_proxy) + = boolean_field %w(proxy auth), + "data-showit" => "true", + "data-showit-target" => "#proxyauth_container", + "data-showit-direct" => "true" + #proxyauth_container + = string_field %w(proxy user) + = password_field %w(proxy password) %fieldset %legend diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 21499cb91b..e4d2f8e0ce 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -37,12 +37,16 @@ en: password: 'Password' proxy_settings: 'Proxy Settings' proxy: + auth: 'Authentication Required' use_proxy: 'Use Proxy' http_host: 'HTTP Host' http_port: 'HTTP Port' https_host: 'HTTPS Host' https_port: 'HTTPS Port' no_proxy: 'No Proxy' + user: 'User' + password: 'Password' + proxy_hint: 'A comma-separated list of hosts which can bypass the proxy' api: protocol: 'Protocol' ssl_header: 'SSL Support' From 6221b602e8a585677baf4615da0abf3d38789710 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 27 Mar 2017 10:51:03 +0200 Subject: [PATCH 013/181] Change order --- oscm.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/oscm.yml b/oscm.yml index 8f3431c997..b5fee34231 100644 --- a/oscm.yml +++ b/oscm.yml @@ -25,6 +25,8 @@ barclamp: - 'heat' crowbar: - run_order: 100 - chef_order: 100 + layout: 1 + run_order: 135 + chef_order: 135 + proposal_schema_version: 1 From 5a8ffd847da6dd6c206c5f6ca3d7d301db38d10b Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 27 Mar 2017 10:54:46 +0200 Subject: [PATCH 014/181] Add order --- oscm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/oscm.yml b/oscm.yml index b5fee34231..19baba9eb4 100644 --- a/oscm.yml +++ b/oscm.yml @@ -26,6 +26,7 @@ barclamp: crowbar: layout: 1 + order: 135 run_order: 135 chef_order: 135 proposal_schema_version: 1 From 2560b6755b4e755c7097cbb0d69f0089ad8314c5 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 28 Mar 2017 08:55:22 +0200 Subject: [PATCH 015/181] Create oscm tenant --- chef/cookbooks/oscm/recipes/main.rb | 0 .../oscm/recipes/role_oscm_server.rb | 2 +- chef/cookbooks/oscm/recipes/server.rb | 44 +++++++++++++++++++ chef/data_bags/crowbar/template-oscm.json | 18 +++++--- chef/data_bags/crowbar/template-oscm.schema | 10 +++++ crowbar_framework/app/models/oscm_service.rb | 1 + crowbar_framework/config/locales/oscm/en.yml | 1 + 7 files changed, 69 insertions(+), 7 deletions(-) delete mode 100644 chef/cookbooks/oscm/recipes/main.rb create mode 100644 chef/cookbooks/oscm/recipes/server.rb diff --git a/chef/cookbooks/oscm/recipes/main.rb b/chef/cookbooks/oscm/recipes/main.rb deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/chef/cookbooks/oscm/recipes/role_oscm_server.rb b/chef/cookbooks/oscm/recipes/role_oscm_server.rb index 6894c3632b..dc5978f152 100644 --- a/chef/cookbooks/oscm/recipes/role_oscm_server.rb +++ b/chef/cookbooks/oscm/recipes/role_oscm_server.rb @@ -1,3 +1,3 @@ if CrowbarRoleRecipe.node_state_valid_for_role?(node, "oscm", "oscm-server") - include_recipe "oscm::main" + include_recipe "#{@cookbook_name}::server" end diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb new file mode 100644 index 0000000000..6a74ac78a6 --- /dev/null +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -0,0 +1,44 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +oscm_tenant = node[:oscm][:keystone][:tenant] + +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) + +register_auth_hash = { + user: keystone_settings["admin_user"], + password: keystone_settings["admin_password"], + tenant: keystone_settings["admin_tenant"] +} + +keystone_register "oscm server wakeup keystone" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + action :wakeup +end + +keystone_register "oscm create tenant oscm" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + tenant_name oscm_tenant + action: add_tenant +end diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index df11b074b1..518ce47eb9 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -2,13 +2,19 @@ "id": "template-oscm", "description": "OSCM", "attributes": { - "oscm" : { + "oscm": { "timeout": 300, - "debug" : false, + "debug": false, + "keystone_instance": "none", "heat_instance": "none", - "heat_template" : { - "image" : "sles12-docker", - "flavor" : "oscm.small", + "keystone": { + "tenant": "oscm", + "user": "oscm", + "password": "oscm" + }, + "heat_template": { + "image": "sles12-docker", + "flavor": "oscm.small", "keypair": "oscm", "user": "oscm", "project":"oscm" @@ -53,7 +59,7 @@ } }, "deployment": { - "oscm" : { + "oscm": { "crowbar-revision": 0, "crowbar-applied": false, "schema-revision": 100, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index a1b7d60f7d..cf6cedb76e 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -14,7 +14,17 @@ "mapping": { "timeout": { "type": "int", "required": true }, "debug": { "type": "bool", "required": true }, + "keystone_instance": { "type": "str", "required": true }, "heat_instance": { "type": "str", "required": true }, + "keystone" : { + "type" : "map", + "required" : true, + "mapping": { + "tenant": { "type": "str", "required": true }, + "user": { "type": "str", "required": true }, + "password": { "type": "str", "required": true } + } + }, "heat_template" : { "type" : "map", "required" : true, diff --git a/crowbar_framework/app/models/oscm_service.rb b/crowbar_framework/app/models/oscm_service.rb index e6d9afe474..95e45834db 100644 --- a/crowbar_framework/app/models/oscm_service.rb +++ b/crowbar_framework/app/models/oscm_service.rb @@ -63,6 +63,7 @@ def create_proposal "oscm-server" => [server_nodes.first.name] } unless server_nodes.nil? + base["attributes"][@bc_name]["keystone_instance"] = find_dep_proposal("keystone") base["attributes"][@bc_name]["heat_instance"] = find_dep_proposal("heat") @logger.debug("Oscm create_proposal: exiting") diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index e4d2f8e0ce..58e1f10eb9 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -18,6 +18,7 @@ en: barclamp: oscm: edit_attributes: + keystone_instance: 'Keystone' heat_instance: 'Heat' mail_settings: 'Mail Settings' mail: From 1b089a0da6d127364553e402dac2f976d5a27da4 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 28 Mar 2017 09:25:49 +0200 Subject: [PATCH 016/181] Create oscm user and assign roles --- chef/cookbooks/oscm/recipes/server.rb | 56 +++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 6a74ac78a6..c626139283 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -15,6 +15,8 @@ # oscm_tenant = node[:oscm][:keystone][:tenant] +oscm_user = node[:oscm][:keystone][:user] +oscm_password = node[:oscm][:keystone][:password] keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -24,7 +26,7 @@ tenant: keystone_settings["admin_tenant"] } -keystone_register "oscm server wakeup keystone" do +keystone_register "oscm wakeup keystone" do protocol keystone_settings["protocol"] insecure keystone_settings["insecure"] host keystone_settings["internal_url_host"] @@ -33,12 +35,60 @@ action :wakeup end -keystone_register "oscm create tenant oscm" do +keystone_register "oscm create tenant" do protocol keystone_settings["protocol"] insecure keystone_settings["insecure"] host keystone_settings["internal_url_host"] port keystone_settings["admin_port"] auth register_auth_hash tenant_name oscm_tenant - action: add_tenant + action :add_tenant +end + +keystone_register "oscm register user" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name oscm_user + user_password oscm_password + tenant_name oscm_tenant + action :add_user +end + +keystone_register "oscm give user admin role" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name oscm_user + tenant_name oscm_tenant + role_name "admin" + action :add_access +end + +keystone_register "oscm give user member role" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name oscm_user + tenant_name oscm_tenant + role_name "Member" + action :add_access +end + +keystone_register "oscm give user _member_ role" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name oscm_user + tenant_name oscm_tenant + role_name "_member_" + action :add_access end From 0b4fadcecddfb041e48da13dec542863096066d6 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 30 Mar 2017 16:56:05 +0200 Subject: [PATCH 017/181] Add flavor --- chef/cookbooks/oscm/recipes/server.rb | 19 +++++++++++++++++++ chef/data_bags/crowbar/template-oscm.json | 9 +++++---- chef/data_bags/crowbar/template-oscm.schema | 11 ++++++----- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index c626139283..b3d041e234 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -17,6 +17,10 @@ oscm_tenant = node[:oscm][:keystone][:tenant] oscm_user = node[:oscm][:keystone][:user] oscm_password = node[:oscm][:keystone][:password] +oscm_flavor_name = node[:oscm][:openstack][:flavor_name] +oscm_flavor_ram = node[:oscm][:openstack][:flavor_ram] +oscm_flavor_vcpus = node[:oscm][:openstack][:flavor_vcpus] +oscm_flavor_disk = node[:oscm][:openstack][:flavor_disk] keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -92,3 +96,18 @@ role_name "_member_" action :add_access end + +bash "add flavor" do + code <<-EOH + nova flavor-create #{oscm_flavor_name} auto #{oscm_flavor_ram} #{oscm_flavor_disk} #{oscm_flavor_vcpus} --is-public false &> /dev/null || exit 0 +EOH + environment ({ + "OS_USERNAME" => oscm_user, + "OS_PASSWORD" => oscm_password, + "OS_TENANT_NAME" => oscm_tenant, + "OS_AUTH_URL" => keystone_settings["internal_auth_url"], + "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], + "OS_USER_DOMAIN_NAME" => "Default", + "OS_PROJECT_DOMAIN_NAME" => "Default" + }) +end \ No newline at end of file diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 518ce47eb9..e5448e520f 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -12,12 +12,13 @@ "user": "oscm", "password": "oscm" }, - "heat_template": { + "openstack": { "image": "sles12-docker", - "flavor": "oscm.small", + "flavor_name": "oscm.small", + "flavor_ram": 1024, + "flavor_vcpus": 1, + "flavor_disk": 1, "keypair": "oscm", - "user": "oscm", - "project":"oscm" }, "mail": { "auth": true, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index cf6cedb76e..998871e4e1 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -25,15 +25,16 @@ "password": { "type": "str", "required": true } } }, - "heat_template" : { + "openstack" : { "type" : "map", "required" : true, "mapping": { "image": { "type": "str", "required": true }, - "flavor": { "type": "str", "required": true }, - "keypair": { "type": "str", "required": true }, - "user": { "type": "str", "required": true }, - "project": { "type": "str", "required": true } + "flavor_name": { "type": "str", "required": true }, + "flavor_ram": { "type": "int", "required": true }, + "flavor_vcpus": { "type": "int", "required": true }, + "flavor_disk": { "type": "int", "required": true }, + "keypair": { "type": "str", "required": true } } }, "mail" : { From 3c251e981893bf49b0b9cf3a97f1b6c979327c97 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 30 Mar 2017 17:08:42 +0200 Subject: [PATCH 018/181] Fix syntax error --- chef/data_bags/crowbar/template-oscm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index e5448e520f..ef249a445e 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -18,7 +18,7 @@ "flavor_ram": 1024, "flavor_vcpus": 1, "flavor_disk": 1, - "keypair": "oscm", + "keypair": "oscm" }, "mail": { "auth": true, From 0496f3455d935c922369afcd8ef89922734835bc Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 3 Apr 2017 09:39:29 +0200 Subject: [PATCH 019/181] Generate certs --- chef/cookbooks/oscm/attributes/default.rb | 21 +++++++++++++++++++++ chef/cookbooks/oscm/recipes/server.rb | 14 +++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 chef/cookbooks/oscm/attributes/default.rb diff --git a/chef/cookbooks/oscm/attributes/default.rb b/chef/cookbooks/oscm/attributes/default.rb new file mode 100644 index 0000000000..b7dc13629c --- /dev/null +++ b/chef/cookbooks/oscm/attributes/default.rb @@ -0,0 +1,21 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +default[:oscm][:ssl][:certfile] = "/etc/oscm/ssl/certs/signing_cert.pem" +default[:oscm][:ssl][:keyfile] = "/etc/oscm/ssl/private/signing_key.pem" +default[:oscm][:ssl][:generate_certs] = false +default[:oscm][:ssl][:insecure] = false +default[:oscm][:ssl][:ca_certs] = "/etc/oscm/ssl/certs/ca.pem" diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index b3d041e234..f2f8b1bbc5 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -21,6 +21,7 @@ oscm_flavor_ram = node[:oscm][:openstack][:flavor_ram] oscm_flavor_vcpus = node[:oscm][:openstack][:flavor_vcpus] oscm_flavor_disk = node[:oscm][:openstack][:flavor_disk] +oscm_group = "" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -110,4 +111,15 @@ "OS_USER_DOMAIN_NAME" => "Default", "OS_PROJECT_DOMAIN_NAME" => "Default" }) -end \ No newline at end of file +end + +if node[:oscm][:api][:protocol] == "https" + ssl_setup "setting up ssl for oscm" do + generate_certs node[:oscm][:ssl][:generate_certs] + certfile node[:oscm][:ssl][:certfile] + keyfile node[:oscm][:ssl][:keyfile] + group oscm_group + fqdn node[:fqdn] + ca_certs node[:oscm][:ssl][:ca_certs] + end +end From 612589aedc4da304935c8e28d009acf6faae92f5 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 3 Apr 2017 16:54:53 +0200 Subject: [PATCH 020/181] UI, add public key for the openstack keypair --- chef/cookbooks/oscm/recipes/server.rb | 2 +- chef/data_bags/crowbar/template-oscm.json | 3 ++- chef/data_bags/crowbar/template-oscm.schema | 3 ++- .../app/views/barclamp/oscm/_edit_attributes.html.haml | 8 ++++++++ crowbar_framework/config/locales/oscm/en.yml | 4 ++++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index f2f8b1bbc5..708ee62cff 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -21,7 +21,7 @@ oscm_flavor_ram = node[:oscm][:openstack][:flavor_ram] oscm_flavor_vcpus = node[:oscm][:openstack][:flavor_vcpus] oscm_flavor_disk = node[:oscm][:openstack][:flavor_disk] -oscm_group = "" +oscm_group = "root" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index ef249a445e..6fd5ee68dd 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -18,7 +18,8 @@ "flavor_ram": 1024, "flavor_vcpus": 1, "flavor_disk": 1, - "keypair": "oscm" + "keypair": "oscm", + "keypair_public_key": "" }, "mail": { "auth": true, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 998871e4e1..64d71a9811 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -34,7 +34,8 @@ "flavor_ram": { "type": "int", "required": true }, "flavor_vcpus": { "type": "int", "required": true }, "flavor_disk": { "type": "int", "required": true }, - "keypair": { "type": "str", "required": true } + "keypair": { "type": "str", "required": true }, + "keypair_public_key": { "type": "str", "required": true } } }, "mail" : { diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index cd66755cd3..71c17d7dc4 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -66,6 +66,14 @@ = string_field %w(proxy user) = password_field %w(proxy password) + %fieldset + %legend + = t(".openstack_settings") + + %span.help-block + = t('.keypair_hint') + = text_field %w(openstack keypair_public_key) + %fieldset %legend = t(".ssl_header") diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 58e1f10eb9..8579eca971 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -48,6 +48,10 @@ en: user: 'User' password: 'Password' proxy_hint: 'A comma-separated list of hosts which can bypass the proxy' + openstack_settings: "OpenStack Settings" + openstack: + keypair_public_key: "Public Key" + keypair_hint: 'SSH public key for the openstack instance where the OSCM will be deployed' api: protocol: 'Protocol' ssl_header: 'SSL Support' From 3cd01115fdbe9d78ca568c39c4e3d60c50a3e30d Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 3 Apr 2017 17:09:27 +0200 Subject: [PATCH 021/181] Add nova keypair --- chef/cookbooks/oscm/recipes/server.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 708ee62cff..17d79b6cc5 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -21,6 +21,8 @@ oscm_flavor_ram = node[:oscm][:openstack][:flavor_ram] oscm_flavor_vcpus = node[:oscm][:openstack][:flavor_vcpus] oscm_flavor_disk = node[:oscm][:openstack][:flavor_disk] +oscm_keypair_name = node[:oscm][:openstack][:keypair] +oscm_keypair_public_key = node[:oscm][:openstack][:keypair_public_key] oscm_group = "root" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -113,6 +115,21 @@ }) end +bash "add keypair" do + code <<-EOH + nova keypair-add #{oscm_keypair_name} --pub-key #{oscm_keypair_public_key} &> /dev/null || exit 0 +EOH + environment ({ + "OS_USERNAME" => oscm_user, + "OS_PASSWORD" => oscm_password, + "OS_TENANT_NAME" => oscm_tenant, + "OS_AUTH_URL" => keystone_settings["internal_auth_url"], + "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], + "OS_USER_DOMAIN_NAME" => "Default", + "OS_PROJECT_DOMAIN_NAME" => "Default" + }) +end + if node[:oscm][:api][:protocol] == "https" ssl_setup "setting up ssl for oscm" do generate_certs node[:oscm][:ssl][:generate_certs] From 8a0f4adb6c3f2600e57155056ce0b98d36a30824 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 4 Apr 2017 09:19:23 +0200 Subject: [PATCH 022/181] Create oscm keypair with imported public key --- chef/cookbooks/oscm/recipes/server.rb | 7 +++++-- chef/data_bags/crowbar/template-oscm.json | 3 ++- chef/data_bags/crowbar/template-oscm.schema | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 17d79b6cc5..6da257efdb 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -22,7 +22,8 @@ oscm_flavor_vcpus = node[:oscm][:openstack][:flavor_vcpus] oscm_flavor_disk = node[:oscm][:openstack][:flavor_disk] oscm_keypair_name = node[:oscm][:openstack][:keypair] -oscm_keypair_public_key = node[:oscm][:openstack][:keypair_public_key] +oscm_keypair_publickey = node[:oscm][:openstack][:keypair_publickey] +oscm_keypair_publickeyfile = node[:oscm][:openstack][:keypair_publickeyfile] oscm_group = "root" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -117,7 +118,9 @@ bash "add keypair" do code <<-EOH - nova keypair-add #{oscm_keypair_name} --pub-key #{oscm_keypair_public_key} &> /dev/null || exit 0 + publickey = #{oscm_keypair_publickey} + [[ ! -z "${publickey// }" ]] && mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null; echo "&{publickey}" > "#{oscm_keypair_publickeyfile}" + nova keypair-add #{oscm_keypair_name} --pub-key #{oscm_keypair_publickeyfile} &> /dev/null || exit 0 EOH environment ({ "OS_USERNAME" => oscm_user, diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 6fd5ee68dd..57b5aa99bc 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -19,7 +19,8 @@ "flavor_vcpus": 1, "flavor_disk": 1, "keypair": "oscm", - "keypair_public_key": "" + "keypair_publickey": "", + "keypair_publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem" }, "mail": { "auth": true, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 64d71a9811..a4673ae8ec 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -35,7 +35,8 @@ "flavor_vcpus": { "type": "int", "required": true }, "flavor_disk": { "type": "int", "required": true }, "keypair": { "type": "str", "required": true }, - "keypair_public_key": { "type": "str", "required": true } + "keypair_publickey": { "type": "str", "required": true }, + "keypair_publickeyfile": { "type": "str", "required": true } } }, "mail" : { From bfa51e3032dfceb6de1dca78e5b5296226114c70 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 4 Apr 2017 09:24:22 +0200 Subject: [PATCH 023/181] Add keypair on UI --- .../app/views/barclamp/oscm/_edit_attributes.html.haml | 2 +- crowbar_framework/config/locales/oscm/en.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 71c17d7dc4..1d7959a8ed 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -72,7 +72,7 @@ %span.help-block = t('.keypair_hint') - = text_field %w(openstack keypair_public_key) + = text_field %w(openstack keypair_publickey) %fieldset %legend diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 8579eca971..4b3395d7ba 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -50,7 +50,7 @@ en: proxy_hint: 'A comma-separated list of hosts which can bypass the proxy' openstack_settings: "OpenStack Settings" openstack: - keypair_public_key: "Public Key" + keypair_publickey: "Public Key" keypair_hint: 'SSH public key for the openstack instance where the OSCM will be deployed' api: protocol: 'Protocol' From c4a81af2e43f52faaef26ebe3a5273889dda0996 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 4 Apr 2017 09:37:10 +0200 Subject: [PATCH 024/181] bash syntax errors --- chef/cookbooks/oscm/recipes/server.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 6da257efdb..4d4a92e3dc 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -118,8 +118,8 @@ bash "add keypair" do code <<-EOH - publickey = #{oscm_keypair_publickey} - [[ ! -z "${publickey// }" ]] && mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null; echo "&{publickey}" > "#{oscm_keypair_publickeyfile}" + publickey="#{oscm_keypair_publickey}" + [[ ! -z "${publickey// }" ]] && mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null; echo "${publickey}" > "#{oscm_keypair_publickeyfile}" nova keypair-add #{oscm_keypair_name} --pub-key #{oscm_keypair_publickeyfile} &> /dev/null || exit 0 EOH environment ({ From e8122c4e0b43c05176fcaf33a0764e32113b4ff1 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 6 Apr 2017 12:17:30 +0200 Subject: [PATCH 025/181] Create oscm keypair for non-empty public key --- chef/cookbooks/oscm/recipes/server.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 4d4a92e3dc..3832deeebc 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -101,7 +101,7 @@ action :add_access end -bash "add flavor" do +bash "add oscm flavor" do code <<-EOH nova flavor-create #{oscm_flavor_name} auto #{oscm_flavor_ram} #{oscm_flavor_disk} #{oscm_flavor_vcpus} --is-public false &> /dev/null || exit 0 EOH @@ -116,11 +116,15 @@ }) end -bash "add keypair" do +bash "add oscm keypair" do code <<-EOH publickey="#{oscm_keypair_publickey}" - [[ ! -z "${publickey// }" ]] && mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null; echo "${publickey}" > "#{oscm_keypair_publickeyfile}" - nova keypair-add #{oscm_keypair_name} --pub-key #{oscm_keypair_publickeyfile} &> /dev/null || exit 0 + if ! -z "${publickey// }" + then + mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null + echo "${publickey}" > "#{oscm_keypair_publickeyfile}" + nova keypair-add #{oscm_keypair_name} --pub-key #{oscm_keypair_publickeyfile} &> /dev/null || exit 0 + fi EOH environment ({ "OS_USERNAME" => oscm_user, From 4500644e27468b3a06e3e6d60fdc6e82dcdecc81 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 6 Apr 2017 13:14:20 +0200 Subject: [PATCH 026/181] Add heat template for volumes stack --- chef/cookbooks/oscm/files/volumes.yaml | 47 +++++++++++++++++++++ chef/data_bags/crowbar/template-oscm.json | 4 +- chef/data_bags/crowbar/template-oscm.schema | 4 +- 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 chef/cookbooks/oscm/files/volumes.yaml diff --git a/chef/cookbooks/oscm/files/volumes.yaml b/chef/cookbooks/oscm/files/volumes.yaml new file mode 100644 index 0000000000..189e597b9f --- /dev/null +++ b/chef/cookbooks/oscm/files/volumes.yaml @@ -0,0 +1,47 @@ +# Example invocation: heat stack-create --poll -f volumes.yaml oscm +heat_template_version: 2015-10-15 + + +parameters: + app_size: + type: number + default: 1 + description: Size of the application volume in gigabytes + db_size: + type: number + default: 1 + description: Size of the database volume in gigabytes + + +resources: + + app_volume: + type: OS::Cinder::Volume + properties: + name: + list_join: + - "_" + - - { get_param: 'OS::stack_name' } + - app_volume + size: { get_param: app_size } + + db_volume: + type: OS::Cinder::Volume + properties: + name: + list_join: + - "_" + - - { get_param: 'OS::stack_name' } + - db_volume + size: { get_param: db_size } + +outputs: + # Retrieval after stack creation (presuming stack is named `oscm`): + # heat output-show oscm app_volume_id + app_volume_id: + value: { get_resource: app_volume } + + # Retrieval after stack creation (presuming stack is named `oscm`): + # heat output-show oscm db_volume_id + db_volume_id: + value: { get_resource: db_volume } diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 57b5aa99bc..af11e2d701 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -20,7 +20,9 @@ "flavor_disk": 1, "keypair": "oscm", "keypair_publickey": "", - "keypair_publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem" + "keypair_publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem", + "db_volume_size": 1, + "app_volume_size": 1 }, "mail": { "auth": true, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index a4673ae8ec..73d85b268b 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -36,7 +36,9 @@ "flavor_disk": { "type": "int", "required": true }, "keypair": { "type": "str", "required": true }, "keypair_publickey": { "type": "str", "required": true }, - "keypair_publickeyfile": { "type": "str", "required": true } + "keypair_publickeyfile": { "type": "str", "required": true }, + "db_volume_size": { "type": "int", "required": true }, + "app_volume_size": { "type": "int", "required": true } } }, "mail" : { From f25b6986f48f9c52c0f6812be6e8cd632a745973 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 6 Apr 2017 13:44:55 +0200 Subject: [PATCH 027/181] Add cookbook file for volumes stack heat template --- chef/cookbooks/oscm/recipes/server.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 3832deeebc..bdb0ad7d03 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -147,3 +147,11 @@ ca_certs node[:oscm][:ssl][:ca_certs] end end + +cookbook_file '/etc/oscm/heat/volumes.yaml' do + source 'volumes.yaml' + owner 'root' + group 'root' + mode '0755' + action :create +end \ No newline at end of file From 4966365782b6fce144865d6744c4a3c54df2b8dc Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 6 Apr 2017 17:09:58 +0200 Subject: [PATCH 028/181] Add dir for the heat templates --- chef/cookbooks/oscm/recipes/server.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index bdb0ad7d03..73751343bb 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -148,6 +148,13 @@ end end +%w[ /etc /etc/oscm /etc/oscm/heat ].each do |path| + directory path do + owner 'root' + group 'root' + mode '0755' + end + cookbook_file '/etc/oscm/heat/volumes.yaml' do source 'volumes.yaml' owner 'root' From 98e747b431a6d7570782b805bf95d02b39d92bad Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 6 Apr 2017 17:14:46 +0200 Subject: [PATCH 029/181] Syntax error --- chef/cookbooks/oscm/recipes/server.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 73751343bb..32dc506397 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -154,6 +154,7 @@ group 'root' mode '0755' end + end cookbook_file '/etc/oscm/heat/volumes.yaml' do source 'volumes.yaml' From acbe92b365446980b9bb57810bc2c6e11f6b39a8 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 6 Apr 2017 17:22:36 +0200 Subject: [PATCH 030/181] Add file to default dir --- .../cookbooks/oscm/files/default/volumes.yaml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 chef/cookbooks/oscm/files/default/volumes.yaml diff --git a/chef/cookbooks/oscm/files/default/volumes.yaml b/chef/cookbooks/oscm/files/default/volumes.yaml new file mode 100644 index 0000000000..189e597b9f --- /dev/null +++ b/chef/cookbooks/oscm/files/default/volumes.yaml @@ -0,0 +1,47 @@ +# Example invocation: heat stack-create --poll -f volumes.yaml oscm +heat_template_version: 2015-10-15 + + +parameters: + app_size: + type: number + default: 1 + description: Size of the application volume in gigabytes + db_size: + type: number + default: 1 + description: Size of the database volume in gigabytes + + +resources: + + app_volume: + type: OS::Cinder::Volume + properties: + name: + list_join: + - "_" + - - { get_param: 'OS::stack_name' } + - app_volume + size: { get_param: app_size } + + db_volume: + type: OS::Cinder::Volume + properties: + name: + list_join: + - "_" + - - { get_param: 'OS::stack_name' } + - db_volume + size: { get_param: db_size } + +outputs: + # Retrieval after stack creation (presuming stack is named `oscm`): + # heat output-show oscm app_volume_id + app_volume_id: + value: { get_resource: app_volume } + + # Retrieval after stack creation (presuming stack is named `oscm`): + # heat output-show oscm db_volume_id + db_volume_id: + value: { get_resource: db_volume } From e8d36d5c7cff27f4507abd91ea23e525b4adabc6 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 6 Apr 2017 17:22:51 +0200 Subject: [PATCH 031/181] Remove file from file dir --- chef/cookbooks/oscm/files/volumes.yaml | 47 -------------------------- 1 file changed, 47 deletions(-) delete mode 100644 chef/cookbooks/oscm/files/volumes.yaml diff --git a/chef/cookbooks/oscm/files/volumes.yaml b/chef/cookbooks/oscm/files/volumes.yaml deleted file mode 100644 index 189e597b9f..0000000000 --- a/chef/cookbooks/oscm/files/volumes.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Example invocation: heat stack-create --poll -f volumes.yaml oscm -heat_template_version: 2015-10-15 - - -parameters: - app_size: - type: number - default: 1 - description: Size of the application volume in gigabytes - db_size: - type: number - default: 1 - description: Size of the database volume in gigabytes - - -resources: - - app_volume: - type: OS::Cinder::Volume - properties: - name: - list_join: - - "_" - - - { get_param: 'OS::stack_name' } - - app_volume - size: { get_param: app_size } - - db_volume: - type: OS::Cinder::Volume - properties: - name: - list_join: - - "_" - - - { get_param: 'OS::stack_name' } - - db_volume - size: { get_param: db_size } - -outputs: - # Retrieval after stack creation (presuming stack is named `oscm`): - # heat output-show oscm app_volume_id - app_volume_id: - value: { get_resource: app_volume } - - # Retrieval after stack creation (presuming stack is named `oscm`): - # heat output-show oscm db_volume_id - db_volume_id: - value: { get_resource: db_volume } From e1a8227cdc8fc7b3775ffe699256d4ea712c20f5 Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 7 Apr 2017 15:42:31 +0200 Subject: [PATCH 032/181] Add create oscm volume stack --- chef/cookbooks/oscm/recipes/server.rb | 62 +++++++++++++-------- chef/data_bags/crowbar/template-oscm.json | 8 ++- chef/data_bags/crowbar/template-oscm.schema | 12 +++- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 32dc506397..3953c81805 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -24,6 +24,10 @@ oscm_keypair_name = node[:oscm][:openstack][:keypair] oscm_keypair_publickey = node[:oscm][:openstack][:keypair_publickey] oscm_keypair_publickeyfile = node[:oscm][:openstack][:keypair_publickeyfile] +oscm_heattemplate_path = node[:oscm][:openstack][:heattemplate_path] +oscm_volumestack_name = node[:oscm][:openstack][:volume_stack][:stack_name] +oscm_db_volume_size = node[:oscm][:openstack][:volume_stack][:db_volume_size] +oscm_app_volume_size = node[:oscm][:openstack][:volume_stack][:app_volume_size] oscm_group = "root" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -101,6 +105,17 @@ action :add_access end +if node[:oscm][:api][:protocol] == "https" + ssl_setup "setting up ssl for oscm" do + generate_certs node[:oscm][:ssl][:generate_certs] + certfile node[:oscm][:ssl][:certfile] + keyfile node[:oscm][:ssl][:keyfile] + group oscm_group + fqdn node[:fqdn] + ca_certs node[:oscm][:ssl][:ca_certs] + end +end + bash "add oscm flavor" do code <<-EOH nova flavor-create #{oscm_flavor_name} auto #{oscm_flavor_ram} #{oscm_flavor_disk} #{oscm_flavor_vcpus} --is-public false &> /dev/null || exit 0 @@ -137,29 +152,32 @@ }) end -if node[:oscm][:api][:protocol] == "https" - ssl_setup "setting up ssl for oscm" do - generate_certs node[:oscm][:ssl][:generate_certs] - certfile node[:oscm][:ssl][:certfile] - keyfile node[:oscm][:ssl][:keyfile] +directory "#{oscm_heattemplate_path}" do + owner oscm_group group oscm_group - fqdn node[:fqdn] - ca_certs node[:oscm][:ssl][:ca_certs] - end -end - -%w[ /etc /etc/oscm /etc/oscm/heat ].each do |path| - directory path do - owner 'root' - group 'root' - mode '0755' - end + mode 0755 end -cookbook_file '/etc/oscm/heat/volumes.yaml' do - source 'volumes.yaml' - owner 'root' - group 'root' - mode '0755' +cookbook_file "#{oscm_heattemplate_path}/volumes.yaml" do + source "volumes.yaml" + owner oscm_group + group oscm_group + mode 0755 action :create -end \ No newline at end of file +end + +bash "create oscm volumes stack with name #{oscm_volumestack_name}" do + code <<-EOH + openstack stack create --parameter db_volume_size=#{oscm_db_volume_size} --parameter app_volume_size=#{oscm_app_volume_size} -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || exit 0 +EOH + environment ({ + "OS_USERNAME" => oscm_user, + "OS_PASSWORD" => oscm_password, + "OS_TENANT_NAME" => oscm_tenant, + "OS_AUTH_URL" => keystone_settings["internal_auth_url"], + "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], + "OS_USER_DOMAIN_NAME" => "Default", + "OS_PROJECT_DOMAIN_NAME" => "Default" + }) +end + diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index af11e2d701..5da2eabb57 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -21,8 +21,12 @@ "keypair": "oscm", "keypair_publickey": "", "keypair_publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem", - "db_volume_size": 1, - "app_volume_size": 1 + "heattemplate_path": "/etc/oscm/heat/", + "volume_stack" : { + "stack_name": "oscm", + "db_volume_size": 1, + "app_volume_size": 1 + } }, "mail": { "auth": true, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 73d85b268b..edc92aefa2 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -37,8 +37,16 @@ "keypair": { "type": "str", "required": true }, "keypair_publickey": { "type": "str", "required": true }, "keypair_publickeyfile": { "type": "str", "required": true }, - "db_volume_size": { "type": "int", "required": true }, - "app_volume_size": { "type": "int", "required": true } + "heattemplate_path": { "type": "str", "required": true }, + "volume_stack" : { + "type" : "map", + "required" : true, + "mapping": { + "stack_name": { "type": "str", "required": true }, + "db_volume_size": { "type": "int", "required": true }, + "app_volume_size": { "type": "int", "required": true } + } + } } }, "mail" : { From a23367b36e1697e6632eabfd1ddafa4314db58a7 Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 7 Apr 2017 16:09:49 +0200 Subject: [PATCH 033/181] Adapt stack parameters --- chef/cookbooks/oscm/recipes/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 3953c81805..d84a3aa07c 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -168,7 +168,7 @@ bash "create oscm volumes stack with name #{oscm_volumestack_name}" do code <<-EOH - openstack stack create --parameter db_volume_size=#{oscm_db_volume_size} --parameter app_volume_size=#{oscm_app_volume_size} -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || exit 0 + openstack stack create --parameter "db_volume_size=#{oscm_db_volume_size}" --parameter "app_volume_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || exit 0 EOH environment ({ "OS_USERNAME" => oscm_user, From aa43c7465f1ea26b0d2d99911bc4496456f0aa87 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 10 Apr 2017 08:20:04 +0200 Subject: [PATCH 034/181] Refactor the oscm openstack json --- chef/cookbooks/oscm/recipes/server.rb | 16 ++++++------- chef/data_bags/crowbar/template-oscm.json | 18 ++++++++------ chef/data_bags/crowbar/template-oscm.schema | 26 +++++++++++++++------ 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index d84a3aa07c..ddf1c77c17 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -17,13 +17,13 @@ oscm_tenant = node[:oscm][:keystone][:tenant] oscm_user = node[:oscm][:keystone][:user] oscm_password = node[:oscm][:keystone][:password] -oscm_flavor_name = node[:oscm][:openstack][:flavor_name] -oscm_flavor_ram = node[:oscm][:openstack][:flavor_ram] -oscm_flavor_vcpus = node[:oscm][:openstack][:flavor_vcpus] -oscm_flavor_disk = node[:oscm][:openstack][:flavor_disk] -oscm_keypair_name = node[:oscm][:openstack][:keypair] -oscm_keypair_publickey = node[:oscm][:openstack][:keypair_publickey] -oscm_keypair_publickeyfile = node[:oscm][:openstack][:keypair_publickeyfile] +oscm_flavor_name = node[:oscm][:openstack][:flavor][:name] +oscm_flavor_ram = node[:oscm][:openstack][:flavor][:ram] +oscm_flavor_vcpus = node[:oscm][:openstack][:flavor][:vcpus] +oscm_flavor_disk = node[:oscm][:openstack][:flavor][:disk] +oscm_keypair_name = node[:oscm][:openstack][:keypair][:name] +oscm_keypair_publickey = node[:oscm][:openstack][:keypair][:publickey] +oscm_keypair_publickeyfile = node[:oscm][:openstack][:keypair][:publickeyfile] oscm_heattemplate_path = node[:oscm][:openstack][:heattemplate_path] oscm_volumestack_name = node[:oscm][:openstack][:volume_stack][:stack_name] oscm_db_volume_size = node[:oscm][:openstack][:volume_stack][:db_volume_size] @@ -168,7 +168,7 @@ bash "create oscm volumes stack with name #{oscm_volumestack_name}" do code <<-EOH - openstack stack create --parameter "db_volume_size=#{oscm_db_volume_size}" --parameter "app_volume_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || exit 0 + openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || exit 0 EOH environment ({ "OS_USERNAME" => oscm_user, diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 5da2eabb57..44d9896159 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -14,13 +14,17 @@ }, "openstack": { "image": "sles12-docker", - "flavor_name": "oscm.small", - "flavor_ram": 1024, - "flavor_vcpus": 1, - "flavor_disk": 1, - "keypair": "oscm", - "keypair_publickey": "", - "keypair_publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem", + "flavor": { + "name": "oscm.small", + "ram": 1024, + "vcpus": 1, + "disk": 1 + }, + "keypair": { + "name": "oscm", + "publickey": "", + "publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem" + }, "heattemplate_path": "/etc/oscm/heat/", "volume_stack" : { "stack_name": "oscm", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index edc92aefa2..0edce1fe0c 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -30,13 +30,25 @@ "required" : true, "mapping": { "image": { "type": "str", "required": true }, - "flavor_name": { "type": "str", "required": true }, - "flavor_ram": { "type": "int", "required": true }, - "flavor_vcpus": { "type": "int", "required": true }, - "flavor_disk": { "type": "int", "required": true }, - "keypair": { "type": "str", "required": true }, - "keypair_publickey": { "type": "str", "required": true }, - "keypair_publickeyfile": { "type": "str", "required": true }, + "flavor": { + "type" : "map", + "required" : true, + "mapping": { + "name": { "type": "str", "required": true }, + "ram": { "type": "int", "required": true }, + "vcpus": { "type": "int", "required": true }, + "disk": { "type": "int", "required": true } + } + }, + "keypair": { + "type" : "map", + "required" : true, + "mapping": { + "name": { "type": "str", "required": true }, + "publickey": { "type": "str", "required": true }, + "publickeyfile": { "type": "str", "required": true } + } + }, "heattemplate_path": { "type": "str", "required": true }, "volume_stack" : { "type" : "map", From d5b9bff591f2526054fb6ef500ad646e5afa6286 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 10 Apr 2017 09:21:16 +0200 Subject: [PATCH 035/181] Refactor oscm openstack json --- chef/cookbooks/oscm/recipes/server.rb | 1 + .../app/views/barclamp/oscm/_edit_attributes.html.haml | 2 +- crowbar_framework/config/locales/oscm/en.yml | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index ddf1c77c17..d236111827 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -156,6 +156,7 @@ owner oscm_group group oscm_group mode 0755 + recursive true end cookbook_file "#{oscm_heattemplate_path}/volumes.yaml" do diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 1d7959a8ed..6c5069fada 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -72,7 +72,7 @@ %span.help-block = t('.keypair_hint') - = text_field %w(openstack keypair_publickey) + = text_field %w(openstack keypair publickey) %fieldset %legend diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 4b3395d7ba..ab9b97629c 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -50,7 +50,8 @@ en: proxy_hint: 'A comma-separated list of hosts which can bypass the proxy' openstack_settings: "OpenStack Settings" openstack: - keypair_publickey: "Public Key" + keypair: + publickey: "Public Key" keypair_hint: 'SSH public key for the openstack instance where the OSCM will be deployed' api: protocol: 'Protocol' From c88e0e4dc4ecc442599ffe490e907726cefe90ea Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 11 Apr 2017 07:45:20 +0200 Subject: [PATCH 036/181] Add instance stack template --- .../oscm/files/default/application.yaml | 288 ++++++++++++++++++ .../files/default/user-data/deploy-oscmserver | 229 ++++++++++++++ .../oscm/files/default/user-data/write-config | 32 ++ chef/cookbooks/oscm/recipes/server.rb | 41 ++- 4 files changed, 585 insertions(+), 5 deletions(-) create mode 100644 chef/cookbooks/oscm/files/default/application.yaml create mode 100644 chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver create mode 100644 chef/cookbooks/oscm/files/default/user-data/write-config diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml new file mode 100644 index 0000000000..40cf3a679f --- /dev/null +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -0,0 +1,288 @@ +# heat stack-create --poll -f application.yaml -P app_volume_id=$app_volume_id -P db_volume_id=$db_volume_id -P key_name=default -P flavor=d1.tiny oscm +heat_template_version: 2015-10-15 + +description: | + # This is how you deploy the whole thing: + # 1) Instantiate volumes.yaml as follows + heat stack-create --poll -f volumes.yaml oscm + + # 2) Retrieve outputs from oscm: + app_volume_id=$(heat output-show oscm app_volume_id | sed 's/"//g') + db_volume_id=$(heat output-show oscm db_volume_id | sed 's/"//g') + + # 3) Create application stack: + heat stack-create --poll -f application.yaml -P app_volume_id=$app_volume_id -P db_volume_id=$db_volume_id -P key_name= oscm + + + +parameters: + floating_network: + type: string + default: floating + description: Network to draw Floating IPs from + image: + type: string + default: sles12-docker + description: Glance image to use for servers + flavor: + type: string + default: m1.small + description: Nova flavor to use for servers + key_name: + type: string + default: mykey + description: SSH key to add to servers' /root/.ssh/authorized_keys + app_volume_id: + type: string + description: The Cinder volume to attach to the application server + db_volume_id: + type: string + description: The Cinder volume to attach to the database server + app_name: + default: oscm + description: The application name + type: string + db_password: + default: postgres + type: OS::Heat::RandomString + description: Postgres database password + keystore_password: + default: root123 + type: type: OS::Heat::RandomString + description: Keystore protection password + as_admin_password: + default: root123 + type: type: OS::Heat::RandomString + description: Asadmin password + http_proxy: + default: '' + type: string + description: Http proxy + https_proxy: + default: '' + type: string + description: Https proxy + registry: + default: localhost:5000 + type: string + description: Docker registry + mail_address: + default: info@oscm.com + type: string + mail_host: + default: oscm.com + type: string + mail_password: + default: secretsecret + type: type: OS::Heat::RandomString + mail_user: + default: oscm + type: string + mail_port: + default: 7171 + type: string + mail_auth: + default: oscm + type: string + keystone_user_pwd: + default: secretsecret + type: string + keystone_user_name: + default: admin + type: string + keystone_url: + default: http://192.168.53.2:5000/v3/ + type: string + keystone_tenant_id: + default: 938ab7ecfc2c4d51b84f251e7bbdbba5 + type: string +resources: + + ### Network infrastructure ### + + oscm_network: + type: OS::Neutron::Net + properties: + name: oscm + + + oscm_subnet: + type: OS::Neutron::Subnet + properties: + cidr: 10.0.0.1/24 + name: oscm + network: + get_resource: oscm_network + + router: + type: OS::Neutron::Router + properties: + external_gateway_info: + network: + get_param: floating_network + + + router_interface: + type: OS::Neutron::RouterInterface + properties: + router: { get_resource: router } + subnet: { get_resource: oscm_subnet } + + allow_inbound: + type: OS::Neutron::SecurityGroup + properties: + description: "Allow inbound SSH and HTTP traffic" + name: oscm + rules: + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 22 + port_range_max: 22 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 80 + port_range_max: 80 + - remote_ip_prefix: 0.0.0.0/0 + protocol: icmp + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 5000 + port_range_max: 5000 + - direction: egress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 5000 + port_range_max: 5000 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8080 + port_range_max: 8080 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8880 + port_range_max: 8880 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8081 + port_range_max: 8081 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8048 + port_range_max: 8048 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8881 + port_range_max: 8881 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8848 + port_range_max: 8848 + + + + + # Parameters that will be available to both user data scripts + user_data_params: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: user-data/write-config} + params: + $APP_VOLUME_DEV: + list_join: + - "" + - - "/dev/disk/by-id/virtio-" + - { get_param: app_volume_id } + $DB_VOLUME_DEV: + list_join: + - "" + - - "/dev/disk/by-id/virtio-" + - { get_param: db_volume_id } + $DB_PASSWORD: { get_param: db_password } + $KEYSTORE_PASSWORD: { get_param: keystore_password } + $AS_ADMIN_PASSWORD: { get_param: as_admin_password } + $APP_NAME: { get_param: app_name } + $HTTP_PROXY: { get_param: http_proxy } + $HTTPS_PROXY: { get_param: https_proxy } + $REGISTRY: { get_param: registry } + $MAIL_ADDRESS: { get_param: mail_address } + $MAIL_HOST: { get_param: mail_host } + $MAIL_PASSWORD: { get_param: mail_password } + $MAIL_USER: { get_param: mail_user } + $MAIL_PORT: { get_param: mail_port } + $MAIL_AUTH: { get_param: mail_auth } + $KEYSTONE_USER_PWD: { get_param: keystone_user_pwd } + $KEYSTONE_USER_NAME: { get_param: keystone_user_name } + $KEYSTONE_URL: { get_param: keystone_url } + $KEYSTONE_TENANT_ID: { get_param: keystone_tenant_id } + + # User data payload for appserver + userdata_appserver: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: user_data_params} + - config: {get_file: user-data/deploy-oscmserver} + + + ### Servers ### + + appserver: + type: OS::Nova::Server + properties: + name: appserver + config_drive: true + flavor: { get_param: flavor } + image: { get_param: image } + key_name: { get_param: key_name } + networks: + - port: { get_resource: port_appserver } + user_data_format: RAW + user_data: { get_resource: userdata_appserver } + + port_appserver: + type: OS::Neutron::Port + properties: + network: + get_resource: oscm_network + security_groups: + - get_resource: allow_inbound + + ip_appserver: + type: OS::Neutron::FloatingIP + properties: + port_id: { get_resource: port_appserver } + floating_network: + get_param: floating_network + + app_volume_attachment: + type: OS::Cinder::VolumeAttachment + properties: + instance_uuid: { get_resource: appserver } + volume_id: { get_param: app_volume_id } + + db_volume_attachment: + type: OS::Cinder::VolumeAttachment + properties: + instance_uuid: { get_resource: appserver } + volume_id: { get_param: db_volume_id } + +outputs: + # Retrieval after stack creation (presuming stack is named `mystack`): + # heat output-show mystack ip_appserver + ip_appserver: + value: + get_attr: + - ip_appserver + - floating_ip_address diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver new file mode 100644 index 0000000000..34a42a9e1c --- /dev/null +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -0,0 +1,229 @@ +#!/bin/sh + +set -x +set -e + +exec &> /var/log/setup-machine.log + +source /etc/oscm-variables + + +sudo cat /etc/sysconfig/proxy < /etc/sysconfig/docker </dev/null 2>&1 || true + +# Create volume container for database +docker create -v $DB:/var/lib/pgsql/database --name $DB_VOLUME $DB_IMAGE /bin/true + +# create volume container for bes and master-indexer +docker create \ + -v $BES_LOGS_PATH:/opt/glassfish3/glassfish/domains/bes-domain/logs \ + -v $BES_CONFIG_PATH:/opt/glassfish3/glassfish/domains/bes-domain/config \ + -v $MI_LOGS_PATH:/opt/glassfish3/glassfish/domains/master-indexer-domain/logs \ + -v $MI_CONFIG_PATH:/opt/glassfish3/glassfish/domains/master-indexer-domain/config \ + -v $APP_CONFIG_PATH:/opt/glassfish3/glassfish/domains/app-domain/config \ + -v $CLIENT_CRT_PATH:/tmp/client-crt \ + --name $BES_VOLUME $OSCM_IMAGE /bin/true + +# create volume container for app +docker create \ + -v $APP_LOGS_PATH:/opt/glassfish3/glassfish/domains/app-domain/logs \ + -v $APP_CONFIG_PATH:/opt/glassfish3/glassfish/domains/app-domain/config \ + -v $BES_CONFIG_PATH:/opt/glassfish3/glassfish/domains/bes-domain/config \ + -v $CLIENT_CRT_PATH:/tmp/client-crt \ + --name $APP_VOLUME $APP_IMAGE /bin/true + + +# Docker compose file for the application deployment +if [ ! -x /oscm/docker-compose.yml ]; then +cat > /oscm/docker-compose.yml <> /oscm/docker-compose.yml +fi + +# OSCM start script +if [ ! -x /oscm/start-oscm.sh ]; then +cat > /oscm/start-oscm.sh </dev/null; then + echo "starting docker service" + systemctl start docker.service +fi + +if ! grep -qs '/db' /proc/mounts; then + mount -t ext4 $volume_dev_db /db +fi + +if ! grep -qs '/oscm' /proc/mounts; then + mount -t ext4 $volume_dev_db /oscm +fi + +echo "oscm is being started" +docker-compose -f /oscm/docker-compose.yml up --force-recreate +EOF +echo "# updated at $(date)" >> /oscm/start-oscm.sh +fi +chmod +x /oscm/start-oscm.sh + +# OSCM Stop script +if [ ! -x /oscm/stop-oscm.sh ]; then +cat > /oscm/stop-oscm.sh <> /oscm/stop-oscm.sh +fi +chmod +x /oscm/stop-oscm.sh + +# OSCM systemd service +cat > /etc/systemd/system/oscm.service <> /etc/systemd/system/oscm.service + +systemctl enable oscm.service +systemctl start oscm.service + +cat > /etc/motd < /etc/oscm-variables < /dev/null || exit 0 From afcb31bb817f79fc36f7c27af40b7146d40676a3 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 11 Apr 2017 12:49:20 +0200 Subject: [PATCH 037/181] Correct the source names for cookbook files --- chef/cookbooks/oscm/recipes/server.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 19b9fb0cac..cb81d88dd9 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -183,7 +183,7 @@ end cookbook_file "#{oscm_heattemplate_path}/user-data/write-config" do - source "write-config" + source "user-data/write-config" owner oscm_group group oscm_group mode 0755 @@ -191,7 +191,7 @@ end cookbook_file "#{oscm_heattemplate_path}/user-data/deploy-oscmserver" do - source "deploy-oscmserver" + source "user-data/deploy-oscmserver" owner oscm_group group oscm_group mode 0755 From 3b573aa62b71df02cfb4e08c6998eb6289ed0335 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 11 Apr 2017 15:47:17 +0200 Subject: [PATCH 038/181] Add instance stack --- .../oscm/files/default/application.yaml | 115 +----------------- chef/cookbooks/oscm/recipes/server.rb | 15 ++- chef/data_bags/crowbar/template-oscm.json | 7 +- chef/data_bags/crowbar/template-oscm.schema | 6 + 4 files changed, 25 insertions(+), 118 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 40cf3a679f..cfe437cc82 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -26,11 +26,11 @@ parameters: description: Glance image to use for servers flavor: type: string - default: m1.small + default: oscm.small description: Nova flavor to use for servers key_name: type: string - default: mykey + default: oscm description: SSH key to add to servers' /root/.ssh/authorized_keys app_volume_id: type: string @@ -38,64 +38,7 @@ parameters: db_volume_id: type: string description: The Cinder volume to attach to the database server - app_name: - default: oscm - description: The application name - type: string - db_password: - default: postgres - type: OS::Heat::RandomString - description: Postgres database password - keystore_password: - default: root123 - type: type: OS::Heat::RandomString - description: Keystore protection password - as_admin_password: - default: root123 - type: type: OS::Heat::RandomString - description: Asadmin password - http_proxy: - default: '' - type: string - description: Http proxy - https_proxy: - default: '' - type: string - description: Https proxy - registry: - default: localhost:5000 - type: string - description: Docker registry - mail_address: - default: info@oscm.com - type: string - mail_host: - default: oscm.com - type: string - mail_password: - default: secretsecret - type: type: OS::Heat::RandomString - mail_user: - default: oscm - type: string - mail_port: - default: 7171 - type: string - mail_auth: - default: oscm - type: string - keystone_user_pwd: - default: secretsecret - type: string - keystone_user_name: - default: admin - type: string - keystone_url: - default: http://192.168.53.2:5000/v3/ - type: string - keystone_tenant_id: - default: 938ab7ecfc2c4d51b84f251e7bbdbba5 - type: string + resources: ### Network infrastructure ### @@ -105,7 +48,6 @@ resources: properties: name: oscm - oscm_subnet: type: OS::Neutron::Subnet properties: @@ -187,55 +129,6 @@ resources: port_range_min: 8848 port_range_max: 8848 - - - - # Parameters that will be available to both user data scripts - user_data_params: - type: OS::Heat::SoftwareConfig - properties: - group: ungrouped - config: - str_replace: - template: {get_file: user-data/write-config} - params: - $APP_VOLUME_DEV: - list_join: - - "" - - - "/dev/disk/by-id/virtio-" - - { get_param: app_volume_id } - $DB_VOLUME_DEV: - list_join: - - "" - - - "/dev/disk/by-id/virtio-" - - { get_param: db_volume_id } - $DB_PASSWORD: { get_param: db_password } - $KEYSTORE_PASSWORD: { get_param: keystore_password } - $AS_ADMIN_PASSWORD: { get_param: as_admin_password } - $APP_NAME: { get_param: app_name } - $HTTP_PROXY: { get_param: http_proxy } - $HTTPS_PROXY: { get_param: https_proxy } - $REGISTRY: { get_param: registry } - $MAIL_ADDRESS: { get_param: mail_address } - $MAIL_HOST: { get_param: mail_host } - $MAIL_PASSWORD: { get_param: mail_password } - $MAIL_USER: { get_param: mail_user } - $MAIL_PORT: { get_param: mail_port } - $MAIL_AUTH: { get_param: mail_auth } - $KEYSTONE_USER_PWD: { get_param: keystone_user_pwd } - $KEYSTONE_USER_NAME: { get_param: keystone_user_name } - $KEYSTONE_URL: { get_param: keystone_url } - $KEYSTONE_TENANT_ID: { get_param: keystone_tenant_id } - - # User data payload for appserver - userdata_appserver: - type: OS::Heat::MultipartMime - properties: - parts: - - config: {get_resource: user_data_params} - - config: {get_file: user-data/deploy-oscmserver} - - ### Servers ### appserver: @@ -248,8 +141,6 @@ resources: key_name: { get_param: key_name } networks: - port: { get_resource: port_appserver } - user_data_format: RAW - user_data: { get_resource: userdata_appserver } port_appserver: type: OS::Neutron::Port diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index cb81d88dd9..294eb9ed7d 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -26,8 +26,10 @@ oscm_keypair_publickeyfile = node[:oscm][:openstack][:keypair][:publickeyfile] oscm_heattemplate_path = node[:oscm][:openstack][:heattemplate_path] oscm_volumestack_name = node[:oscm][:openstack][:volume_stack][:stack_name] +oscm_instancestack_name = node[:oscm][:openstack][:instance_stack][:stack_name] oscm_db_volume_size = node[:oscm][:openstack][:volume_stack][:db_volume_size] oscm_app_volume_size = node[:oscm][:openstack][:volume_stack][:app_volume_size] +oscm_image = node[:oscm][:openstack][:image] oscm_group = "root" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -116,9 +118,11 @@ end end -bash "add oscm flavor" do +bash "add oscm flavor and flavor access" do code <<-EOH - nova flavor-create #{oscm_flavor_name} auto #{oscm_flavor_ram} #{oscm_flavor_disk} #{oscm_flavor_vcpus} --is-public false &> /dev/null || exit 0 + nova flavor-create #{oscm_flavor_name} auto #{oscm_flavor_ram} #{oscm_flavor_disk} #{oscm_flavor_vcpus} --is-public false &> /dev/null + tenant_id=$(openstack project show -f shell #{oscm_tenant} | grep -Po "(?<=^id=\")[^\"]*") + nova flavor-access-add #{oscm_flavor_name} $tenant_id &> /dev/null EOH environment ({ "OS_USERNAME" => oscm_user, @@ -198,9 +202,12 @@ action :create end -bash "create oscm volumes stack with name #{oscm_volumestack_name}" do +bash "create oscm stacks" do code <<-EOH - openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || exit 0 + openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null + app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po "(?<=^output_value=\")[^\"]*") + db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po "(?<=^output_value=\")[^\"]*") + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" -t #{oscm_heattemplate_path}/application.yaml #{oscm_instancestack_name} &> /dev/null EOH environment ({ "OS_USERNAME" => oscm_user, diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 44d9896159..7ad7c539fb 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -26,10 +26,13 @@ "publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem" }, "heattemplate_path": "/etc/oscm/heat/", - "volume_stack" : { - "stack_name": "oscm", + "volume_stack": { + "stack_name": "oscm-volumes", "db_volume_size": 1, "app_volume_size": 1 + }, + "instance_stack": { + "stack_name": "oscm-instances" } }, "mail": { diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 0edce1fe0c..e4ab74a60b 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -57,6 +57,12 @@ "stack_name": { "type": "str", "required": true }, "db_volume_size": { "type": "int", "required": true }, "app_volume_size": { "type": "int", "required": true } + }, + "instance_stack" : { + "type" : "map", + "required" : true, + "mapping": { + "stack_name": { "type": "str", "required": true } } } } From e6a6648f41868893d8fa3d36b5e6b5f7a2694ab1 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 11 Apr 2017 15:54:03 +0200 Subject: [PATCH 039/181] Syntax error --- chef/data_bags/crowbar/template-oscm.schema | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index e4ab74a60b..b329bcb3b2 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -63,6 +63,7 @@ "required" : true, "mapping": { "stack_name": { "type": "str", "required": true } + } } } } From 960231554146162bf38b10a0999aff4c27af409d Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 11 Apr 2017 16:02:15 +0200 Subject: [PATCH 040/181] Coorect syntax --- chef/data_bags/crowbar/template-oscm.schema | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index b329bcb3b2..e380c9e829 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -57,13 +57,13 @@ "stack_name": { "type": "str", "required": true }, "db_volume_size": { "type": "int", "required": true }, "app_volume_size": { "type": "int", "required": true } - }, - "instance_stack" : { - "type" : "map", - "required" : true, - "mapping": { - "stack_name": { "type": "str", "required": true } - } + } + }, + "instance_stack" : { + "type" : "map", + "required" : true, + "mapping": { + "stack_name": { "type": "str", "required": true } } } } From 24eba661027b5b65c2a6519381ca3e21ef8975c1 Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 12 Apr 2017 09:10:11 +0200 Subject: [PATCH 041/181] Ignore resource exists errors --- chef/cookbooks/oscm/recipes/server.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 294eb9ed7d..4ead2ba07e 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -120,9 +120,9 @@ bash "add oscm flavor and flavor access" do code <<-EOH - nova flavor-create #{oscm_flavor_name} auto #{oscm_flavor_ram} #{oscm_flavor_disk} #{oscm_flavor_vcpus} --is-public false &> /dev/null - tenant_id=$(openstack project show -f shell #{oscm_tenant} | grep -Po "(?<=^id=\")[^\"]*") - nova flavor-access-add #{oscm_flavor_name} $tenant_id &> /dev/null + nova flavor-create #{oscm_flavor_name} auto #{oscm_flavor_ram} #{oscm_flavor_disk} #{oscm_flavor_vcpus} --is-public false &> /dev/null || true + tenant_id=$(openstack project show -f shell #{oscm_tenant} | grep -Po '(?<=^id=\")[^\"]*') + nova flavor-access-add #{oscm_flavor_name} $tenant_id &> /dev/null || true EOH environment ({ "OS_USERNAME" => oscm_user, @@ -204,10 +204,10 @@ bash "create oscm stacks" do code <<-EOH - openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null - app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po "(?<=^output_value=\")[^\"]*") - db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po "(?<=^output_value=\")[^\"]*") - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" -t #{oscm_heattemplate_path}/application.yaml #{oscm_instancestack_name} &> /dev/null + openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || true + app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') + db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" -t #{oscm_heattemplate_path}/application.yaml #{oscm_instancestack_name} &> /dev/null || true EOH environment ({ "OS_USERNAME" => oscm_user, From 552f995b572574db0428a936cad53bd6820084ac Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 12 Apr 2017 09:32:08 +0200 Subject: [PATCH 042/181] Correct cookbook file name --- chef/cookbooks/oscm/recipes/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 4ead2ba07e..7e09647218 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -172,7 +172,7 @@ end cookbook_file "#{oscm_heattemplate_path}/application.yaml" do - source "volumes.yaml" + source "application.yaml" owner oscm_group group oscm_group mode 0755 From 371f402fed9e488b8226907750b34a83152e9ba9 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 13 Apr 2017 11:33:01 +0200 Subject: [PATCH 043/181] Modified heat params --- .../oscm/files/default/application.yaml | 137 +++++++++++++++++- .../files/default/user-data/deploy-oscmserver | 9 +- .../oscm/files/default/user-data/write-config | 3 + 3 files changed, 140 insertions(+), 9 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index cfe437cc82..ad0599cfde 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -26,7 +26,7 @@ parameters: description: Glance image to use for servers flavor: type: string - default: oscm.small + default: m1.small description: Nova flavor to use for servers key_name: type: string @@ -38,9 +38,77 @@ parameters: db_volume_id: type: string description: The Cinder volume to attach to the database server + app_name: + default: oscm + description: The application name + type: string + http_proxy: + default: '' + type: string + description: Http proxy + https_proxy: + default: '' + type: string + description: Https proxy + registry: + default: localhost:5000 + type: string + description: Docker registry + mail_address: + default: "" + type: string + mail_host: + default: "" + type: string + mail_password: + default: "" + type: string + mail_user: + default: oscm + type: string + mail_port: + default: 7171 + type: string + mail_auth: + default: oscm + type: string + keystone_user_pwd: + default: secretsecret + type: string + keystone_user_name: + default: admin + type: string + keystone_url: + default: http://192.168.53.2:5000/v3/ + type: string + keystone_tenant_id: + default: 938ab7ecfc2c4d51b84f251e7bbdbba5 + type: string + keystone_domain: + default: default + type: string + oscm_image_name: + default: oscm + type: string + app_image_name: + default: app + type: string + db_image_name: + default: db + type: string resources: + + db_password: + type: OS::Heat::RandomString + + keystore_password: + type: OS::Heat::RandomString + + as_admin_password: + type: OS::Heat::RandomString + ### Network infrastructure ### oscm_network: @@ -48,6 +116,7 @@ resources: properties: name: oscm + oscm_subnet: type: OS::Neutron::Subnet properties: @@ -116,18 +185,74 @@ resources: - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp - port_range_min: 8048 - port_range_max: 8048 + port_range_min: 8881 + port_range_max: 8881 - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp - port_range_min: 8881 - port_range_max: 8881 + port_range_min: 8048 + port_range_max: 8048 - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp port_range_min: 8848 port_range_max: 8848 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8448 + port_range_max: 8448 + + + # Parameters that will be available to both user data scripts + user_data_params: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: user-data/write-config} + params: + $APP_VOLUME_DEV: + list_join: + - "" + - - "/dev/disk/by-id/virtio-" + - { get_param: app_volume_id } + $DB_VOLUME_DEV: + list_join: + - "" + - - "/dev/disk/by-id/virtio-" + - { get_param: db_volume_id } + $DB_PASSWORD: { get_attr: [ db_password, value ] } + $KEYSTORE_PASSWORD: { get_attr: [ keystore_password, value ] } + $AS_ADMIN_PASSWORD: { get_attr: [ as_admin_password, value ] } + $APP_NAME: { get_param: app_name } + $HTTP_PROXY: { get_param: http_proxy } + $HTTPS_PROXY: { get_param: https_proxy } + $REGISTRY: { get_param: registry } + $MAIL_ADDRESS: { get_param: mail_address } + $MAIL_HOST: { get_param: mail_host } + $MAIL_PASSWORD: { get_param: mail_password } + $MAIL_USER: { get_param: mail_user } + $MAIL_PORT: { get_param: mail_port } + $MAIL_AUTH: { get_param: mail_auth } + $KEYSTONE_USER_PWD: { get_param: keystone_user_pwd } + $KEYSTONE_USER_NAME: { get_param: keystone_user_name } + $KEYSTONE_URL: { get_param: keystone_url } + $KEYSTONE_TENANT_ID: { get_param: keystone_tenant_id } + $KEYSTONE_DOMAIN: { get_param: keystone_domain } + $OSCM_IMAGE_NAME: { get_param: oscm_image_name } + $APP_IMAGE_NAME: { get_param: app_image_name } + $DB_IMAGE_NAME: { get_param: db_image_name } + + # User data payload for appserver + userdata_appserver: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: user_data_params} + - config: {get_file: user-data/deploy-oscmserver} + ### Servers ### @@ -141,6 +266,8 @@ resources: key_name: { get_param: key_name } networks: - port: { get_resource: port_appserver } + user_data_format: RAW + user_data: { get_resource: userdata_appserver } port_appserver: type: OS::Neutron::Port diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 34a42a9e1c..89d5c9981f 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -61,9 +61,9 @@ if ! grep -qs '/oscm' /proc/mounts; then fi -OSCM_IMAGE=$REGISTRY/oscm-web -APP_IMAGE=$REGISTRY/oscm-app -DB_IMAGE=$REGISTRY/oscm-db +OSCM_IMAGE=$REGISTRY/$OSCM_IMAGE_NAME +APP_IMAGE=$REGISTRY/$APP_IMAGE_NAME +DB_IMAGE=$REGISTRY/$DB_IMAGE_NAME DB=/db/database APP_LOGS_PATH=/oscm/app/logs APP_CONFIG_PATH=/oscm/app/config @@ -129,6 +129,7 @@ services: - "8080:8080" - "8081:8081" - "8048:8048" + - "8448:8448" volumes_from: - container:$BES_VOLUME:rw $APP_HOSTNAME: @@ -187,7 +188,7 @@ cat > /oscm/stop-oscm.sh <> /oscm/stop-oscm.sh diff --git a/chef/cookbooks/oscm/files/default/user-data/write-config b/chef/cookbooks/oscm/files/default/user-data/write-config index 9078f555b0..4422b9b3cf 100644 --- a/chef/cookbooks/oscm/files/default/user-data/write-config +++ b/chef/cookbooks/oscm/files/default/user-data/write-config @@ -29,4 +29,7 @@ KEYSTONE_USER_PWD=$KEYSTONE_USER_PWD KEYSTONE_USER_NAME=$KEYSTONE_USER_NAME KEYSTONE_URL=$KEYSTONE_URL KEYSTONE_TENANT_ID=$KEYSTONE_TENANT_ID +OSCM_IMAGE_NAME=$OSCM_IMAGE_NAME +APP_IMAGE_NAME=$APP_IMAGE_NAME +DB_IMAGE_NAME=$DB_IMAGE_NAME EOF From 80fd84d0d6feddc259841bfd01ddbcad82725256 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 13 Apr 2017 11:46:02 +0200 Subject: [PATCH 044/181] Add proxy and docker params --- chef/cookbooks/oscm/recipes/server.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 7e09647218..0eec1e0396 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -30,6 +30,12 @@ oscm_db_volume_size = node[:oscm][:openstack][:volume_stack][:db_volume_size] oscm_app_volume_size = node[:oscm][:openstack][:volume_stack][:app_volume_size] oscm_image = node[:oscm][:openstack][:image] +oscm_docker_host = node[:oscm][:docker][:host] +oscm_docker_port = node[:oscm][:docker][:port] +oscm_proxy_httphost = node[:oscm][:proxy][:http_host] +oscm_proxy_httpport = node[:oscm][:proxy][:http_port] +oscm_proxy_httpshost = node[:oscm][:proxy][:https_host] +oscm_proxy_httpsport = node[:oscm][:proxy][:https_port] oscm_group = "root" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -207,7 +213,9 @@ openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || true app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" -t #{oscm_heattemplate_path}/application.yaml #{oscm_instancestack_name} &> /dev/null || true + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}"\ + --parameter "http_proxy=#{oscm_proxy_httphost}:#{oscm_proxy_httpport}" --parameter "https_proxy=#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}" --parameter "registry=#{oscm_docker_host}:#{oscm_docker_port}"\ + -t #{oscm_heattemplate_path}/application.yaml #{oscm_instancestack_name} &> /dev/null || true EOH environment ({ "OS_USERNAME" => oscm_user, From a03fe08a2661fd7405674d4030816463721ec44e Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 13 Apr 2017 12:43:13 +0200 Subject: [PATCH 045/181] Add mail params --- chef/cookbooks/oscm/files/default/application.yaml | 12 ++++++------ chef/cookbooks/oscm/recipes/server.rb | 10 +++++++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index ad0599cfde..20cd1a4f56 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -67,22 +67,22 @@ parameters: default: oscm type: string mail_port: - default: 7171 + default: 25 type: string mail_auth: - default: oscm + default: false type: string keystone_user_pwd: - default: secretsecret + default: oscm type: string keystone_user_name: - default: admin + default: oscm type: string keystone_url: - default: http://192.168.53.2:5000/v3/ + default: http://192.168.53.2:5000/v3/ type: string keystone_tenant_id: - default: 938ab7ecfc2c4d51b84f251e7bbdbba5 + default: oscm type: string keystone_domain: default: default diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 0eec1e0396..79821a0135 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -36,6 +36,13 @@ oscm_proxy_httpport = node[:oscm][:proxy][:http_port] oscm_proxy_httpshost = node[:oscm][:proxy][:https_host] oscm_proxy_httpsport = node[:oscm][:proxy][:https_port] +oscm_mail_host = node[:oscm][:mail][:host] +oscm_mail_port = node[:oscm][:mail][:port] +oscm_mail_from = node[:oscm][:mail][:from] +oscm_mail_auth = node[:oscm][:mail][:auth] +oscm_mail_user = node[:oscm][:mail][:user] +oscm_mail_pwd = node[:oscm][:mail][:password] + oscm_group = "root" keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) @@ -213,7 +220,8 @@ openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || true app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}"\ + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ + --parameter "mail_host=#{oscm_mail_host}" --parameter "mail_port=#{oscm_mail_port}" --parameter "mail_address=#{oscm_mail_from}" --parameter "mail_auth=#{oscm_mail_auth}" --parameter "mail_user=#{oscm_mail_user}" --parameter "mail_password=#{oscm_mail_pwd}"\ --parameter "http_proxy=#{oscm_proxy_httphost}:#{oscm_proxy_httpport}" --parameter "https_proxy=#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}" --parameter "registry=#{oscm_docker_host}:#{oscm_docker_port}"\ -t #{oscm_heattemplate_path}/application.yaml #{oscm_instancestack_name} &> /dev/null || true EOH From 99a95dd84f72eef2d7de1adb70e31b88ea23ec14 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 13 Apr 2017 14:27:02 +0200 Subject: [PATCH 046/181] Add mail port in oscm security group --- chef/cookbooks/oscm/files/default/application.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 20cd1a4f56..9d48ea5e04 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -202,6 +202,11 @@ resources: protocol: tcp port_range_min: 8448 port_range_max: 8448 + - direction: egress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: { get_param: mail_port } + port_range_max: { get_param: mail_port } # Parameters that will be available to both user data scripts From 293b8e37319eb2aad9805aac835392186c3ddd29 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 27 Apr 2017 14:49:55 +0200 Subject: [PATCH 047/181] Inject ssh certs --- .../oscm/files/default/application.yaml | 43 ++++++++- .../files/default/user-data/deploy-oscmserver | 89 +++++++++---------- .../oscm/files/default/user-data/write-config | 7 +- chef/cookbooks/oscm/recipes/server.rb | 42 +++++++-- chef/data_bags/crowbar/template-oscm.json | 1 + chef/data_bags/crowbar/template-oscm.schema | 3 +- .../barclamp/oscm/_edit_attributes.html.haml | 1 - 7 files changed, 124 insertions(+), 62 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 9d48ea5e04..ff124a5980 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -26,7 +26,7 @@ parameters: description: Glance image to use for servers flavor: type: string - default: m1.small + default: oscm.medium description: Nova flavor to use for servers key_name: type: string @@ -43,11 +43,11 @@ parameters: description: The application name type: string http_proxy: - default: '' + default: "" type: string description: Http proxy https_proxy: - default: '' + default: "" type: string description: Https proxy registry: @@ -96,9 +96,33 @@ parameters: db_image_name: default: db type: string + ssl_path: + default: "" + type: string + wait_condition_timeout: + default: 1800 + type: number + ssh_cert: + default: "" + type: string + heat_host_cidr: + default: "" + type: string + heat_port: + default: 8004 + type: number resources: + ssl_wait_handle: + type: OS::Heat::WaitConditionHandle + + ssl_wait_condition: + type: OS::Heat::WaitCondition + depends_on: appserver + properties: + handle: { get_resource: ssl_wait_handle } + timeout: { get_param: wait_condition_timeout } db_password: type: OS::Heat::RandomString @@ -207,6 +231,16 @@ resources: protocol: tcp port_range_min: { get_param: mail_port } port_range_max: { get_param: mail_port } + - direction: ingress + remote_ip_prefix: { get_param: heat_host_cidr } + protocol: tcp + port_range_min: { get_param: heat_port } + port_range_max: { get_param: heat_port } + - direction: egress + remote_ip_prefix: { get_param: heat_host_cidr } + protocol: tcp + port_range_min: { get_param: heat_port } + port_range_max: { get_param: heat_port } # Parameters that will be available to both user data scripts @@ -249,6 +283,9 @@ resources: $OSCM_IMAGE_NAME: { get_param: oscm_image_name } $APP_IMAGE_NAME: { get_param: app_image_name } $DB_IMAGE_NAME: { get_param: db_image_name } + $SSL_PATH: { get_param: ssl_path } + $WAIT_CURL: { get_attr: [ ssl_wait_handle, curl_cli ] } + $SSH_CERT: { get_param: ssh_cert } # User data payload for appserver userdata_appserver: diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 89d5c9981f..e11d453456 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -7,6 +7,7 @@ exec &> /var/log/setup-machine.log source /etc/oscm-variables +WEB_CERT_PATH=/var/web_cert sudo cat /etc/sysconfig/proxy < /var/web_cert +fi + # For some reason the volume id is shortened to 27 characters when the device # link in /dev/disk/by-id/ is created. Account for this by truncating the @@ -65,25 +79,19 @@ OSCM_IMAGE=$REGISTRY/$OSCM_IMAGE_NAME APP_IMAGE=$REGISTRY/$APP_IMAGE_NAME DB_IMAGE=$REGISTRY/$DB_IMAGE_NAME DB=/db/database -APP_LOGS_PATH=/oscm/app/logs -APP_CONFIG_PATH=/oscm/app/config -BES_LOGS_PATH=/oscm/bes/logs -BES_CONFIG_PATH=/oscm/bes/config -MI_LOGS_PATH=/oscm/master-indexer/logs -MI_CONFIG_PATH=/oscm/master-indexer/config +APP_PATH=/oscm/app/ +BES_PATH=/oscm/bes/ +MI_PATH=/oscm/master-indexer CLIENT_CRT_PATH=/oscm/client-crt/ + DB_VOLUME=dbvolume BES_VOLUME=besvolume APP_VOLUME=appvolume - mkdir -p $DB -mkdir -p $APP_LOGS_PATH -mkdir -p $APP_CONFIG_PATH -mkdir -p $BES_LOGS_PATH -mkdir -p $BES_CONFIG_PATH -mkdir -p $MI_LOGS_PATH -mkdir -p $MI_CONFIG_PATH +mkdir -p $APP_PATH +mkdir -p $BES_PATH +mkdir -p $MI_PATH # Start docker service systemctl start docker.service @@ -96,26 +104,25 @@ docker create -v $DB:/var/lib/pgsql/database --name $DB_VOLUME $DB_IMAGE /bin/tr # create volume container for bes and master-indexer docker create \ - -v $BES_LOGS_PATH:/opt/glassfish3/glassfish/domains/bes-domain/logs \ - -v $BES_CONFIG_PATH:/opt/glassfish3/glassfish/domains/bes-domain/config \ - -v $MI_LOGS_PATH:/opt/glassfish3/glassfish/domains/master-indexer-domain/logs \ - -v $MI_CONFIG_PATH:/opt/glassfish3/glassfish/domains/master-indexer-domain/config \ - -v $APP_CONFIG_PATH:/opt/glassfish3/glassfish/domains/app-domain/config \ + -v $BES_PATH:/opt/glassfish3/glassfish/domains/bes-domain/ \ + -v $MI_PATH:/opt/glassfish3/glassfish/domains/master-indexer-domain/ \ + -v $APP_PATH:/opt/glassfish3/glassfish/domains/app-domain/ \ -v $CLIENT_CRT_PATH:/tmp/client-crt \ --name $BES_VOLUME $OSCM_IMAGE /bin/true # create volume container for app docker create \ - -v $APP_LOGS_PATH:/opt/glassfish3/glassfish/domains/app-domain/logs \ - -v $APP_CONFIG_PATH:/opt/glassfish3/glassfish/domains/app-domain/config \ - -v $BES_CONFIG_PATH:/opt/glassfish3/glassfish/domains/bes-domain/config \ + -v $APP_PATH:/opt/glassfish3/glassfish/domains/app-domain/ \ + -v $BES_PATH:/opt/glassfish3/glassfish/domains/bes-domain/ \ -v $CLIENT_CRT_PATH:/tmp/client-crt \ --name $APP_VOLUME $APP_IMAGE /bin/true +OSCM_DIR=/opt/oscm +mkdir -p $OSCM_DIR # Docker compose file for the application deployment -if [ ! -x /oscm/docker-compose.yml ]; then -cat > /oscm/docker-compose.yml < $OSCM_DIR/docker-compose.yml <> /oscm/docker-compose.yml +echo "# updated at $(date)" >> $OSCM_DIR/docker-compose.yml fi # OSCM start script -if [ ! -x /oscm/start-oscm.sh ]; then -cat > /oscm/start-oscm.sh < $OSCM_DIR/start-oscm.sh </dev/null; then echo "starting docker service" systemctl start docker.service fi - if ! grep -qs '/db' /proc/mounts; then mount -t ext4 $volume_dev_db /db fi - if ! grep -qs '/oscm' /proc/mounts; then mount -t ext4 $volume_dev_db /oscm fi - echo "oscm is being started" -docker-compose -f /oscm/docker-compose.yml up --force-recreate +docker-compose -f $OSCM_DIR/docker-compose.yml up --force-recreate EOF -echo "# updated at $(date)" >> /oscm/start-oscm.sh +echo "# updated at $(date)" >> $OSCM_DIR/start-oscm.sh fi -chmod +x /oscm/start-oscm.sh +chmod +x $OSCM_DIR/start-oscm.sh # OSCM Stop script -if [ ! -x /oscm/stop-oscm.sh ]; then -cat > /oscm/stop-oscm.sh < $OSCM_DIR/stop-oscm.sh <> /oscm/stop-oscm.sh +echo "# updated at $(date)" >> $OSCM_DIR/stop-oscm.sh fi -chmod +x /oscm/stop-oscm.sh +chmod +x $OSCM_DIR/stop-oscm.sh # OSCM systemd service cat > /etc/systemd/system/oscm.service < /etc/systemd/system/oscm.service < /etc/motd <> /root/.ssh/authorized_keys \ No newline at end of file diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 79821a0135..f3963bf275 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -17,6 +17,10 @@ oscm_tenant = node[:oscm][:keystone][:tenant] oscm_user = node[:oscm][:keystone][:user] oscm_password = node[:oscm][:keystone][:password] +oscm_ssl_certfile = node[:oscm][:ssl][:certfile] +oscm_ssl_keyfile = node[:oscm][:ssl][:keyfile] +oscm_ssl_cacerts = node[:oscm][:ssl][:ca_certs] +oscm_ssl_scp_path = node[:oscm][:ssl][:scp_path] oscm_flavor_name = node[:oscm][:openstack][:flavor][:name] oscm_flavor_ram = node[:oscm][:openstack][:flavor][:ram] oscm_flavor_vcpus = node[:oscm][:openstack][:flavor][:vcpus] @@ -42,9 +46,13 @@ oscm_mail_auth = node[:oscm][:mail][:auth] oscm_mail_user = node[:oscm][:mail][:user] oscm_mail_pwd = node[:oscm][:mail][:password] - +oscm_keypair_crowbar_sshkey = "/etc/oscm/ssh/oscm_ssh.key" oscm_group = "root" +heat_node = node_search_with_cache("roles:heat-server").first +heat_public_host = CrowbarHelper.get_host_for_public_url(heat_node, false) +heat_port = heat_node[:heat][:api][:port] + keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) register_auth_hash = { @@ -217,14 +225,30 @@ bash "create oscm stacks" do code <<-EOH - openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml #{oscm_volumestack_name} &> /dev/null || true - app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ - --parameter "mail_host=#{oscm_mail_host}" --parameter "mail_port=#{oscm_mail_port}" --parameter "mail_address=#{oscm_mail_from}" --parameter "mail_auth=#{oscm_mail_auth}" --parameter "mail_user=#{oscm_mail_user}" --parameter "mail_password=#{oscm_mail_pwd}"\ - --parameter "http_proxy=#{oscm_proxy_httphost}:#{oscm_proxy_httpport}" --parameter "https_proxy=#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}" --parameter "registry=#{oscm_docker_host}:#{oscm_docker_port}"\ - -t #{oscm_heattemplate_path}/application.yaml #{oscm_instancestack_name} &> /dev/null || true -EOH + openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml --wait #{oscm_volumestack_name} &> /dev/null || true + app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') + db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') + mkdir -p "$(dirname "#{oscm_keypair_crowbar_sshkey}")" + if [ ! -f #{oscm_keypair_crowbar_sshkey} ]; + then + ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} + fi + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ + --parameter "mail_host=#{oscm_mail_host}" --parameter "mail_port=#{oscm_mail_port}" --parameter "mail_address=#{oscm_mail_from}" --parameter "mail_auth=#{oscm_mail_auth}" --parameter "mail_user=#{oscm_mail_user}" --parameter "mail_password=#{oscm_mail_pwd}"\ + --parameter "http_proxy=#{oscm_proxy_httphost}:#{oscm_proxy_httpport}" --parameter "https_proxy=#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}"\ + --parameter "registry=#{oscm_docker_host}:#{oscm_docker_port}"\ + --parameter "ssl_path=#{oscm_ssl_scp_path}"\ + --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ + --parameter-file "ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub"\ + -t #{oscm_heattemplate_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true + ip_appserver=$(openstack stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*') + ssh-keygen -R ${ip_appserver} -f /root/.ssh/known_hosts + ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_ssl_scp_path}" || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_ssl_scp_path} || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_keyfile} ${ip_appserver}:#{oscm_ssl_scp_path} || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_ssl_scp_path} || true + ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_ssl_scp_path}/scp_finished" + EOH environment ({ "OS_USERNAME" => oscm_user, "OS_PASSWORD" => oscm_password, diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 7ad7c539fb..7e2ae32d10 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -63,6 +63,7 @@ "password": "" }, "ssl": { + "scp_path": "/etc/oscm/ssl/", "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", "keyfile": "/etc/oscm/ssl/private/signing_key.pem", "generate_certs": false, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index e380c9e829..8fa4acbd15 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -109,8 +109,9 @@ }, "ssl": { "type": "map", - "required": true, + "required": true, "mapping": { + "scp_path": { "type" : "str", "required" : true }, "certfile": { "type" : "str", "required" : true }, "keyfile": { "type" : "str", "required" : true }, "generate_certs": { "type" : "bool", "required" : true }, diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 6c5069fada..42229ee55b 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -85,7 +85,6 @@ "data-sslkey" => "/etc/oscm/ssl/private/signing_key.pem" #ssl_container - = boolean_field %w(ssl generate_certs) = string_field %w(ssl certfile) = string_field %w(ssl keyfile) = boolean_field %w(ssl insecure) From df26831a3f03f834086cb44ce4c664b33c759146 Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 2 Jun 2017 08:35:42 +0200 Subject: [PATCH 048/181] Add missing line --- bin/crowbar_oscm | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/crowbar_oscm b/bin/crowbar_oscm index 7e69f96668..83f0b39dd9 100644 --- a/bin/crowbar_oscm +++ b/bin/crowbar_oscm @@ -1,3 +1,4 @@ +#!/usr/bin/env ruby # # Copyright 2017, SUSE LINUX GmbH # From c6471ef17acd4ab5648662dc84dead8c6627aad0 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 19 Sep 2017 11:18:23 +0200 Subject: [PATCH 049/181] Correct whitespace --- chef/roles/oscm-server.rb | 2 +- crowbar_framework/app/controllers/oscm_controller.rb | 2 +- crowbar_framework/app/models/oscm_service.rb | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/chef/roles/oscm-server.rb b/chef/roles/oscm-server.rb index ca5a7df52b..3948922df6 100644 --- a/chef/roles/oscm-server.rb +++ b/chef/roles/oscm-server.rb @@ -2,4 +2,4 @@ description "OSCM Server Role" run_list("recipe[oscm::role_oscm_server]") default_attributes -override_attributes \ No newline at end of file +override_attributes diff --git a/crowbar_framework/app/controllers/oscm_controller.rb b/crowbar_framework/app/controllers/oscm_controller.rb index d9cdb5e90f..ef16e99827 100644 --- a/crowbar_framework/app/controllers/oscm_controller.rb +++ b/crowbar_framework/app/controllers/oscm_controller.rb @@ -22,4 +22,4 @@ class OscmController < BarclampController def initialize_service @service_object = OscmService.new logger end -end \ No newline at end of file +end diff --git a/crowbar_framework/app/models/oscm_service.rb b/crowbar_framework/app/models/oscm_service.rb index 95e45834db..e82f77d19c 100644 --- a/crowbar_framework/app/models/oscm_service.rb +++ b/crowbar_framework/app/models/oscm_service.rb @@ -42,7 +42,6 @@ def role_constraints end end - def proposal_dependencies(role) answer = [] ["heat"].each do |dep| @@ -62,7 +61,7 @@ def create_proposal base["deployment"][@bc_name]["elements"] = { "oscm-server" => [server_nodes.first.name] } unless server_nodes.nil? - + base["attributes"][@bc_name]["keystone_instance"] = find_dep_proposal("keystone") base["attributes"][@bc_name]["heat_instance"] = find_dep_proposal("heat") @@ -75,5 +74,4 @@ def validate_proposal_after_save(proposal) super end - end From e70c8d965754602a3185023d798ea1a9c34b412a Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 20 Sep 2017 13:51:05 +0200 Subject: [PATCH 050/181] Refator - prepare for tommee-based containers --- .../oscm/files/default/application.yaml | 49 ++++++++++--------- .../files/default/user-data/deploy-oscmserver | 22 ++++----- .../oscm/files/default/user-data/write-config | 5 +- chef/cookbooks/oscm/recipes/server.rb | 2 +- chef/data_bags/crowbar/template-oscm.json | 1 - chef/data_bags/crowbar/template-oscm.schema | 1 - .../barclamp/oscm/_edit_attributes.html.haml | 23 ++++----- crowbar_framework/config/locales/oscm/en.yml | 2 +- 8 files changed, 50 insertions(+), 55 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index ff124a5980..edef016596 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -50,10 +50,23 @@ parameters: default: "" type: string description: Https proxy - registry: - default: localhost:5000 + registry_host: + default: "" + type: string + description: Docker registry host + registry_port: + default: "" + type: string + description: Docker registry port + registry_user: + default: "" type: string - description: Docker registry + description: Docker registry user + registry_pwd: + default: "" + type: string + hidden: true + description: Docker registry password mail_address: default: "" type: string @@ -79,7 +92,7 @@ parameters: default: oscm type: string keystone_url: - default: http://192.168.53.2:5000/v3/ + default: "" type: string keystone_tenant_id: default: oscm @@ -184,13 +197,13 @@ resources: - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp - port_range_min: 5000 - port_range_max: 5000 + port_range_min: { get_param: registry_port } + port_range_max: { get_param: registry_port } - direction: egress remote_ip_prefix: 0.0.0.0/0 protocol: tcp - port_range_min: 5000 - port_range_max: 5000 + port_range_min: { get_param: registry_port } + port_range_max: { get_param: registry_port } - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp @@ -211,21 +224,6 @@ resources: protocol: tcp port_range_min: 8881 port_range_max: 8881 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8048 - port_range_max: 8048 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8848 - port_range_max: 8848 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8448 - port_range_max: 8448 - direction: egress remote_ip_prefix: 0.0.0.0/0 protocol: tcp @@ -268,7 +266,10 @@ resources: $APP_NAME: { get_param: app_name } $HTTP_PROXY: { get_param: http_proxy } $HTTPS_PROXY: { get_param: https_proxy } - $REGISTRY: { get_param: registry } + $REGISTRY_HOST: { get_param: registry_host } + $REGISTRY_PORT: { get_param: registry_port } + $REGISTRY_USER: { get_param: registry_user } + $REGISTRY_PWD: { get_param: registry_pwd } $MAIL_ADDRESS: { get_param: mail_address } $MAIL_HOST: { get_param: mail_host } $MAIL_PASSWORD: { get_param: mail_password } diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index e11d453456..438a1204cf 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -17,14 +17,17 @@ FTP_PROXY="" GOPHER_PROXY="" SOCKS_PROXY="" SOCKS5_SERVER="" -NO_PROXY="localhost, 127.0.0.1, $REGISTRY" +NO_PROXY="localhost, 127.0.0.1, $REGISTRY_HOST" EOF +# TODO: +# If $REGISTRY_HOST empty, docker hub is default. +# Remove insecure sudo cat > /etc/sysconfig/docker < /var/web_cert fi @@ -81,7 +84,6 @@ DB_IMAGE=$REGISTRY/$DB_IMAGE_NAME DB=/db/database APP_PATH=/oscm/app/ BES_PATH=/oscm/bes/ -MI_PATH=/oscm/master-indexer CLIENT_CRT_PATH=/oscm/client-crt/ DB_VOLUME=dbvolume @@ -91,7 +93,6 @@ APP_VOLUME=appvolume mkdir -p $DB mkdir -p $APP_PATH mkdir -p $BES_PATH -mkdir -p $MI_PATH # Start docker service systemctl start docker.service @@ -102,10 +103,9 @@ docker rm -f $APP_VOLUME $BES_VOLUME $DB_VOLUME db web app >/dev/null 2>&1 || tr # Create volume container for database docker create -v $DB:/var/lib/pgsql/database --name $DB_VOLUME $DB_IMAGE /bin/true -# create volume container for bes and master-indexer +# create volume container for bes and app docker create \ -v $BES_PATH:/opt/glassfish3/glassfish/domains/bes-domain/ \ - -v $MI_PATH:/opt/glassfish3/glassfish/domains/master-indexer-domain/ \ -v $APP_PATH:/opt/glassfish3/glassfish/domains/app-domain/ \ -v $CLIENT_CRT_PATH:/tmp/client-crt \ --name $BES_VOLUME $OSCM_IMAGE /bin/true @@ -135,8 +135,6 @@ services: ports: - "8080:8080" - "8081:8081" - - "8048:8048" - - "8448:8448" volumes_from: - container:$BES_VOLUME:rw $APP_HOSTNAME: @@ -150,7 +148,6 @@ services: ports: - "8880:8880" - "8881:8881" - - "8848:8848" volumes_from: - container:$APP_VOLUME:rw $DB_HOSTNAME: @@ -164,7 +161,8 @@ EOF echo "# updated at $(date)" >> $OSCM_DIR/docker-compose.yml fi -# OSCM start script +# OSCM start script +# TODO: Add docker login -u="$REGISTRY_USER" -p="REGISTRY_PWD" if $REGISTRY_USER not empty. if [ ! -x $OSCM_DIR/start-oscm.sh ]; then cat > $OSCM_DIR/start-oscm.sh < "true", - "data-showit-target" => "#docker_container", + "data-showit-target" => "#dockerauth_container", "data-showit-direct" => "true" - #docker_container - = string_field %w(docker host) - = integer_field %w(docker port) - = boolean_field %w(docker auth), - "data-showit" => "true", - "data-showit-target" => "#dockerauth_container", - "data-showit-direct" => "true" - - #dockerauth_container - = string_field %w(docker user) - = password_field %w(docker password) + #dockerauth_container + = string_field %w(docker user) + = password_field %w(docker password) %fieldset %legend diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index ab9b97629c..56c0501afe 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -31,11 +31,11 @@ en: docker_settings: 'Docker Registry' docker: auth: 'Authentication Required' - use_registry: 'Install from Local Registry' host: 'Registry Host' port: 'Registry Port' user: 'User' password: 'Password' + docker_hint: 'The registry host and port can be leaved empty for docker hub' proxy_settings: 'Proxy Settings' proxy: auth: 'Authentication Required' From b341660cdbd327ab6677712d9e6b7468ee4c285c Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 20 Sep 2017 13:51:05 +0200 Subject: [PATCH 051/181] Refator - prepare for tommee-based containers --- .../oscm/files/default/application.yaml | 49 ++++++++++--------- .../files/default/user-data/deploy-oscmserver | 22 ++++----- .../oscm/files/default/user-data/write-config | 5 +- chef/cookbooks/oscm/recipes/server.rb | 4 +- chef/data_bags/crowbar/template-oscm.json | 1 - chef/data_bags/crowbar/template-oscm.schema | 1 - .../barclamp/oscm/_edit_attributes.html.haml | 23 ++++----- crowbar_framework/config/locales/oscm/en.yml | 2 +- 8 files changed, 52 insertions(+), 55 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index ff124a5980..edef016596 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -50,10 +50,23 @@ parameters: default: "" type: string description: Https proxy - registry: - default: localhost:5000 + registry_host: + default: "" + type: string + description: Docker registry host + registry_port: + default: "" + type: string + description: Docker registry port + registry_user: + default: "" type: string - description: Docker registry + description: Docker registry user + registry_pwd: + default: "" + type: string + hidden: true + description: Docker registry password mail_address: default: "" type: string @@ -79,7 +92,7 @@ parameters: default: oscm type: string keystone_url: - default: http://192.168.53.2:5000/v3/ + default: "" type: string keystone_tenant_id: default: oscm @@ -184,13 +197,13 @@ resources: - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp - port_range_min: 5000 - port_range_max: 5000 + port_range_min: { get_param: registry_port } + port_range_max: { get_param: registry_port } - direction: egress remote_ip_prefix: 0.0.0.0/0 protocol: tcp - port_range_min: 5000 - port_range_max: 5000 + port_range_min: { get_param: registry_port } + port_range_max: { get_param: registry_port } - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp @@ -211,21 +224,6 @@ resources: protocol: tcp port_range_min: 8881 port_range_max: 8881 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8048 - port_range_max: 8048 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8848 - port_range_max: 8848 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8448 - port_range_max: 8448 - direction: egress remote_ip_prefix: 0.0.0.0/0 protocol: tcp @@ -268,7 +266,10 @@ resources: $APP_NAME: { get_param: app_name } $HTTP_PROXY: { get_param: http_proxy } $HTTPS_PROXY: { get_param: https_proxy } - $REGISTRY: { get_param: registry } + $REGISTRY_HOST: { get_param: registry_host } + $REGISTRY_PORT: { get_param: registry_port } + $REGISTRY_USER: { get_param: registry_user } + $REGISTRY_PWD: { get_param: registry_pwd } $MAIL_ADDRESS: { get_param: mail_address } $MAIL_HOST: { get_param: mail_host } $MAIL_PASSWORD: { get_param: mail_password } diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index e11d453456..438a1204cf 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -17,14 +17,17 @@ FTP_PROXY="" GOPHER_PROXY="" SOCKS_PROXY="" SOCKS5_SERVER="" -NO_PROXY="localhost, 127.0.0.1, $REGISTRY" +NO_PROXY="localhost, 127.0.0.1, $REGISTRY_HOST" EOF +# TODO: +# If $REGISTRY_HOST empty, docker hub is default. +# Remove insecure sudo cat > /etc/sysconfig/docker < /var/web_cert fi @@ -81,7 +84,6 @@ DB_IMAGE=$REGISTRY/$DB_IMAGE_NAME DB=/db/database APP_PATH=/oscm/app/ BES_PATH=/oscm/bes/ -MI_PATH=/oscm/master-indexer CLIENT_CRT_PATH=/oscm/client-crt/ DB_VOLUME=dbvolume @@ -91,7 +93,6 @@ APP_VOLUME=appvolume mkdir -p $DB mkdir -p $APP_PATH mkdir -p $BES_PATH -mkdir -p $MI_PATH # Start docker service systemctl start docker.service @@ -102,10 +103,9 @@ docker rm -f $APP_VOLUME $BES_VOLUME $DB_VOLUME db web app >/dev/null 2>&1 || tr # Create volume container for database docker create -v $DB:/var/lib/pgsql/database --name $DB_VOLUME $DB_IMAGE /bin/true -# create volume container for bes and master-indexer +# create volume container for bes and app docker create \ -v $BES_PATH:/opt/glassfish3/glassfish/domains/bes-domain/ \ - -v $MI_PATH:/opt/glassfish3/glassfish/domains/master-indexer-domain/ \ -v $APP_PATH:/opt/glassfish3/glassfish/domains/app-domain/ \ -v $CLIENT_CRT_PATH:/tmp/client-crt \ --name $BES_VOLUME $OSCM_IMAGE /bin/true @@ -135,8 +135,6 @@ services: ports: - "8080:8080" - "8081:8081" - - "8048:8048" - - "8448:8448" volumes_from: - container:$BES_VOLUME:rw $APP_HOSTNAME: @@ -150,7 +148,6 @@ services: ports: - "8880:8880" - "8881:8881" - - "8848:8848" volumes_from: - container:$APP_VOLUME:rw $DB_HOSTNAME: @@ -164,7 +161,8 @@ EOF echo "# updated at $(date)" >> $OSCM_DIR/docker-compose.yml fi -# OSCM start script +# OSCM start script +# TODO: Add docker login -u="$REGISTRY_USER" -p="REGISTRY_PWD" if $REGISTRY_USER not empty. if [ ! -x $OSCM_DIR/start-oscm.sh ]; then cat > $OSCM_DIR/start-oscm.sh < "true", - "data-showit-target" => "#docker_container", + "data-showit-target" => "#dockerauth_container", "data-showit-direct" => "true" - #docker_container - = string_field %w(docker host) - = integer_field %w(docker port) - = boolean_field %w(docker auth), - "data-showit" => "true", - "data-showit-target" => "#dockerauth_container", - "data-showit-direct" => "true" - - #dockerauth_container - = string_field %w(docker user) - = password_field %w(docker password) + #dockerauth_container + = string_field %w(docker user) + = password_field %w(docker password) %fieldset %legend diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index ab9b97629c..56c0501afe 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -31,11 +31,11 @@ en: docker_settings: 'Docker Registry' docker: auth: 'Authentication Required' - use_registry: 'Install from Local Registry' host: 'Registry Host' port: 'Registry Port' user: 'User' password: 'Password' + docker_hint: 'The registry host and port can be leaved empty for docker hub' proxy_settings: 'Proxy Settings' proxy: auth: 'Authentication Required' From 3365d47d63f3f38256726ab6e8287a8ec202741d Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 20 Sep 2017 14:20:23 +0200 Subject: [PATCH 052/181] Add property for secure mailserver connection --- chef/cookbooks/oscm/files/default/application.yaml | 4 ++++ chef/cookbooks/oscm/files/default/user-data/write-config | 1 + chef/cookbooks/oscm/recipes/server.rb | 8 +++++--- chef/data_bags/crowbar/template-oscm.json | 1 + chef/data_bags/crowbar/template-oscm.schema | 1 + .../app/views/barclamp/oscm/_edit_attributes.html.haml | 1 + crowbar_framework/config/locales/oscm/en.yml | 1 + 7 files changed, 14 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index edef016596..7c0e5bc097 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -85,6 +85,9 @@ parameters: mail_auth: default: false type: string + mail_tls: + default: false + type: string keystone_user_pwd: default: oscm type: string @@ -276,6 +279,7 @@ resources: $MAIL_USER: { get_param: mail_user } $MAIL_PORT: { get_param: mail_port } $MAIL_AUTH: { get_param: mail_auth } + $MAIL_TLS: { get_param: mail_tls } $KEYSTONE_USER_PWD: { get_param: keystone_user_pwd } $KEYSTONE_USER_NAME: { get_param: keystone_user_name } $KEYSTONE_URL: { get_param: keystone_url } diff --git a/chef/cookbooks/oscm/files/default/user-data/write-config b/chef/cookbooks/oscm/files/default/user-data/write-config index e8084215b4..9761b9f6a9 100644 --- a/chef/cookbooks/oscm/files/default/user-data/write-config +++ b/chef/cookbooks/oscm/files/default/user-data/write-config @@ -26,6 +26,7 @@ MAIL_PASSWORD=$MAIL_PASSWORD MAIL_USER=$MAIL_USER MAIL_PORT=$MAIL_PORT MAIL_AUTH=$MAIL_AUTH +MAIL_TLS=$MAIL_TLS KEYSTONE_USER_PWD=$KEYSTONE_USER_PWD KEYSTONE_USER_NAME=$KEYSTONE_USER_NAME KEYSTONE_URL=$KEYSTONE_URL diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 8624189406..585abfd6f8 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -42,8 +42,9 @@ oscm_proxy_httpport = node[:oscm][:proxy][:http_port] oscm_proxy_httpshost = node[:oscm][:proxy][:https_host] oscm_proxy_httpsport = node[:oscm][:proxy][:https_port] -oscm_mail_host = node[:oscm][:mail][:host] -oscm_mail_port = node[:oscm][:mail][:port] +oscm_mail_host = node[:oscm][:mail][:host] +oscm_mail_port = node[:oscm][:mail][:port] +oscm_mail_tls = node[:oscm][:mail][:tls] oscm_mail_from = node[:oscm][:mail][:from] oscm_mail_auth = node[:oscm][:mail][:auth] oscm_mail_user = node[:oscm][:mail][:user] @@ -236,7 +237,8 @@ ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} fi openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ - --parameter "mail_host=#{oscm_mail_host}" --parameter "mail_port=#{oscm_mail_port}" --parameter "mail_address=#{oscm_mail_from}" --parameter "mail_auth=#{oscm_mail_auth}" --parameter "mail_user=#{oscm_mail_user}" --parameter "mail_password=#{oscm_mail_pwd}"\ + --parameter "mail_host=#{oscm_mail_host}" --parameter "mail_port=#{oscm_mail_port}" --parameter "mail_tls=#{oscm_mail_tls}" --parameter "mail_address=#{oscm_mail_from}"\ + --parameter "mail_auth=#{oscm_mail_auth}" --parameter "mail_user=#{oscm_mail_user}" --parameter "mail_password=#{oscm_mail_pwd}"\ --parameter "http_proxy=#{oscm_proxy_httphost}:#{oscm_proxy_httpport}" --parameter "https_proxy=#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}"\ --parameter "registry_host=#{oscm_docker_host}" --parameter "registry_port=#{oscm_docker_port}" --parameter "registry_user=#{oscm_docker_user}" --parameter "registry_port=#{oscm_docker_pwd}"\ --parameter "ssl_path=#{oscm_ssl_scp_path}"\ diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 785edc7d88..f898db6382 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -39,6 +39,7 @@ "auth": true, "host": "", "port": 25, + "tls": false, "user": "", "password": "", "from": "" diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 379ccc8dbc..dc69f1c2d7 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -75,6 +75,7 @@ "auth": { "type": "bool", "required": true }, "host": { "type": "str", "required": true }, "port": { "type": "int", "required": true }, + "tls": { "type": "bool", "required": true }, "user": { "type": "str", "required": true }, "password": { "type": "str", "required": true }, "from": { "type": "str", "required": true } diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 6b3b93e79b..53dea04a31 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -9,6 +9,7 @@ = string_field %w(mail host) = integer_field %w(mail port) + = boolean_field %w(mail tls) = string_field %w(mail from) = boolean_field %w(mail auth), diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 56c0501afe..4d0d78ff2a 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -25,6 +25,7 @@ en: auth: 'Authentication Required' host: 'Mail Host' port: 'SMTP Port' + tls: 'Enable TLS' user: 'User' password: 'Password' from: 'Oscm Email Address' From a9b98b376acdf1e253cab29460eddbcaf01d0c96 Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 22 Sep 2017 12:31:41 +0200 Subject: [PATCH 053/181] Refactor --- .../oscm/files/default/application.yaml | 130 +------------ .../files/default/user-data/deploy-oscmserver | 183 +++--------------- .../oscm/files/default/user-data/heat-config | 20 ++ .../oscm/files/default/user-data/oscm-config | 20 ++ chef/cookbooks/oscm/recipes/server.rb | 88 ++++++--- chef/data_bags/crowbar/template-oscm.json | 4 +- chef/data_bags/crowbar/template-oscm.schema | 4 +- 7 files changed, 147 insertions(+), 302 deletions(-) create mode 100644 chef/cookbooks/oscm/files/default/user-data/heat-config create mode 100644 chef/cookbooks/oscm/files/default/user-data/oscm-config diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 7c0e5bc097..998181bd63 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -28,99 +28,26 @@ parameters: type: string default: oscm.medium description: Nova flavor to use for servers - key_name: - type: string - default: oscm - description: SSH key to add to servers' /root/.ssh/authorized_keys app_volume_id: type: string description: The Cinder volume to attach to the application server db_volume_id: type: string description: The Cinder volume to attach to the database server - app_name: - default: oscm - description: The application name - type: string - http_proxy: - default: "" - type: string - description: Http proxy - https_proxy: - default: "" - type: string - description: Https proxy - registry_host: - default: "" - type: string - description: Docker registry host registry_port: default: "" type: string description: Docker registry port - registry_user: - default: "" - type: string - description: Docker registry user - registry_pwd: - default: "" - type: string - hidden: true - description: Docker registry password - mail_address: - default: "" - type: string - mail_host: - default: "" - type: string - mail_password: - default: "" - type: string - mail_user: - default: oscm - type: string mail_port: default: 25 type: string - mail_auth: - default: false - type: string - mail_tls: - default: false - type: string - keystone_user_pwd: - default: oscm - type: string - keystone_user_name: - default: oscm - type: string - keystone_url: - default: "" - type: string - keystone_tenant_id: - default: oscm - type: string - keystone_domain: - default: default - type: string - oscm_image_name: - default: oscm - type: string - app_image_name: - default: app - type: string - db_image_name: - default: db - type: string - ssl_path: - default: "" - type: string wait_condition_timeout: default: 1800 type: number ssh_cert: default: "" type: string + description: SSH key to add to servers' /root/.ssh/authorized_keys heat_host_cidr: default: "" type: string @@ -130,25 +57,19 @@ parameters: resources: - ssl_wait_handle: + inject_wait_handle: type: OS::Heat::WaitConditionHandle - ssl_wait_condition: + inject_wait_condition: type: OS::Heat::WaitCondition depends_on: appserver properties: - handle: { get_resource: ssl_wait_handle } + handle: { get_resource: inject_wait_handle } timeout: { get_param: wait_condition_timeout } db_password: type: OS::Heat::RandomString - keystore_password: - type: OS::Heat::RandomString - - as_admin_password: - type: OS::Heat::RandomString - ### Network infrastructure ### oscm_network: @@ -156,7 +77,6 @@ resources: properties: name: oscm - oscm_subnet: type: OS::Neutron::Subnet properties: @@ -197,16 +117,6 @@ resources: port_range_max: 80 - remote_ip_prefix: 0.0.0.0/0 protocol: icmp - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: { get_param: registry_port } - port_range_max: { get_param: registry_port } - - direction: egress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: { get_param: registry_port } - port_range_max: { get_param: registry_port } - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp @@ -227,6 +137,11 @@ resources: protocol: tcp port_range_min: 8881 port_range_max: 8881 + - direction: ergress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: { get_param: registry_port } + port_range_max: { get_param: registry_port } - direction: egress remote_ip_prefix: 0.0.0.0/0 protocol: tcp @@ -251,7 +166,7 @@ resources: group: ungrouped config: str_replace: - template: {get_file: user-data/write-config} + template: {get_file: user-data/heat-config} params: $APP_VOLUME_DEV: list_join: @@ -264,31 +179,6 @@ resources: - - "/dev/disk/by-id/virtio-" - { get_param: db_volume_id } $DB_PASSWORD: { get_attr: [ db_password, value ] } - $KEYSTORE_PASSWORD: { get_attr: [ keystore_password, value ] } - $AS_ADMIN_PASSWORD: { get_attr: [ as_admin_password, value ] } - $APP_NAME: { get_param: app_name } - $HTTP_PROXY: { get_param: http_proxy } - $HTTPS_PROXY: { get_param: https_proxy } - $REGISTRY_HOST: { get_param: registry_host } - $REGISTRY_PORT: { get_param: registry_port } - $REGISTRY_USER: { get_param: registry_user } - $REGISTRY_PWD: { get_param: registry_pwd } - $MAIL_ADDRESS: { get_param: mail_address } - $MAIL_HOST: { get_param: mail_host } - $MAIL_PASSWORD: { get_param: mail_password } - $MAIL_USER: { get_param: mail_user } - $MAIL_PORT: { get_param: mail_port } - $MAIL_AUTH: { get_param: mail_auth } - $MAIL_TLS: { get_param: mail_tls } - $KEYSTONE_USER_PWD: { get_param: keystone_user_pwd } - $KEYSTONE_USER_NAME: { get_param: keystone_user_name } - $KEYSTONE_URL: { get_param: keystone_url } - $KEYSTONE_TENANT_ID: { get_param: keystone_tenant_id } - $KEYSTONE_DOMAIN: { get_param: keystone_domain } - $OSCM_IMAGE_NAME: { get_param: oscm_image_name } - $APP_IMAGE_NAME: { get_param: app_image_name } - $DB_IMAGE_NAME: { get_param: db_image_name } - $SSL_PATH: { get_param: ssl_path } $WAIT_CURL: { get_attr: [ ssl_wait_handle, curl_cli ] } $SSH_CERT: { get_param: ssh_cert } diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 438a1204cf..dedf8338c6 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -5,50 +5,37 @@ set -e exec &> /var/log/setup-machine.log -source /etc/oscm-variables - -WEB_CERT_PATH=/var/web_cert - -sudo cat /etc/sysconfig/proxy < /etc/sysconfig/docker < /var/web_cert + # echo $SSL_CONFIG_PATH/somekey.pem > /var/web_cert fi +WAIT_CURL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" +eval $WAIT_CURL + +source ${OSCM_CONFIG} # For some reason the volume id is shortened to 27 characters when the device # link in /dev/disk/by-id/ is created. Account for this by truncating the @@ -82,137 +69,21 @@ OSCM_IMAGE=$REGISTRY/$OSCM_IMAGE_NAME APP_IMAGE=$REGISTRY/$APP_IMAGE_NAME DB_IMAGE=$REGISTRY/$DB_IMAGE_NAME DB=/db/database -APP_PATH=/oscm/app/ -BES_PATH=/oscm/bes/ +APP_PATH=/oscm/oscm-app/ +OSCM_PATH=/oscm/oscm-core/ CLIENT_CRT_PATH=/oscm/client-crt/ -DB_VOLUME=dbvolume -BES_VOLUME=besvolume -APP_VOLUME=appvolume +DB_VOLUME=oscm-db-volume +OSCM_VOLUME=oscm-core-volume +APP_VOLUME=oscm-app-volume mkdir -p $DB mkdir -p $APP_PATH -mkdir -p $BES_PATH - -# Start docker service -systemctl start docker.service - -# Delete all containers before creating if they are already created before -docker rm -f $APP_VOLUME $BES_VOLUME $DB_VOLUME db web app >/dev/null 2>&1 || true - -# Create volume container for database -docker create -v $DB:/var/lib/pgsql/database --name $DB_VOLUME $DB_IMAGE /bin/true - -# create volume container for bes and app -docker create \ - -v $BES_PATH:/opt/glassfish3/glassfish/domains/bes-domain/ \ - -v $APP_PATH:/opt/glassfish3/glassfish/domains/app-domain/ \ - -v $CLIENT_CRT_PATH:/tmp/client-crt \ - --name $BES_VOLUME $OSCM_IMAGE /bin/true - -# create volume container for app -docker create \ - -v $APP_PATH:/opt/glassfish3/glassfish/domains/app-domain/ \ - -v $BES_PATH:/opt/glassfish3/glassfish/domains/bes-domain/ \ - -v $CLIENT_CRT_PATH:/tmp/client-crt \ - --name $APP_VOLUME $APP_IMAGE /bin/true +mkdir -p $OSCM_PATH OSCM_DIR=/opt/oscm mkdir -p $OSCM_DIR -# Docker compose file for the application deployment -if [ ! -x $OSCM_DIR/docker-compose.yml ]; then -cat > $OSCM_DIR/docker-compose.yml <> $OSCM_DIR/docker-compose.yml -fi - -# OSCM start script -# TODO: Add docker login -u="$REGISTRY_USER" -p="REGISTRY_PWD" if $REGISTRY_USER not empty. -if [ ! -x $OSCM_DIR/start-oscm.sh ]; then -cat > $OSCM_DIR/start-oscm.sh </dev/null; then - echo "starting docker service" - systemctl start docker.service -fi -if ! grep -qs '/db' /proc/mounts; then - mount -t ext4 $volume_dev_db /db -fi -if ! grep -qs '/oscm' /proc/mounts; then - mount -t ext4 $volume_dev_db /oscm -fi -echo "oscm is being started" -docker-compose -f $OSCM_DIR/docker-compose.yml up --force-recreate -EOF -echo "# updated at $(date)" >> $OSCM_DIR/start-oscm.sh -fi -chmod +x $OSCM_DIR/start-oscm.sh - -# OSCM Stop script -if [ ! -x $OSCM_DIR/stop-oscm.sh ]; then -cat > $OSCM_DIR/stop-oscm.sh <> $OSCM_DIR/stop-oscm.sh -fi -chmod +x $OSCM_DIR/stop-oscm.sh - -# OSCM systemd service -cat > /etc/systemd/system/oscm.service <> /etc/systemd/system/oscm.service - -systemctl enable oscm.service -systemctl start oscm.service - cat > /etc/motd < /etc/oscm/config/heatconfig <> /root/.ssh/authorized_keys \ No newline at end of file diff --git a/chef/cookbooks/oscm/files/default/user-data/oscm-config b/chef/cookbooks/oscm/files/default/user-data/oscm-config new file mode 100644 index 0000000000..6368ae0096 --- /dev/null +++ b/chef/cookbooks/oscm/files/default/user-data/oscm-config @@ -0,0 +1,20 @@ +#proxy +HTTP_PROXY=$HTTP_PROXY +HTTPS_PROXY=$HTTPS_PROXY +PROXY_USER=$PROXY_USER +PROXY_PWD=$PROXY_PWD +#mail +MAIL_HOST=$MAIL_HOST +MAIL_PORT=$MAIL_PORT +MAIL_TLS=$MAIL_TLS +MAIL_USER=$MAIL_USER +MAIL_PWD=$MAIL_PWD +MAIL_ADDRESS=$MAIL_ADDRESS +MAIL_AUTH=$MAIL_AUTH +#docker registry +REGISTRY_HOST=$REGISTRY_HOST +REGISTRY_PORT=$REGISTRY_PORT +REGISTRY_USER=$REGISTRY_USER +REGISTRY_PWD=$REGISTRY_PWD +#docker image tag +OSCM_IMAGES_TAG=$OSCM_IMAGES_TAG diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 585abfd6f8..c3864197cf 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -20,7 +20,6 @@ oscm_ssl_certfile = node[:oscm][:ssl][:certfile] oscm_ssl_keyfile = node[:oscm][:ssl][:keyfile] oscm_ssl_cacerts = node[:oscm][:ssl][:ca_certs] -oscm_ssl_scp_path = node[:oscm][:ssl][:scp_path] oscm_flavor_name = node[:oscm][:openstack][:flavor][:name] oscm_flavor_ram = node[:oscm][:openstack][:flavor][:ram] oscm_flavor_vcpus = node[:oscm][:openstack][:flavor][:vcpus] @@ -28,7 +27,8 @@ oscm_keypair_name = node[:oscm][:openstack][:keypair][:name] oscm_keypair_publickey = node[:oscm][:openstack][:keypair][:publickey] oscm_keypair_publickeyfile = node[:oscm][:openstack][:keypair][:publickeyfile] -oscm_heattemplate_path = node[:oscm][:openstack][:heattemplate_path] +oscm_install_path = "/etc/oscm/install/" +oscm_config_path = "/etc/oscm/config/" oscm_volumestack_name = node[:oscm][:openstack][:volume_stack][:stack_name] oscm_instancestack_name = node[:oscm][:openstack][:instance_stack][:stack_name] oscm_db_volume_size = node[:oscm][:openstack][:volume_stack][:db_volume_size] @@ -38,10 +38,13 @@ oscm_docker_port = node[:oscm][:docker][:port] oscm_docker_user = node[:oscm][:docker][:user] oscm_docker_pwd = node[:oscm][:docker][:password] +oscm_docker_tag = node[:oscm][:docker][:tag] oscm_proxy_httphost = node[:oscm][:proxy][:http_host] oscm_proxy_httpport = node[:oscm][:proxy][:http_port] oscm_proxy_httpshost = node[:oscm][:proxy][:https_host] oscm_proxy_httpsport = node[:oscm][:proxy][:https_port] +oscm_proxy_user = node[:oscm][:proxy][:user] +oscm_proxy_pwd = node[:oscm][:proxy][:password] oscm_mail_host = node[:oscm][:mail][:host] oscm_mail_port = node[:oscm][:mail][:port] oscm_mail_tls = node[:oscm][:mail][:tls] @@ -180,14 +183,14 @@ }) end -directory "#{oscm_heattemplate_path}" do +directory "#{oscm_install_path}" do owner oscm_group group oscm_group mode 0755 recursive true end -cookbook_file "#{oscm_heattemplate_path}/volumes.yaml" do +cookbook_file "#{oscm_install_path}/volumes.yaml" do source "volumes.yaml" owner oscm_group group oscm_group @@ -195,7 +198,7 @@ action :create end -cookbook_file "#{oscm_heattemplate_path}/application.yaml" do +cookbook_file "#{oscm_install_path}/application.yaml" do source "application.yaml" owner oscm_group group oscm_group @@ -203,22 +206,30 @@ action :create end -directory "#{oscm_heattemplate_path}/user-data" do +directory "#{oscm_install_path}/user-data" do owner oscm_group group oscm_group mode 0755 recursive true end -cookbook_file "#{oscm_heattemplate_path}/user-data/write-config" do - source "user-data/write-config" +cookbook_file "#{oscm_install_path}/user-data/heat-config" do + source "user-data/heat-config" owner oscm_group group oscm_group mode 0755 action :create end -cookbook_file "#{oscm_heattemplate_path}/user-data/deploy-oscmserver" do +cookbook_file "#{oscm_install_path}/user-data/oscm-config" do + source "user-data/oscm-config" + owner oscm_group + group oscm_group + mode 0755 + action :create +end + +cookbook_file "#{oscm_install_path}/user-data/deploy-oscmserver" do source "user-data/deploy-oscmserver" owner oscm_group group oscm_group @@ -226,9 +237,38 @@ action :create end +oscm_mail_host = node[:oscm][:mail][:host] +oscm_mail_port = node[:oscm][:mail][:port] +oscm_mail_tls = node[:oscm][:mail][:tls] +oscm_mail_from = node[:oscm][:mail][:from] +oscm_mail_auth = node[:oscm][:mail][:auth] +oscm_mail_user = node[:oscm][:mail][:user] +oscm_mail_pwd = node[:oscm][:mail][:password] + +bash "create oscm stacks" do + code <<-EOH + sed -i 'g/$HTTP_PROXY/#{oscm_proxy_httphost}:#{oscm_proxy_httpport}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$HTTPS_PROXY/#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$PROXY_USER/#{oscm_proxy_user}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$PROXY_PWD/#{oscm_proxy_pwd}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$MAIL_HOST/#{oscm_mail_host}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$MAIL_PORT/#{oscm_mail_port}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$MAIL_TLS/#{oscm_mail_tls}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$MAIL_USER/#{oscm_mail_user}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$MAIL_PWD/#{oscm_mail_pwd}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$REGISTRY_HOST/#{oscm_docker_host}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$REGISTRY_PORT/#{oscm_docker_port}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$REGISTRY_USER/#{oscm_docker_user}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$REGISTRY_PWD/#{oscm_docker_pwd}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 'g/$OSCM_IMAGES_TAG/#{oscm_docker_tag}/g' #{oscm_install_path}/user-data/oscm-config + EOH + environment ({ + }) +end + bash "create oscm stacks" do code <<-EOH - openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_heattemplate_path}/volumes.yaml --wait #{oscm_volumestack_name} &> /dev/null || true + openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_install_path}/volumes.yaml --wait #{oscm_volumestack_name} &> /dev/null || true app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') mkdir -p "$(dirname "#{oscm_keypair_crowbar_sshkey}")" @@ -236,22 +276,26 @@ then ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} fi - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}" --parameter "key_name=#{oscm_keypair_name}" --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ - --parameter "mail_host=#{oscm_mail_host}" --parameter "mail_port=#{oscm_mail_port}" --parameter "mail_tls=#{oscm_mail_tls}" --parameter "mail_address=#{oscm_mail_from}"\ - --parameter "mail_auth=#{oscm_mail_auth}" --parameter "mail_user=#{oscm_mail_user}" --parameter "mail_password=#{oscm_mail_pwd}"\ - --parameter "http_proxy=#{oscm_proxy_httphost}:#{oscm_proxy_httpport}" --parameter "https_proxy=#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}"\ - --parameter "registry_host=#{oscm_docker_host}" --parameter "registry_port=#{oscm_docker_port}" --parameter "registry_user=#{oscm_docker_user}" --parameter "registry_port=#{oscm_docker_pwd}"\ - --parameter "ssl_path=#{oscm_ssl_scp_path}"\ + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ + --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ + --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ --parameter-file "ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub"\ - -t #{oscm_heattemplate_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true + -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true ip_appserver=$(openstack stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*') ssh-keygen -R ${ip_appserver} -f /root/.ssh/known_hosts - ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_ssl_scp_path}" || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_ssl_scp_path} || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_keyfile} ${ip_appserver}:#{oscm_ssl_scp_path} || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_ssl_scp_path} || true - ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_ssl_scp_path}/scp_finished" + ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}" || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/oscm-config ${ip_appserver}:#{oscm_config_path} || true + ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/finished" + if [ -f #{oscm_ssl_certfile} ]; then + ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}/ssl/" || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_keyfile} ${ip_appserver}:#{oscm_config_path}/ssl || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_config_path}/ssl || true + ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/ssl/finished" + fi + EOH environment ({ "OS_USERNAME" => oscm_user, diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index f898db6382..ca7e69a427 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -25,7 +25,6 @@ "publickey": "", "publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem" }, - "heattemplate_path": "/etc/oscm/heat/", "volume_stack": { "stack_name": "oscm-volumes", "db_volume_size": 1, @@ -49,7 +48,8 @@ "port": 0, "auth": false, "user": "", - "password": "" + "password": "", + "tag": "17.5" }, "proxy": { "use_proxy": false, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index dc69f1c2d7..2a38c9c421 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -49,7 +49,6 @@ "publickeyfile": { "type": "str", "required": true } } }, - "heattemplate_path": { "type": "str", "required": true }, "volume_stack" : { "type" : "map", "required" : true, @@ -89,7 +88,8 @@ "port": { "type": "int", "required": true }, "auth": { "type": "bool", "required": true }, "user": { "type": "str", "required": true }, - "password": { "type": "str", "required": true } + "password": { "type": "str", "required": true }, + "tag": { "type": "str", "required": true } } }, "proxy" : { From 1a863bf78afa7a10016c6502ee0d158cfec83b75 Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 22 Sep 2017 16:00:14 +0200 Subject: [PATCH 054/181] Remove unnecessary properties --- .../oscm/files/default/user-data/write-config | 42 ------------------- chef/cookbooks/oscm/recipes/server.rb | 32 +++++++------- chef/data_bags/crowbar/template-oscm.json | 1 - chef/data_bags/crowbar/template-oscm.schema | 1 - 4 files changed, 16 insertions(+), 60 deletions(-) delete mode 100644 chef/cookbooks/oscm/files/default/user-data/write-config diff --git a/chef/cookbooks/oscm/files/default/user-data/write-config b/chef/cookbooks/oscm/files/default/user-data/write-config deleted file mode 100644 index 9761b9f6a9..0000000000 --- a/chef/cookbooks/oscm/files/default/user-data/write-config +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh - -# Writes the configuration parameters passed in by Heat/cloud-init to /etc/heatconfig. - -touch /etc/oscm-variables -chmod 400 /etc/oscm-variables - -cat > /etc/oscm-variables <> /root/.ssh/authorized_keys \ No newline at end of file diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index c3864197cf..8177e42ba8 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -245,22 +245,22 @@ oscm_mail_user = node[:oscm][:mail][:user] oscm_mail_pwd = node[:oscm][:mail][:password] -bash "create oscm stacks" do +bash "create oscm configuration" do code <<-EOH - sed -i 'g/$HTTP_PROXY/#{oscm_proxy_httphost}:#{oscm_proxy_httpport}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$HTTPS_PROXY/#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$PROXY_USER/#{oscm_proxy_user}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$PROXY_PWD/#{oscm_proxy_pwd}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$MAIL_HOST/#{oscm_mail_host}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$MAIL_PORT/#{oscm_mail_port}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$MAIL_TLS/#{oscm_mail_tls}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$MAIL_USER/#{oscm_mail_user}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$MAIL_PWD/#{oscm_mail_pwd}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$REGISTRY_HOST/#{oscm_docker_host}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$REGISTRY_PORT/#{oscm_docker_port}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$REGISTRY_USER/#{oscm_docker_user}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$REGISTRY_PWD/#{oscm_docker_pwd}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 'g/$OSCM_IMAGES_TAG/#{oscm_docker_tag}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$HTTP_PROXY/#{oscm_proxy_httphost}:#{oscm_proxy_httpport}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$HTTPS_PROXY/#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$PROXY_USER/#{oscm_proxy_user}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$PROXY_PWD/#{oscm_proxy_pwd}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$MAIL_HOST/#{oscm_mail_host}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$MAIL_PORT/#{oscm_mail_port}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$MAIL_TLS/#{oscm_mail_tls}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$MAIL_USER/#{oscm_mail_user}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$MAIL_PWD/#{oscm_mail_pwd}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$REGISTRY_HOST/#{oscm_docker_host}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$REGISTRY_PORT/#{oscm_docker_port}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$REGISTRY_USER/#{oscm_docker_user}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$REGISTRY_PWD/#{oscm_docker_pwd}/g' #{oscm_install_path}/user-data/oscm-config + sed -i 's/$OSCM_IMAGES_TAG/#{oscm_docker_tag}/g' #{oscm_install_path}/user-data/oscm-config EOH environment ({ }) @@ -295,7 +295,7 @@ scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_config_path}/ssl || true ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/ssl/finished" fi - + EOH environment ({ "OS_USERNAME" => oscm_user, diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index ca7e69a427..957156bf2f 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -63,7 +63,6 @@ "password": "" }, "ssl": { - "scp_path": "/etc/oscm/ssl/", "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", "keyfile": "/etc/oscm/ssl/private/signing_key.pem", "generate_certs": false, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 2a38c9c421..5d6ad18c2b 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -111,7 +111,6 @@ "type": "map", "required": true, "mapping": { - "scp_path": { "type" : "str", "required" : true }, "certfile": { "type" : "str", "required" : true }, "keyfile": { "type" : "str", "required" : true }, "generate_certs": { "type" : "bool", "required" : true }, From 4060d296dcc9bb8fbe41baba35f91a75656a53c9 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 25 Sep 2017 09:53:44 +0200 Subject: [PATCH 055/181] Split the oscm resources creation --- chef/cookbooks/oscm/recipes/server.rb | 51 ++++++++++++++++++--------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 8177e42ba8..62064336a6 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -237,14 +237,6 @@ action :create end -oscm_mail_host = node[:oscm][:mail][:host] -oscm_mail_port = node[:oscm][:mail][:port] -oscm_mail_tls = node[:oscm][:mail][:tls] -oscm_mail_from = node[:oscm][:mail][:from] -oscm_mail_auth = node[:oscm][:mail][:auth] -oscm_mail_user = node[:oscm][:mail][:user] -oscm_mail_pwd = node[:oscm][:mail][:password] - bash "create oscm configuration" do code <<-EOH sed -i 's/$HTTP_PROXY/#{oscm_proxy_httphost}:#{oscm_proxy_httpport}/g' #{oscm_install_path}/user-data/oscm-config @@ -266,22 +258,51 @@ }) end -bash "create oscm stacks" do +bash "create oscm volume stack" do code <<-EOH openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_install_path}/volumes.yaml --wait #{oscm_volumestack_name} &> /dev/null || true + EOH + environment ({ + "OS_USERNAME" => oscm_user, + "OS_PASSWORD" => oscm_password, + "OS_TENANT_NAME" => oscm_tenant, + "OS_AUTH_URL" => keystone_settings["internal_auth_url"], + "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], + "OS_USER_DOMAIN_NAME" => "Default", + "OS_PROJECT_DOMAIN_NAME" => "Default" + }) +end + +bash "create oscm instance stack" do + code <<-EOH app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - mkdir -p "$(dirname "#{oscm_keypair_crowbar_sshkey}")" - if [ ! -f #{oscm_keypair_crowbar_sshkey} ]; - then - ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} - fi openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ --parameter-file "ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub"\ -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true + EOH + environment ({ + "OS_USERNAME" => oscm_user, + "OS_PASSWORD" => oscm_password, + "OS_TENANT_NAME" => oscm_tenant, + "OS_AUTH_URL" => keystone_settings["internal_auth_url"], + "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], + "OS_USER_DOMAIN_NAME" => "Default", + "OS_PROJECT_DOMAIN_NAME" => "Default" + }) +end + + +bash "inject oscm certificates" do + code <<-EOH + mkdir -p "$(dirname "#{oscm_keypair_crowbar_sshkey}")" + if [ ! -f #{oscm_keypair_crowbar_sshkey} ]; + then + ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} + fi ip_appserver=$(openstack stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*') ssh-keygen -R ${ip_appserver} -f /root/.ssh/known_hosts ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}" || true @@ -295,7 +316,6 @@ scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_config_path}/ssl || true ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/ssl/finished" fi - EOH environment ({ "OS_USERNAME" => oscm_user, @@ -307,4 +327,3 @@ "OS_PROJECT_DOMAIN_NAME" => "Default" }) end - From 8f7b25ecf9a18a4bcf216413138891ada19b6fde Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 25 Sep 2017 10:45:18 +0200 Subject: [PATCH 056/181] check if cert exists --- chef/cookbooks/oscm/recipes/server.rb | 25 ++++++++++++++------- chef/data_bags/crowbar/template-oscm.json | 3 +-- chef/data_bags/crowbar/template-oscm.schema | 3 +-- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 62064336a6..b59dac1882 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -26,7 +26,7 @@ oscm_flavor_disk = node[:oscm][:openstack][:flavor][:disk] oscm_keypair_name = node[:oscm][:openstack][:keypair][:name] oscm_keypair_publickey = node[:oscm][:openstack][:keypair][:publickey] -oscm_keypair_publickeyfile = node[:oscm][:openstack][:keypair][:publickeyfile] +oscm_keypair_publickeyfile = "/etc/oscm/install/openstack_keypair_public.pem" oscm_install_path = "/etc/oscm/install/" oscm_config_path = "/etc/oscm/config/" oscm_volumestack_name = node[:oscm][:openstack][:volume_stack][:stack_name] @@ -52,7 +52,7 @@ oscm_mail_auth = node[:oscm][:mail][:auth] oscm_mail_user = node[:oscm][:mail][:user] oscm_mail_pwd = node[:oscm][:mail][:password] -oscm_keypair_crowbar_sshkey = "/etc/oscm/ssh/oscm_ssh.key" +oscm_keypair_crowbar_sshkey = "/etc/oscm/install/oscm_ssh.key" oscm_group = "root" heat_node = node_search_with_cache("roles:heat-server").first @@ -277,12 +277,21 @@ code <<-EOH app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ - --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ - --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ - --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ - --parameter-file "ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub"\ - -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true + if [ -f #{oscm_keypair_crowbar_sshkey}.pub ]; + then + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ + --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ + --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ + --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ + --parameter-file "ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub"\ + -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true + else + openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ + --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ + --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ + --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ + -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true + fi EOH environment ({ "OS_USERNAME" => oscm_user, diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 957156bf2f..39e23504ab 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -22,8 +22,7 @@ }, "keypair": { "name": "oscm", - "publickey": "", - "publickeyfile": "/etc/oscm/ssl/openstack/keypair_publickey.pem" + "publickey": "" }, "volume_stack": { "stack_name": "oscm-volumes", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 5d6ad18c2b..483b9731d5 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -45,8 +45,7 @@ "required" : true, "mapping": { "name": { "type": "str", "required": true }, - "publickey": { "type": "str", "required": true }, - "publickeyfile": { "type": "str", "required": true } + "publickey": { "type": "str", "required": true } } }, "volume_stack" : { From 72c61840d1741caf2f775ccc4565351a90fc6df9 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 25 Sep 2017 11:06:16 +0200 Subject: [PATCH 057/181] Fix param name in heat template --- chef/cookbooks/oscm/files/default/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 998181bd63..2a37fec63c 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -179,7 +179,7 @@ resources: - - "/dev/disk/by-id/virtio-" - { get_param: db_volume_id } $DB_PASSWORD: { get_attr: [ db_password, value ] } - $WAIT_CURL: { get_attr: [ ssl_wait_handle, curl_cli ] } + $WAIT_CURL: { get_attr: [ inject_wait_handle, curl_cli ] } $SSH_CERT: { get_param: ssh_cert } # User data payload for appserver From 207b050144c10482bf77101d8b179c07be35de74 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 25 Sep 2017 11:31:16 +0200 Subject: [PATCH 058/181] Correct typo --- chef/cookbooks/oscm/files/default/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 2a37fec63c..a8d8465e8b 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -137,7 +137,7 @@ resources: protocol: tcp port_range_min: 8881 port_range_max: 8881 - - direction: ergress + - direction: egress remote_ip_prefix: 0.0.0.0/0 protocol: tcp port_range_min: { get_param: registry_port } From 0f520ef9bb07321479ec86415c28be649fad1602 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 25 Sep 2017 11:40:27 +0200 Subject: [PATCH 059/181] Add key_name param --- chef/cookbooks/oscm/files/default/application.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index a8d8465e8b..e2bf4c54c8 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -22,12 +22,16 @@ parameters: description: Network to draw Floating IPs from image: type: string - default: sles12-docker + default: sles12-sp1 description: Glance image to use for servers flavor: type: string default: oscm.medium description: Nova flavor to use for servers + key_name: + type: string + default: oscm + description: Keypair name app_volume_id: type: string description: The Cinder volume to attach to the application server From bf8c4627b6d4001d9f8cc3ce3c4fcfe7f3f5db1a Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 25 Sep 2017 16:09:31 +0200 Subject: [PATCH 060/181] Change setting --- .../oscm/files/default/application.yaml | 25 ------------------- .../files/default/user-data/deploy-oscmserver | 4 +-- .../oscm/files/default/user-data/heat-config | 9 ++----- .../oscm/files/default/user-data/oscm-config | 5 ++++ chef/data_bags/crowbar/template-oscm.json | 6 ++--- 5 files changed, 12 insertions(+), 37 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index e2bf4c54c8..3f694f7a74 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -45,9 +45,6 @@ parameters: mail_port: default: 25 type: string - wait_condition_timeout: - default: 1800 - type: number ssh_cert: default: "" type: string @@ -61,16 +58,6 @@ parameters: resources: - inject_wait_handle: - type: OS::Heat::WaitConditionHandle - - inject_wait_condition: - type: OS::Heat::WaitCondition - depends_on: appserver - properties: - handle: { get_resource: inject_wait_handle } - timeout: { get_param: wait_condition_timeout } - db_password: type: OS::Heat::RandomString @@ -183,18 +170,8 @@ resources: - - "/dev/disk/by-id/virtio-" - { get_param: db_volume_id } $DB_PASSWORD: { get_attr: [ db_password, value ] } - $WAIT_CURL: { get_attr: [ inject_wait_handle, curl_cli ] } $SSH_CERT: { get_param: ssh_cert } - # User data payload for appserver - userdata_appserver: - type: OS::Heat::MultipartMime - properties: - parts: - - config: {get_resource: user_data_params} - - config: {get_file: user-data/deploy-oscmserver} - - ### Servers ### appserver: @@ -207,8 +184,6 @@ resources: key_name: { get_param: key_name } networks: - port: { get_resource: port_appserver } - user_data_format: RAW - user_data: { get_resource: userdata_appserver } port_appserver: type: OS::Neutron::Port diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index dedf8338c6..1392af219c 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -7,9 +7,9 @@ exec &> /var/log/setup-machine.log CONFIG_PATH=/etc/oscm/config SSL_CONFIG_PATH=${CONFIG_PATH}/ssl -OSCM_CONFIG=${CONFIG_PATH}/oscmconfig +OSCM_CONFIG=${CONFIG_PATH}/oscm-config -source ${CONFIG_PATH}/heatconfig +source ${CONFIG_PATH}/heat-config if [ ${CONFIG_PATH} ]; then diff --git a/chef/cookbooks/oscm/files/default/user-data/heat-config b/chef/cookbooks/oscm/files/default/user-data/heat-config index 9d3ea26cb5..8059a847e4 100644 --- a/chef/cookbooks/oscm/files/default/user-data/heat-config +++ b/chef/cookbooks/oscm/files/default/user-data/heat-config @@ -2,18 +2,13 @@ # Writes the configuration parameters passed in by Heat/cloud-init to /etc/heatconfig. -touch /etc/oscm/config/heatconfig -chmod 400 /etc/oscm/config/heatconfig +touch /etc/oscm/config/heat-config +chmod 400 /etc/oscm/config/heat-config cat > /etc/oscm/config/heatconfig < Date: Tue, 26 Sep 2017 08:34:23 +0200 Subject: [PATCH 061/181] Remove Heat endpoint --- .../oscm/files/default/application.yaml | 17 ----------------- chef/cookbooks/oscm/recipes/server.rb | 6 ------ 2 files changed, 23 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 3f694f7a74..11086ca028 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -49,12 +49,6 @@ parameters: default: "" type: string description: SSH key to add to servers' /root/.ssh/authorized_keys - heat_host_cidr: - default: "" - type: string - heat_port: - default: 8004 - type: number resources: @@ -138,17 +132,6 @@ resources: protocol: tcp port_range_min: { get_param: mail_port } port_range_max: { get_param: mail_port } - - direction: ingress - remote_ip_prefix: { get_param: heat_host_cidr } - protocol: tcp - port_range_min: { get_param: heat_port } - port_range_max: { get_param: heat_port } - - direction: egress - remote_ip_prefix: { get_param: heat_host_cidr } - protocol: tcp - port_range_min: { get_param: heat_port } - port_range_max: { get_param: heat_port } - # Parameters that will be available to both user data scripts user_data_params: diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index b59dac1882..1a870e77df 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -55,10 +55,6 @@ oscm_keypair_crowbar_sshkey = "/etc/oscm/install/oscm_ssh.key" oscm_group = "root" -heat_node = node_search_with_cache("roles:heat-server").first -heat_public_host = CrowbarHelper.get_host_for_public_url(heat_node, false) -heat_port = heat_node[:heat][:api][:port] - keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) register_auth_hash = { @@ -282,14 +278,12 @@ openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ - --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ --parameter-file "ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub"\ -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true else openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ - --parameter "heat_host_cidr=#{heat_public_host}/32" --parameter "heat_port=#{heat_port}"\ -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true fi EOH From 6df74b7c0bd829351a9a3fb732b5d25dfe885730 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 26 Sep 2017 09:04:50 +0200 Subject: [PATCH 062/181] Add script for writing Heat output --- chef/cookbooks/oscm/files/default/application.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 11086ca028..e3aabf3a50 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -155,6 +155,13 @@ resources: $DB_PASSWORD: { get_attr: [ db_password, value ] } $SSH_CERT: { get_param: ssh_cert } + # User data payload for appserver + userdata_appserver: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: user_data_params} + ### Servers ### appserver: @@ -167,7 +174,9 @@ resources: key_name: { get_param: key_name } networks: - port: { get_resource: port_appserver } - + user_data_format: RAW + user_data: { get_resource: userdata_appserver } + port_appserver: type: OS::Neutron::Port properties: From a193c250a8156c31d35508887c24abf5d2b6fced Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 26 Sep 2017 09:14:44 +0200 Subject: [PATCH 063/181] Add oscm deployment script --- chef/cookbooks/oscm/recipes/server.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 1a870e77df..e2fd136555 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -310,6 +310,7 @@ ssh-keygen -R ${ip_appserver} -f /root/.ssh/known_hosts ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}" || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/oscm-config ${ip_appserver}:#{oscm_config_path} || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/deploy-oscmserver ${ip_appserver}:#{oscm_config_path} || true ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/finished" if [ -f #{oscm_ssl_certfile} ]; then ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}/ssl/" || true From 6a98b8b88abd36b61d9f99aa27f94966009cac5d Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 26 Sep 2017 10:43:31 +0200 Subject: [PATCH 064/181] Add dir creation --- chef/cookbooks/oscm/files/default/user-data/heat-config | 2 +- chef/cookbooks/oscm/recipes/server.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/heat-config b/chef/cookbooks/oscm/files/default/user-data/heat-config index 8059a847e4..fe65fd577a 100644 --- a/chef/cookbooks/oscm/files/default/user-data/heat-config +++ b/chef/cookbooks/oscm/files/default/user-data/heat-config @@ -1,7 +1,7 @@ #!/bin/sh # Writes the configuration parameters passed in by Heat/cloud-init to /etc/heatconfig. - +mkdir -p /etc/oscm/config touch /etc/oscm/config/heat-config chmod 400 /etc/oscm/config/heat-config diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index e2fd136555..2bd62351fa 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -309,8 +309,8 @@ ip_appserver=$(openstack stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*') ssh-keygen -R ${ip_appserver} -f /root/.ssh/known_hosts ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}" || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/oscm-config ${ip_appserver}:#{oscm_config_path} || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/deploy-oscmserver ${ip_appserver}:#{oscm_config_path} || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/oscm-config ${ip_appserver}:#{oscm_config_path} || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/deploy-oscmserver ${ip_appserver}:#{oscm_config_path} || true ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/finished" if [ -f #{oscm_ssl_certfile} ]; then ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}/ssl/" || true From d3ca226d61e40b1934151ead739800e1fdc5ec10 Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 27 Sep 2017 15:47:53 +0200 Subject: [PATCH 065/181] Refactor oscm recipe --- .../oscm/files/default/user-data/heat-config | 2 +- chef/cookbooks/oscm/recipes/server.rb | 109 ++++++++---------- chef/data_bags/crowbar/template-oscm.json | 2 + chef/data_bags/crowbar/template-oscm.schema | 2 + .../barclamp/oscm/_edit_attributes.html.haml | 13 ++- crowbar_framework/config/locales/oscm/en.yml | 2 +- 6 files changed, 66 insertions(+), 64 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/heat-config b/chef/cookbooks/oscm/files/default/user-data/heat-config index fe65fd577a..9fe0486068 100644 --- a/chef/cookbooks/oscm/files/default/user-data/heat-config +++ b/chef/cookbooks/oscm/files/default/user-data/heat-config @@ -5,7 +5,7 @@ mkdir -p /etc/oscm/config touch /etc/oscm/config/heat-config chmod 400 /etc/oscm/config/heat-config -cat > /etc/oscm/config/heatconfig < /etc/oscm/config/heat-config < /dev/null || true tenant_id=$(openstack project show -f shell #{oscm_tenant} | grep -Po '(?<=^id=\")[^\"]*') nova flavor-access-add #{oscm_flavor_name} $tenant_id &> /dev/null || true EOH @@ -158,25 +179,22 @@ }) end -bash "add oscm keypair" do +bash "create_oscm_keypair_file" do code <<-EOH publickey="#{oscm_keypair_publickey}" - if ! -z "${publickey// }" - then - mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null - echo "${publickey}" > "#{oscm_keypair_publickeyfile}" - nova keypair-add #{oscm_keypair_name} --pub-key #{oscm_keypair_publickeyfile} &> /dev/null || exit 0 - fi + mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null + echo "${publickey}" > "#{oscm_keypair_publickeyfile}" EOH - environment ({ - "OS_USERNAME" => oscm_user, - "OS_PASSWORD" => oscm_password, - "OS_TENANT_NAME" => oscm_tenant, - "OS_AUTH_URL" => keystone_settings["internal_auth_url"], - "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], - "OS_USER_DOMAIN_NAME" => "Default", - "OS_PROJECT_DOMAIN_NAME" => "Default" - }) +end + +execute "delete_oscm_keypair" do + command "#{openstack_cmd} #{openstack_args_nova} keypair delete #{oscm_keypair_name}" + only_if "#{openstack_cmd} #{openstack_args_nova} keypair list | grep -q #{oscm_keypair_name}" +end + +execute "create_oscm_keypair" do + command "#{openstack_cmd} #{openstack_args_nova} keypair create #{oscm_keypair_name} --public-key #{oscm_keypair_publickeyfile}" + not_if "#{openstack_cmd} #{openstack_args_nova} keypair list | grep -q #{oscm_keypair_name}" end directory "#{oscm_install_path}" do @@ -217,14 +235,6 @@ action :create end -cookbook_file "#{oscm_install_path}/user-data/oscm-config" do - source "user-data/oscm-config" - owner oscm_group - group oscm_group - mode 0755 - action :create -end - cookbook_file "#{oscm_install_path}/user-data/deploy-oscmserver" do source "user-data/deploy-oscmserver" owner oscm_group @@ -233,25 +243,16 @@ action :create end -bash "create oscm configuration" do - code <<-EOH - sed -i 's/$HTTP_PROXY/#{oscm_proxy_httphost}:#{oscm_proxy_httpport}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$HTTPS_PROXY/#{oscm_proxy_httpshost}:#{oscm_proxy_httpsport}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$PROXY_USER/#{oscm_proxy_user}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$PROXY_PWD/#{oscm_proxy_pwd}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$MAIL_HOST/#{oscm_mail_host}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$MAIL_PORT/#{oscm_mail_port}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$MAIL_TLS/#{oscm_mail_tls}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$MAIL_USER/#{oscm_mail_user}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$MAIL_PWD/#{oscm_mail_pwd}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$REGISTRY_HOST/#{oscm_docker_host}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$REGISTRY_PORT/#{oscm_docker_port}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$REGISTRY_USER/#{oscm_docker_user}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$REGISTRY_PWD/#{oscm_docker_pwd}/g' #{oscm_install_path}/user-data/oscm-config - sed -i 's/$OSCM_IMAGES_TAG/#{oscm_docker_tag}/g' #{oscm_install_path}/user-data/oscm-config - EOH - environment ({ - }) +template "#{oscm_install_path}/user-data/oscm-config" do + source "oscm.conf.erb" + owner oscm_group + group oscm_group + mode 0640 + variables( + mail: node[:oscm][:mail], + docker: node[:oscm][:docker], + proxy: node[:oscm][:proxy] + ) end bash "create oscm volume stack" do @@ -299,7 +300,7 @@ end -bash "inject oscm certificates" do +bash "inject oscm configuration and certificates" do code <<-EOH mkdir -p "$(dirname "#{oscm_keypair_crowbar_sshkey}")" if [ ! -f #{oscm_keypair_crowbar_sshkey} ]; @@ -312,7 +313,8 @@ scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/oscm-config ${ip_appserver}:#{oscm_config_path} || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/deploy-oscmserver ${ip_appserver}:#{oscm_config_path} || true ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/finished" - if [ -f #{oscm_ssl_certfile} ]; then + if [ -f #{oscm_ssl_certfile} ]; + then ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}/ssl/" || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl || true @@ -321,13 +323,4 @@ ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/ssl/finished" fi EOH - environment ({ - "OS_USERNAME" => oscm_user, - "OS_PASSWORD" => oscm_password, - "OS_TENANT_NAME" => oscm_tenant, - "OS_AUTH_URL" => keystone_settings["internal_auth_url"], - "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], - "OS_USER_DOMAIN_NAME" => "Default", - "OS_PROJECT_DOMAIN_NAME" => "Default" - }) end diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index d3d016e731..83b14c7311 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -6,6 +6,7 @@ "timeout": 300, "debug": false, "keystone_instance": "none", + "nova_instance": "none", "heat_instance": "none", "keystone": { "tenant": "oscm", @@ -43,6 +44,7 @@ "from": "" }, "docker": { + "dockerhub": true, "host": "", "port": 0, "auth": false, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 483b9731d5..97dc28a570 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -15,6 +15,7 @@ "timeout": { "type": "int", "required": true }, "debug": { "type": "bool", "required": true }, "keystone_instance": { "type": "str", "required": true }, + "nova_instance": { "type": "str", "required": true }, "heat_instance": { "type": "str", "required": true }, "keystone" : { "type" : "map", @@ -83,6 +84,7 @@ "type" : "map", "required" : true, "mapping": { + "dockerhub": { "type": "bool", "required": true }, "host": { "type": "str", "required": true }, "port": { "type": "int", "required": true }, "auth": { "type": "bool", "required": true }, diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 53dea04a31..8e238c7b1a 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -24,10 +24,15 @@ %fieldset %legend = t(".docker_settings") - %span.help-block - = t('.docker_hint') - = string_field %w(docker host) - = integer_field %w(docker port) + = boolean_field %w(docker dockerhub), + "data-showit" => "false", + "data-showit-target" => "#dockerhub_container", + "data-showit-direct" => "true" + + #dockerhub_container + = string_field %w(docker host) + = integer_field %w(docker port) + = boolean_field %w(docker auth), "data-showit" => "true", "data-showit-target" => "#dockerauth_container", diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 4d0d78ff2a..81a099e277 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -31,12 +31,12 @@ en: from: 'Oscm Email Address' docker_settings: 'Docker Registry' docker: + dockerhub: 'Use Docker Hub' auth: 'Authentication Required' host: 'Registry Host' port: 'Registry Port' user: 'User' password: 'Password' - docker_hint: 'The registry host and port can be leaved empty for docker hub' proxy_settings: 'Proxy Settings' proxy: auth: 'Authentication Required' From 64d3950ad9ead8f3954c0616cec7e69a8979b365 Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 27 Sep 2017 15:48:30 +0200 Subject: [PATCH 066/181] Add oscm configuration template --- .../oscm/files/default/user-data/oscm-config | 25 ----------------- .../oscm/templates/default/oscm.conf.erb | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+), 25 deletions(-) delete mode 100644 chef/cookbooks/oscm/files/default/user-data/oscm-config create mode 100644 chef/cookbooks/oscm/templates/default/oscm.conf.erb diff --git a/chef/cookbooks/oscm/files/default/user-data/oscm-config b/chef/cookbooks/oscm/files/default/user-data/oscm-config deleted file mode 100644 index f52dca4df2..0000000000 --- a/chef/cookbooks/oscm/files/default/user-data/oscm-config +++ /dev/null @@ -1,25 +0,0 @@ -#proxy -HTTP_PROXY=$HTTP_PROXY -HTTPS_PROXY=$HTTPS_PROXY -PROXY_USER=$PROXY_USER -PROXY_PWD=$PROXY_PWD -#mail -MAIL_HOST=$MAIL_HOST -MAIL_PORT=$MAIL_PORT -MAIL_TLS=$MAIL_TLS -MAIL_USER=$MAIL_USER -MAIL_PWD=$MAIL_PWD -MAIL_ADDRESS=$MAIL_ADDRESS -MAIL_AUTH=$MAIL_AUTH -#docker registry -REGISTRY_HOST=$REGISTRY_HOST -REGISTRY_PORT=$REGISTRY_PORT -REGISTRY_USER=$REGISTRY_USER -REGISTRY_PWD=$REGISTRY_PWD -#docker image tag -OSCM_IMAGES_TAG=$OSCM_IMAGES_TAG -#docker host names -DB_HOSTNAME=oscm-db -OSCM_HOSTNAME=oscm-core -APP_HOSTNAME=oscm-app -BIRT_HOSTNAME=oscm-birt diff --git a/chef/cookbooks/oscm/templates/default/oscm.conf.erb b/chef/cookbooks/oscm/templates/default/oscm.conf.erb new file mode 100644 index 0000000000..cb406692d6 --- /dev/null +++ b/chef/cookbooks/oscm/templates/default/oscm.conf.erb @@ -0,0 +1,27 @@ +#proxy +USE_PROXY=<%= @proxy["use_proxy"] %> +HTTP_PROXY_HOST=<%= @proxy["http_host"] %> +HTTP_PROXY_PORT=<%= @proxy["http_port"] %> +HTTPS_PROXY_HOST=<%= @proxy["https_host"] %> +HTTPS_PROXY_PORT=<%= @proxy["https_port"] %> +NO_PROXY=<%= @proxy["no_proxy"] %> +PROXY_AUTH=<%= @proxy["auth"] %> +PROXY_USER=<%= @proxy["user"] %> +PROXY_PWD=<%= @proxy["password"] %> + +#mail +MAIL_HOST=<%= @mail["host"] %> +MAIL_PORT=<%= @mail["port"] %> +MAIL_TLS=<%= @mail["tls"] %> +MAIL_USER=<%= @mail["user"] %> +MAIL_PWD=<%= @mail["password"] %> +MAIL_ADDRESS=<%= @mail["from"] %> +MAIL_AUTH=<%= @mail["auth"] %> + +#docker +DOCKER_HUB=<%= @docker["dockerhub"] %> +REGISTRY_HOST=<%= @docker["host"] %> +REGISTRY_PORT=<%= @docker["port"] %> +REGISTRY_USER=<%= @docker["user"] %> +REGISTRY_PWD=<%= @docker["password"] %> +OSCM_IMAGES_TAG=<%= @docker["tag"] %> From 444374b62bbb72c2f442b6ce1af3cff275a5eed5 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 28 Sep 2017 08:24:17 +0200 Subject: [PATCH 067/181] Add docker organization setting --- .../oscm/templates/default/oscm.conf.erb | 39 ++++++++++--------- chef/data_bags/crowbar/template-oscm.json | 3 +- chef/data_bags/crowbar/template-oscm.schema | 1 + 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/chef/cookbooks/oscm/templates/default/oscm.conf.erb b/chef/cookbooks/oscm/templates/default/oscm.conf.erb index cb406692d6..a384c074b5 100644 --- a/chef/cookbooks/oscm/templates/default/oscm.conf.erb +++ b/chef/cookbooks/oscm/templates/default/oscm.conf.erb @@ -1,27 +1,28 @@ #proxy -USE_PROXY=<%= @proxy["use_proxy"] %> -HTTP_PROXY_HOST=<%= @proxy["http_host"] %> -HTTP_PROXY_PORT=<%= @proxy["http_port"] %> -HTTPS_PROXY_HOST=<%= @proxy["https_host"] %> -HTTPS_PROXY_PORT=<%= @proxy["https_port"] %> -NO_PROXY=<%= @proxy["no_proxy"] %> +PROXY_ENABLED=<%= @proxy["use_proxy"] %> +PROXY_HTTP_HOST=<%= @proxy["http_host"] %> +PROXY_HTTP_PORT=<%= @proxy["http_port"] %> +PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> +PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> +PROXY_NOPROXY=<%= @proxy["no_proxy"] %> PROXY_AUTH=<%= @proxy["auth"] %> PROXY_USER=<%= @proxy["user"] %> PROXY_PWD=<%= @proxy["password"] %> #mail -MAIL_HOST=<%= @mail["host"] %> -MAIL_PORT=<%= @mail["port"] %> -MAIL_TLS=<%= @mail["tls"] %> -MAIL_USER=<%= @mail["user"] %> -MAIL_PWD=<%= @mail["password"] %> -MAIL_ADDRESS=<%= @mail["from"] %> -MAIL_AUTH=<%= @mail["auth"] %> +SMTP_HOST=<%= @mail["host"] %> +SMTP_PORT=<%= @mail["port"] %> +SMTP_TLS=<%= @mail["tls"] %> +SMTP_USER=<%= @mail["user"] %> +SMTP_PWD=<%= @mail["password"] %> +SMTP_FROM=<%= @mail["from"] %> +SMTP_AUTH=<%= @mail["auth"] %> #docker -DOCKER_HUB=<%= @docker["dockerhub"] %> -REGISTRY_HOST=<%= @docker["host"] %> -REGISTRY_PORT=<%= @docker["port"] %> -REGISTRY_USER=<%= @docker["user"] %> -REGISTRY_PWD=<%= @docker["password"] %> -OSCM_IMAGES_TAG=<%= @docker["tag"] %> +DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> +DOCKER_REGISTRY_HOST=<%= @docker["host"] %> +DOCKER_REGISTRY_PORT=<%= @docker["port"] %> +DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> +DOCKER_REGISTRY_USER=<%= @docker["user"] %> +DOCKER_REGISTRY_PWD=<%= @docker["password"] %> +DOCKER_OSCM_TAG=<%= @docker["tag"] %> diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 83b14c7311..93c8cacc1b 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -46,10 +46,11 @@ "docker": { "dockerhub": true, "host": "", - "port": 0, + "port": 5000, "auth": false, "user": "", "password": "", + "organization": "servicecatalog" "tag": "17.5" }, "proxy": { diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 97dc28a570..9f736ea8ad 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -90,6 +90,7 @@ "auth": { "type": "bool", "required": true }, "user": { "type": "str", "required": true }, "password": { "type": "str", "required": true }, + "organization": { "type": "str", "required": true }, "tag": { "type": "str", "required": true } } }, From 72c15d79d35748c13d314456ce26800d84e65955 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 28 Sep 2017 09:46:52 +0200 Subject: [PATCH 068/181] Correct json --- chef/data_bags/crowbar/template-oscm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 93c8cacc1b..26c6de23f4 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -50,7 +50,7 @@ "auth": false, "user": "", "password": "", - "organization": "servicecatalog" + "organization": "servicecatalog", "tag": "17.5" }, "proxy": { From 5a0c4b76a8204bf5e8fa170694ef5e556815dc0c Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 2 Oct 2017 11:44:08 +0200 Subject: [PATCH 069/181] Add new settings --- .../oscm/files/default/application.yaml | 14 ++- chef/cookbooks/oscm/recipes/server.rb | 96 +++++++++---------- .../oscm/templates/default/oscm.conf.erb | 8 ++ chef/data_bags/crowbar/template-oscm.json | 1 + chef/data_bags/crowbar/template-oscm.schema | 1 + .../barclamp/oscm/_edit_attributes.html.haml | 2 + crowbar_framework/config/locales/oscm/en.yml | 1 + 7 files changed, 69 insertions(+), 54 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index e3aabf3a50..13e3aa46e6 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -55,6 +55,15 @@ resources: db_password: type: OS::Heat::RandomString + key_secret: + type: OS::Heat::RandomString + + db_core_password: + type: OS::Heat::RandomString + + db_app_password: + type: OS::Heat::RandomString + ### Network infrastructure ### oscm_network: @@ -152,7 +161,10 @@ resources: - "" - - "/dev/disk/by-id/virtio-" - { get_param: db_volume_id } - $DB_PASSWORD: { get_attr: [ db_password, value ] } + $PGPASSWORD: { get_attr: [ db_password, value ] } + $DB_PWD_CORE: { get_attr: [ db_core_password, value ] } + $DB_PWD_APP: { get_attr: [ db_app_password, value ] } + $KEY_SECRET: { get_attr: [ key_secret, value ] } $SSH_CERT: { get_param: ssh_cert } # User data payload for appserver diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 7bded77bed..2a08a0bf3b 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -73,6 +73,10 @@ nova_insecure = CrowbarOpenStackHelper.insecure(nova_config) openstack_args_nova = nova_insecure || keystone_settings["insecure"] ? "--insecure" : "" +heat_config = Barclamp::Config.load("openstack", "heat", node[:oscm][:heat_instance]) +heat_insecure = CrowbarOpenStackHelper.insecure(heat_config) +openstack_args_heat = heat_insecure || keystone_settings["insecure"] ? "--insecure" : "" + register_auth_hash = { user: keystone_settings["admin_user"], password: keystone_settings["admin_password"], @@ -243,6 +247,42 @@ action :create end +execute "create_oscm_volume_stack" do + command "#{openstack_cmd} #{openstack_args_heat} stack create --parameter db_size=#{oscm_db_volume_size} --parameter app_size=#{oscm_app_volume_size} \ + -t #{oscm_install_path}/volumes.yaml --wait #{oscm_volumestack_name}" + not_if "#{openstack_cmd} #{openstack_args_heat} stack list | grep -q #{oscm_volumestack_name}" +end + +ruby_block "get_oscm_volume_ids" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:volume_stack][:db_volume_id] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:volume_stack][:app_volume_id] = command_out.stdout.strip + command = "mkdir -p '$(dirname #{oscm_keypair_crowbar_sshkey})'" + command_out = shell_out(command) + command = "yes y | ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} -N ''" + command_out = shell_out(command) + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:instance_stack][:ip_appserver] = command_out.stdout.strip + end + action :create +end + +execute "create_oscm_instance_stack" do + command lazy { "#{openstack_cmd} #{openstack_args_heat} stack create --parameter app_volume_id=#{node[:oscm][:openstack][:volume_stack][:app_volume_id]} \ + --parameter db_volume_id=#{node[:oscm][:openstack][:volume_stack][:db_volume_id]} \ + --parameter image=#{oscm_image} --parameter flavor=#{oscm_flavor_name} \ + --parameter mail_port=#{oscm_mail_port} --parameter registry_port=#{oscm_docker_port} \ + --parameter-file ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub \ + -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name}" } + not_if "#{openstack_cmd} #{openstack_args_heat} stack list | grep -q #{oscm_instancestack_name}" +end + template "#{oscm_install_path}/user-data/oscm-config" do source "oscm.conf.erb" owner oscm_group @@ -251,64 +291,14 @@ variables( mail: node[:oscm][:mail], docker: node[:oscm][:docker], - proxy: node[:oscm][:proxy] + proxy: node[:oscm][:proxy], + host_fqdn: node[:oscm][:host_fqdn], + floating_ip: node[:oscm][:openstack][:instance_stack][:ip_appserver] ) end -bash "create oscm volume stack" do - code <<-EOH - openstack stack create --parameter "db_size=#{oscm_db_volume_size}" --parameter "app_size=#{oscm_app_volume_size}" -t #{oscm_install_path}/volumes.yaml --wait #{oscm_volumestack_name} &> /dev/null || true - EOH - environment ({ - "OS_USERNAME" => oscm_user, - "OS_PASSWORD" => oscm_password, - "OS_TENANT_NAME" => oscm_tenant, - "OS_AUTH_URL" => keystone_settings["internal_auth_url"], - "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], - "OS_USER_DOMAIN_NAME" => "Default", - "OS_PROJECT_DOMAIN_NAME" => "Default" - }) -end - -bash "create oscm instance stack" do - code <<-EOH - app_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - db_volume_id=$(openstack stack output show -f shell #{oscm_volumestack_name} db_volume_id | grep -Po '(?<=^output_value=\")[^\"]*') - if [ -f #{oscm_keypair_crowbar_sshkey}.pub ]; - then - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ - --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ - --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ - --parameter-file "ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub"\ - -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true - else - openstack stack create --parameter "app_volume_id=${app_volume_id}" --parameter "db_volume_id=${db_volume_id}"\ - --parameter "image=#{oscm_image}" --parameter "flavor=#{oscm_flavor_name}"\ - --parameter "mail_port=#{oscm_mail_port}" --parameter "registry_port=#{oscm_docker_port}"\ - -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name} &> /dev/null || true - fi - EOH - environment ({ - "OS_USERNAME" => oscm_user, - "OS_PASSWORD" => oscm_password, - "OS_TENANT_NAME" => oscm_tenant, - "OS_AUTH_URL" => keystone_settings["internal_auth_url"], - "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], - "OS_USER_DOMAIN_NAME" => "Default", - "OS_PROJECT_DOMAIN_NAME" => "Default" - }) -end - - bash "inject oscm configuration and certificates" do code <<-EOH - mkdir -p "$(dirname "#{oscm_keypair_crowbar_sshkey}")" - if [ ! -f #{oscm_keypair_crowbar_sshkey} ]; - then - ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} - fi - ip_appserver=$(openstack stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*') - ssh-keygen -R ${ip_appserver} -f /root/.ssh/known_hosts ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}" || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/oscm-config ${ip_appserver}:#{oscm_config_path} || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/deploy-oscmserver ${ip_appserver}:#{oscm_config_path} || true diff --git a/chef/cookbooks/oscm/templates/default/oscm.conf.erb b/chef/cookbooks/oscm/templates/default/oscm.conf.erb index a384c074b5..ddf2815b57 100644 --- a/chef/cookbooks/oscm/templates/default/oscm.conf.erb +++ b/chef/cookbooks/oscm/templates/default/oscm.conf.erb @@ -26,3 +26,11 @@ DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> DOCKER_REGISTRY_USER=<%= @docker["user"] %> DOCKER_REGISTRY_PWD=<%= @docker["password"] %> DOCKER_OSCM_TAG=<%= @docker["tag"] %> + +#host +<% if @host_fqdn.empty? -%> + HOST_FQDN=<%= @host_fqdn %> +<% else %> + HOST_FQDN=<%= @floating_ip %> +<% end %> + \ No newline at end of file diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 26c6de23f4..bad844b239 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -64,6 +64,7 @@ "user": "", "password": "" }, + "host_fqdn": "", "ssl": { "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", "keyfile": "/etc/oscm/ssl/private/signing_key.pem", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 9f736ea8ad..484cf6faf5 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -109,6 +109,7 @@ "password": { "type": "str", "required": true } } }, + "host_fqdn": { "type" : "str", "required" : true }, "ssl": { "type": "map", "required": true, diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 8e238c7b1a..eb54d0d918 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -79,6 +79,8 @@ %legend = t(".ssl_header") + = string_field %w(host_fqdn) + = select_field %w(api protocol), :collection => :api_protocols_for_oscm, "data-sslprefix" => "ssl", diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 81a099e277..f43d120608 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -57,6 +57,7 @@ en: api: protocol: 'Protocol' ssl_header: 'SSL Support' + host_fqdn: 'OSCM Host FQDN (if left empty, the floating IP will be used)' ssl: generate_certs: 'Generate (self-signed) certificates (implies insecure)' certfile: 'SSL Certificate File' From 0e6d1b5e5287e94c085ffc94e5019f4b5fbad931 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 2 Oct 2017 12:44:45 +0200 Subject: [PATCH 070/181] Host fqdn setting --- chef/cookbooks/oscm/templates/default/oscm.conf.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/oscm/templates/default/oscm.conf.erb b/chef/cookbooks/oscm/templates/default/oscm.conf.erb index ddf2815b57..ca343678b5 100644 --- a/chef/cookbooks/oscm/templates/default/oscm.conf.erb +++ b/chef/cookbooks/oscm/templates/default/oscm.conf.erb @@ -28,9 +28,9 @@ DOCKER_REGISTRY_PWD=<%= @docker["password"] %> DOCKER_OSCM_TAG=<%= @docker["tag"] %> #host -<% if @host_fqdn.empty? -%> - HOST_FQDN=<%= @host_fqdn %> +<% if @host_fqdn.strip.empty? -%> +HOST_FQDN=<%= @floating_ip %> <% else %> - HOST_FQDN=<%= @floating_ip %> +HOST_FQDN=<%= @host_fqdn %> <% end %> \ No newline at end of file From dd9d0448f65d8d279e9e3d609f4b798291fadc2e Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 2 Oct 2017 12:59:34 +0200 Subject: [PATCH 071/181] Refactor flavor access creation --- chef/cookbooks/oscm/recipes/server.rb | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 2a08a0bf3b..7d12dd48b8 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -68,7 +68,8 @@ openstack_cmd = "#{env} openstack" -# the flavor is created via the nova API +openstack_args_keystone = keystone_settings["insecure"] ? "--insecure" : "" + nova_config = Barclamp::Config.load("openstack", "nova", node[:oscm][:nova_instance]) nova_insecure = CrowbarOpenStackHelper.insecure(nova_config) openstack_args_nova = nova_insecure || keystone_settings["insecure"] ? "--insecure" : "" @@ -167,20 +168,19 @@ not_if "#{openstack_cmd} #{openstack_args_nova} flavor list --all | grep -q #{oscm_flavor_name}" end -bash "create_oscm_flavor_access" do - code <<-EOH - tenant_id=$(openstack project show -f shell #{oscm_tenant} | grep -Po '(?<=^id=\")[^\"]*') - nova flavor-access-add #{oscm_flavor_name} $tenant_id &> /dev/null || true -EOH - environment ({ - "OS_USERNAME" => oscm_user, - "OS_PASSWORD" => oscm_password, - "OS_TENANT_NAME" => oscm_tenant, - "OS_AUTH_URL" => keystone_settings["internal_auth_url"], - "OS_IDENTITY_API_VERSION" => keystone_settings["api_version"], - "OS_USER_DOMAIN_NAME" => "Default", - "OS_PROJECT_DOMAIN_NAME" => "Default" - }) +ruby_block "get_oscm_tenant_id" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_keystone} project show -f shell #{oscm_tenant} | grep -Po '(?<=^id=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:keystone][:tenant_id] = command_out.stdout.strip + end + action :create +end + +execute "create_oscm_flavor_access" do + command = lazy { "#{openstack_cmd} #{openstack_args_nova} flavor-access-add #{oscm_flavor_name} #{node[:oscm][:keystone][:tenant_id]}" } + not_if "#{openstack_cmd} #{openstack_args_nova} flavor-access-list --flavor #{oscm_flavor_name} | grep -q #{node[:oscm][:keystone][:tenant_id]}" end bash "create_oscm_keypair_file" do From 6ccc22b3998ab871493a4b23a8498311760af0b8 Mon Sep 17 00:00:00 2001 From: stavreva Date: Mon, 2 Oct 2017 16:05:38 +0200 Subject: [PATCH 072/181] Rename tenant to project --- chef/cookbooks/oscm/recipes/server.rb | 54 +++++++++++---------- chef/data_bags/crowbar/template-oscm.json | 2 +- chef/data_bags/crowbar/template-oscm.schema | 2 +- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 7d12dd48b8..a309e902c6 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -14,7 +14,7 @@ # limitations under the License. # -oscm_tenant = node[:oscm][:keystone][:tenant] +oscm_project = node[:oscm][:keystone][:project] oscm_user = node[:oscm][:keystone][:user] oscm_password = node[:oscm][:keystone][:password] oscm_ssl_certfile = node[:oscm][:ssl][:certfile] @@ -59,7 +59,7 @@ env = "OS_USERNAME='#{oscm_user}' " env << "OS_PASSWORD='#{oscm_password}' " -env << "OS_PROJECT_NAME='#{oscm_tenant}' " +env << "OS_PROJECT_NAME='#{oscm_project}' " env << "OS_AUTH_URL='#{keystone_settings["internal_auth_url"]}' " env << "OS_INTERFACE=internal " env << "OS_IDENTITY_API_VERSION='#{keystone_settings["api_version"]}' " @@ -81,7 +81,7 @@ register_auth_hash = { user: keystone_settings["admin_user"], password: keystone_settings["admin_password"], - tenant: keystone_settings["admin_tenant"] + project: keystone_settings["admin_project"] } keystone_register "oscm wakeup keystone" do @@ -93,14 +93,14 @@ action :wakeup end -keystone_register "oscm create tenant" do +keystone_register "oscm create project" do protocol keystone_settings["protocol"] insecure keystone_settings["insecure"] host keystone_settings["internal_url_host"] port keystone_settings["admin_port"] auth register_auth_hash - tenant_name oscm_tenant - action :add_tenant + project_name oscm_project + action :add_project end keystone_register "oscm register user" do @@ -111,7 +111,7 @@ auth register_auth_hash user_name oscm_user user_password oscm_password - tenant_name oscm_tenant + project_name oscm_project action :add_user end @@ -122,7 +122,7 @@ port keystone_settings["admin_port"] auth register_auth_hash user_name oscm_user - tenant_name oscm_tenant + project_name oscm_project role_name "admin" action :add_access end @@ -134,7 +134,7 @@ port keystone_settings["admin_port"] auth register_auth_hash user_name oscm_user - tenant_name oscm_tenant + project_name oscm_project role_name "Member" action :add_access end @@ -146,7 +146,7 @@ port keystone_settings["admin_port"] auth register_auth_hash user_name oscm_user - tenant_name oscm_tenant + project_name oscm_project role_name "_member_" action :add_access end @@ -168,19 +168,9 @@ not_if "#{openstack_cmd} #{openstack_args_nova} flavor list --all | grep -q #{oscm_flavor_name}" end -ruby_block "get_oscm_tenant_id" do - block do - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "#{openstack_cmd} #{openstack_args_keystone} project show -f shell #{oscm_tenant} | grep -Po '(?<=^id=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:keystone][:tenant_id] = command_out.stdout.strip - end - action :create -end - execute "create_oscm_flavor_access" do - command = lazy { "#{openstack_cmd} #{openstack_args_nova} flavor-access-add #{oscm_flavor_name} #{node[:oscm][:keystone][:tenant_id]}" } - not_if "#{openstack_cmd} #{openstack_args_nova} flavor-access-list --flavor #{oscm_flavor_name} | grep -q #{node[:oscm][:keystone][:tenant_id]}" + command "#{openstack_cmd} #{openstack_args_nova} flavor set --project #{oscm_project} #{oscm_flavor_name}" + ignore_failure true end bash "create_oscm_keypair_file" do @@ -262,13 +252,17 @@ command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell #{oscm_volumestack_name} app_volume_id | grep -Po '(?<=^output_value=\")[^\"]*'" command_out = shell_out(command) node[:oscm][:openstack][:volume_stack][:app_volume_id] = command_out.stdout.strip + end + action :create +end + +ruby_block "generate_oscm_crowbar_ssh_keys" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) command = "mkdir -p '$(dirname #{oscm_keypair_crowbar_sshkey})'" command_out = shell_out(command) command = "yes y | ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} -N ''" command_out = shell_out(command) - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:instance_stack][:ip_appserver] = command_out.stdout.strip end action :create end @@ -283,6 +277,16 @@ not_if "#{openstack_cmd} #{openstack_args_heat} stack list | grep -q #{oscm_instancestack_name}" end +ruby_block "get_oscm_floating_ip" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:instance_stack][:ip_appserver] = command_out.stdout.strip + end + action :create +end + template "#{oscm_install_path}/user-data/oscm-config" do source "oscm.conf.erb" owner oscm_group diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index bad844b239..75399885c6 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -9,7 +9,7 @@ "nova_instance": "none", "heat_instance": "none", "keystone": { - "tenant": "oscm", + "project": "oscm", "user": "oscm", "password": "oscm" }, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 484cf6faf5..f7a25d72da 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -21,7 +21,7 @@ "type" : "map", "required" : true, "mapping": { - "tenant": { "type": "str", "required": true }, + "project": { "type": "str", "required": true }, "user": { "type": "str", "required": true }, "password": { "type": "str", "required": true } } From e2f39bfd91925baf63d59f74316a8aae5751fb5e Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 5 Oct 2017 12:04:02 +0200 Subject: [PATCH 073/181] Revert project to tenant --- chef/cookbooks/oscm/recipes/server.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index a309e902c6..dd135b9fff 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -81,7 +81,7 @@ register_auth_hash = { user: keystone_settings["admin_user"], password: keystone_settings["admin_password"], - project: keystone_settings["admin_project"] + tenant: keystone_settings["admin_tenant"] } keystone_register "oscm wakeup keystone" do @@ -99,8 +99,8 @@ host keystone_settings["internal_url_host"] port keystone_settings["admin_port"] auth register_auth_hash - project_name oscm_project - action :add_project + tenant_name oscm_project + action :add_tenant end keystone_register "oscm register user" do @@ -111,7 +111,7 @@ auth register_auth_hash user_name oscm_user user_password oscm_password - project_name oscm_project + tenant_name oscm_project action :add_user end @@ -122,7 +122,7 @@ port keystone_settings["admin_port"] auth register_auth_hash user_name oscm_user - project_name oscm_project + tenant_name oscm_project role_name "admin" action :add_access end @@ -134,7 +134,7 @@ port keystone_settings["admin_port"] auth register_auth_hash user_name oscm_user - project_name oscm_project + tenant_name oscm_project role_name "Member" action :add_access end @@ -146,7 +146,7 @@ port keystone_settings["admin_port"] auth register_auth_hash user_name oscm_user - project_name oscm_project + tenant_name oscm_project role_name "_member_" action :add_access end From 2ef5de4f147365a3781f435193ca23a57171fdbe Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 6 Oct 2017 11:51:10 +0200 Subject: [PATCH 074/181] Add docker compose templates --- .../oscm/files/default/application.yaml | 20 +++++ chef/cookbooks/oscm/recipes/server.rb | 63 +++++++++++++- .../default/docker-compose-initdb.yml.erb | 71 ++++++++++++++++ .../default/docker-compose-oscm.yml.erb | 82 +++++++++++++++++++ .../oscm/templates/default/oscm.conf.erb | 1 + .../oscm/templates/default/var.env.erb | 34 ++++++++ 6 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb create mode 100644 chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb create mode 100644 chef/cookbooks/oscm/templates/default/var.env.erb diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 13e3aa46e6..7d80fb3029 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -224,3 +224,23 @@ outputs: get_attr: - ip_appserver - floating_ip_address + db_password: + value: + get_attr: + - db_password + - value + db_core_password: + value: + get_attr: + - db_core_password + - value + db_app_password: + value: + get_attr: + - db_app_password + - value + key_secret: + value: + get_attr: + - key_secret + - value diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index dd135b9fff..9e7ba44570 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -287,6 +287,25 @@ action :create end +ruby_block "get_oscm_secrets" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} db_password | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:instance_stack][:db_password] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} db_core_password | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:instance_stack][:db_core_password] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} db_app_password | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:instance_stack][:db_app_password] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} key_secret | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:oscm][:openstack][:instance_stack][:key_secret] = command_out.stdout.strip + end + action :create +end + template "#{oscm_install_path}/user-data/oscm-config" do source "oscm.conf.erb" owner oscm_group @@ -301,19 +320,55 @@ ) end +template "#{oscm_install_path}/docker-compose-initdb.yml" do + source "docker-compose-initdb.yml.erb" + owner oscm_group + group oscm_group + mode 0640 + variables( + docker: node[:oscm][:docker] + ) +end + +template "#{oscm_install_path}/docker-compose-oscm.yml" do + source "docker-compose-oscm.yml.erb" + owner oscm_group + group oscm_group + mode 0640 + variables( + docker: node[:oscm][:docker] + ) +end + +template "#{oscm_install_path}/var.env" do + source "var.env.erb" + owner oscm_group + group oscm_group + mode 0640 + variables( + mail: node[:oscm][:mail], + docker: node[:oscm][:docker], + proxy: node[:oscm][:proxy], + host_fqdn: node[:oscm][:host_fqdn], + instance: node[:oscm][:openstack][:instance_stack] + ) +end + bash "inject oscm configuration and certificates" do code <<-EOH ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}" || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/oscm-config ${ip_appserver}:#{oscm_config_path} || true scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/deploy-oscmserver ${ip_appserver}:#{oscm_config_path} || true ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/finished" + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/docker-compose-initdb.yml ${ip_appserver}:#{oscm_config_path}/docker-compose || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/docker-compose-oscm.yml ${ip_appserver}:#{oscm_config_path}/docker-compose || true + ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/docker-compose/finished" if [ -f #{oscm_ssl_certfile} ]; then ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}/ssl/" || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_keyfile} ${ip_appserver}:#{oscm_config_path}/ssl || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_config_path}/ssl || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl/oscm.crt || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_keyfile} ${ip_appserver}:#{oscm_config_path}/ssl/oscm.key || true + scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_config_path}/ssl/oscm.chain || true ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/ssl/finished" fi EOH diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb new file mode 100644 index 0000000000..1121664f69 --- /dev/null +++ b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb @@ -0,0 +1,71 @@ +version: "3" +services: + oscm-db: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-db:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-db:@docker["tag"]%> +<% end %> + container_name: oscm-db + volumes: + - /docker/data/oscm-db/data:/var/lib/postgresql/data + + oscm-initdb-core: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-initdb-core:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-core:@docker["tag"]%> +<% end %> + container_name: oscm-initdb-core + env_file: var.env + environment: + - TARGET=CORE + - SOURCE=INIT + - OVERWRITE=true + links: + - oscm-db:oscm-db + + oscm-initdb-jms: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-initdb-jms:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-jms:@docker["tag"]%> +<% end %> + container_name: oscm-initdb-jms + env_file: var.env + environment: + - TARGET=JMS + - SOURCE=INIT + - OVERWRITE=true + links: + - oscm-db:oscm-db + + oscm-initdb-app: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-initdb-app:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-app:@docker["tag"]%> +<% end %> + container_name: oscm-initdb-app + env_file: var.env + environment: + - TARGET=APP + - SOURCE=INIT + - OVERWRITE=true + links: + - oscm-db:oscm-db + + oscm-initdb-controller-openstack: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-initdb-controller-openstack:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-controller-openstack:@docker["tag"]%> +<% end %> + container_name: oscm-initdb-controller-openstack + env_file: var.env + environment: + - TARGET=CONTROLLER + - SOURCE=NONE + - OVERWRITE=true + links: + - oscm-db:oscm-db diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb new file mode 100644 index 0000000000..f391056699 --- /dev/null +++ b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb @@ -0,0 +1,82 @@ +version: "3" +services: + oscm-db: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-db:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-db:@docker["tag"]%> +<% end %> + container_name: oscm-db + volumes: + - /docker/data/oscm-db/data:/var/lib/postgresql/data + ports: + - 5432:5432 + + oscm-core: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-core:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-core:@docker["tag"]%> +<% end %> + container_name: oscm-core + env_file: var.env + environment: + - JPDA_ADDRESS=8000 + - JPDA_TRANSPORT=dt_socket + links: + - oscm-db:oscm-db + volumes: + - /docker/config/oscm-core/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 10080:8080 + - 10081:8081 + + oscm-app: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-app:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-app:@docker["tag"]%> +<% end %> + container_name: oscm-app + env_file: var.env + environment: + - JPDA_ADDRESS=8000 + - JPDA_TRANSPORT=dt_socket + links: + - oscm-db:oscm-db + volumes: + - /docker/config/oscm-app/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 8880:8880 + - 8881:8881 + + oscm-birt: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-birt:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-birt:@docker["tag"]%> +<% end %> + container_name: oscm-birt + env_file: var.env + volumes: + - /docker/config/oscm-birt/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 8181:8181 + + oscm-branding: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]/oscm-branding:@docker["tag"] %> +<% else %> + image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-branding:@docker["tag"]%> +<% end %> + container_name: oscm-branding + env_file: var.env + volumes: + - /docker/config/brandings:/brandings:ro + - /docker/config/oscm-branding/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 4443:443 diff --git a/chef/cookbooks/oscm/templates/default/oscm.conf.erb b/chef/cookbooks/oscm/templates/default/oscm.conf.erb index ca343678b5..fbd92801a8 100644 --- a/chef/cookbooks/oscm/templates/default/oscm.conf.erb +++ b/chef/cookbooks/oscm/templates/default/oscm.conf.erb @@ -23,6 +23,7 @@ DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> DOCKER_REGISTRY_HOST=<%= @docker["host"] %> DOCKER_REGISTRY_PORT=<%= @docker["port"] %> DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> +DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> DOCKER_REGISTRY_USER=<%= @docker["user"] %> DOCKER_REGISTRY_PWD=<%= @docker["password"] %> DOCKER_OSCM_TAG=<%= @docker["tag"] %> diff --git a/chef/cookbooks/oscm/templates/default/var.env.erb b/chef/cookbooks/oscm/templates/default/var.env.erb new file mode 100644 index 0000000000..cb9c864116 --- /dev/null +++ b/chef/cookbooks/oscm/templates/default/var.env.erb @@ -0,0 +1,34 @@ +SMTP_HOST=<%= @mail["host"] %> +SMTP_PORT=<%= @mail["port"] %> +SMTP_TLS=<%= @mail["tls"] %> +SMTP_USER=<%= @mail["user"] %> +SMTP_PWD=<%= @mail["password"] %> +SMTP_FROM=<%= @mail["from"] %> +SMTP_AUTH=<%= @mail["auth"] %> + + +KEY_SECRET=${KEY_SECRET} + +<% if @host_fqdn.strip.empty? -%> +HOST_FQDN=<%= @instance["ip_appserver"] %> +<% else %> +HOST_FQDN=<%= @host_fqdn %> +<% end %> + +DB_PORT_CORE=5432 +DB_PORT_JMS=5432 +DB_PORT_APP=5432 + +DB_PWD_CORE=<%= @instance["db_core_password"] %> +DB_PWD_APP=<%= @instance["db_app_password"] %> +DB_SUPERPWD=<%= @instance["db_password"] %> + +APP_ADMIN_MAIL_ADDRESS=<%= @mail["from"] %> + +CONTROLLER_ID=ess.openstack +CONTROLLER_ORG_ID=PLATFORM_OPERATOR +CONTROLLER_USER_KEY=1000 +CONTROLLER_USER_NAME=administrator +CONTROLLER_USER_PASS=admin123 + +TOMEE_DEBUG=false From 4f3fb54e4639a2f2ce563dd7abe0bbe838d4f176 Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 6 Oct 2017 12:26:20 +0200 Subject: [PATCH 075/181] docker compose templates --- .../default/docker-compose-initdb.yml.erb | 30 ++++++++--------- .../default/docker-compose-oscm.yml.erb | 32 +++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb index 1121664f69..6c68bf9996 100644 --- a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb +++ b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb @@ -1,20 +1,20 @@ version: "3" services: oscm-db: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-db:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-db:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% end %> container_name: oscm-db volumes: - /docker/data/oscm-db/data:/var/lib/postgresql/data oscm-initdb-core: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-initdb-core:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-core:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-core env_file: var.env @@ -26,10 +26,10 @@ services: - oscm-db:oscm-db oscm-initdb-jms: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-initdb-jms:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-jms:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-jms env_file: var.env @@ -41,10 +41,10 @@ services: - oscm-db:oscm-db oscm-initdb-app: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-initdb-app:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-app:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-app env_file: var.env @@ -56,10 +56,10 @@ services: - oscm-db:oscm-db oscm-initdb-controller-openstack: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-initdb-controller-openstack:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-initdb-controller-openstack:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-controller-openstack env_file: var.env diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb index f391056699..7b6ad66ad9 100644 --- a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb +++ b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb @@ -1,10 +1,10 @@ version: "3" services: oscm-db: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-db:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-db:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% end %> container_name: oscm-db volumes: @@ -13,11 +13,11 @@ services: - 5432:5432 oscm-core: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-core:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-core:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-core:@docker["tag"]%> -<% end %> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> +<% end %> container_name: oscm-core env_file: var.env environment: @@ -33,10 +33,10 @@ services: - 10081:8081 oscm-app: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-app:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-app:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-app:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> <% end %> container_name: oscm-app env_file: var.env @@ -53,10 +53,10 @@ services: - 8881:8881 oscm-birt: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-birt:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-birt:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-birt:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> <% end %> container_name: oscm-birt env_file: var.env @@ -67,10 +67,10 @@ services: - 8181:8181 oscm-branding: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]/oscm-branding:@docker["tag"] %> +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-branding:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]:@docker["port"]/@docker["organization"]/oscm-branding:@docker["tag"]%> + image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> <% end %> container_name: oscm-branding env_file: var.env From 0bce4e680a544c0e9a0448958770bfdc338b401a Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 6 Oct 2017 12:38:00 +0200 Subject: [PATCH 076/181] Remove unnecessary variables --- chef/cookbooks/oscm/files/default/application.yaml | 4 ---- .../oscm/files/default/user-data/heat-config | 1 - chef/cookbooks/oscm/templates/default/oscm.conf.erb | 11 +---------- chef/cookbooks/oscm/templates/default/var.env.erb | 5 ++--- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 7d80fb3029..4e9446f3ed 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -161,10 +161,6 @@ resources: - "" - - "/dev/disk/by-id/virtio-" - { get_param: db_volume_id } - $PGPASSWORD: { get_attr: [ db_password, value ] } - $DB_PWD_CORE: { get_attr: [ db_core_password, value ] } - $DB_PWD_APP: { get_attr: [ db_app_password, value ] } - $KEY_SECRET: { get_attr: [ key_secret, value ] } $SSH_CERT: { get_param: ssh_cert } # User data payload for appserver diff --git a/chef/cookbooks/oscm/files/default/user-data/heat-config b/chef/cookbooks/oscm/files/default/user-data/heat-config index 9fe0486068..835b27b5ed 100644 --- a/chef/cookbooks/oscm/files/default/user-data/heat-config +++ b/chef/cookbooks/oscm/files/default/user-data/heat-config @@ -8,7 +8,6 @@ chmod 400 /etc/oscm/config/heat-config cat > /etc/oscm/config/heat-config < PROXY_USER=<%= @proxy["user"] %> PROXY_PWD=<%= @proxy["password"] %> -#mail -SMTP_HOST=<%= @mail["host"] %> -SMTP_PORT=<%= @mail["port"] %> -SMTP_TLS=<%= @mail["tls"] %> -SMTP_USER=<%= @mail["user"] %> -SMTP_PWD=<%= @mail["password"] %> -SMTP_FROM=<%= @mail["from"] %> -SMTP_AUTH=<%= @mail["auth"] %> - #docker DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> DOCKER_REGISTRY_HOST=<%= @docker["host"] %> @@ -30,7 +21,7 @@ DOCKER_OSCM_TAG=<%= @docker["tag"] %> #host <% if @host_fqdn.strip.empty? -%> -HOST_FQDN=<%= @floating_ip %> +HOST_FQDN=<%= @instance["ip_appserver"] %> <% else %> HOST_FQDN=<%= @host_fqdn %> <% end %> diff --git a/chef/cookbooks/oscm/templates/default/var.env.erb b/chef/cookbooks/oscm/templates/default/var.env.erb index cb9c864116..2b45afbd22 100644 --- a/chef/cookbooks/oscm/templates/default/var.env.erb +++ b/chef/cookbooks/oscm/templates/default/var.env.erb @@ -7,8 +7,6 @@ SMTP_FROM=<%= @mail["from"] %> SMTP_AUTH=<%= @mail["auth"] %> -KEY_SECRET=${KEY_SECRET} - <% if @host_fqdn.strip.empty? -%> HOST_FQDN=<%= @instance["ip_appserver"] %> <% else %> @@ -21,7 +19,8 @@ DB_PORT_APP=5432 DB_PWD_CORE=<%= @instance["db_core_password"] %> DB_PWD_APP=<%= @instance["db_app_password"] %> -DB_SUPERPWD=<%= @instance["db_password"] %> +DB_SUPERPWD=<%= @instance["db_password"] %> +KEY_SECRET=<%= @instance["key_secret"] %> APP_ADMIN_MAIL_ADDRESS=<%= @mail["from"] %> From 1d5f2164f94cd578108eb9158b4afc6c1dec5a04 Mon Sep 17 00:00:00 2001 From: stavreva Date: Fri, 6 Oct 2017 12:45:07 +0200 Subject: [PATCH 077/181] oscm config template variable --- chef/cookbooks/oscm/recipes/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 9e7ba44570..1e8c5c450b 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -316,7 +316,7 @@ docker: node[:oscm][:docker], proxy: node[:oscm][:proxy], host_fqdn: node[:oscm][:host_fqdn], - floating_ip: node[:oscm][:openstack][:instance_stack][:ip_appserver] + instance: node[:oscm][:openstack][:instance_stack] ) end From e38ffc04c861dddabcbd748c9ba055a56dedb6fb Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 6 Oct 2017 11:16:18 +0200 Subject: [PATCH 078/181] refactored oscm deploy script: basic host setup working --- .../files/default/user-data/deploy-oscmserver | 255 +++++++++++++----- 1 file changed, 187 insertions(+), 68 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 1392af219c..89a44efe05 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -1,96 +1,215 @@ #!/bin/sh - set -x set -e - exec &> /var/log/setup-machine.log -CONFIG_PATH=/etc/oscm/config -SSL_CONFIG_PATH=${CONFIG_PATH}/ssl -OSCM_CONFIG=${CONFIG_PATH}/oscm-config - -source ${CONFIG_PATH}/heat-config - - -if [ ${CONFIG_PATH} ]; then - until [ -f $CONFIG_PATH/finished ] +# Variables for this script +CONFIG_BASE=/etc/oscm +CONFIG_PATH=${CONFIG_BASE}/config +SSL_CONFIG_PATH=${CONFIG_BASE}/ssl +COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose +DOCKER_PATH=/docker + +# Fixed values; neither from oscm-config nor heat-config +export DB_PORT=5432 # Fixed by deployment script: 5432 +export DB_PWD_JMS=jmsuser # Fixed by deployment script: jmsuser +export CONTROLLER_ORG_ID=PLATFORM_OPERATOR # Fixed by deployment script: PLATFORM_OPERATOR +export CONTROLLER_USER_KEY=1000 # Fixed by deployment script: 1000 +export CONTROLLER_USER_NAME=administrator # Fixed by deployment script: administrator +export CONTROLLER_USER_PASS=admin123 # Fixed by deployment script: admin123 +export DB_VOLUME_DATA_SRC=${DOCKER_PATH}/data/oscm-db/data +export BRANDING_VOLUME_BRANDINGS_SRC=${DOCKER_PATH}/config/brandings +export TOMEE_DEBUG=false + +# To be removed from this script +export IMAGE_DB=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-db:soc +export IMAGE_CORE=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-core:soc +export IMAGE_APP=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-app:soc +export IMAGE_BIRT=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-birt:soc +export IMAGE_BRANDING=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-branding:soc +export IMAGE_INITDB=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-initdb:soc +export KEY_SECRET=secret # Generate with Barclamp (heat-config) +export HOST_FQDN=estshimizusles.intern.est.fujitsu.com # Operator set or generated in Barclamp (oscm-config) +export DB_PWD_CORE=bssuser # Generate with Barclamp (heat-config) +export DB_PWD_APP=bssappuser # Generate with Barclamp (heat-config) +export DB_SUPERPWD=postgres # Generate with Barclamp (heat-config) + +# Wait for remote copying to finish +if [ ${CONFIG_PATH} ] && [ ${SSL_CONFIG_PATH} ] && [ ${COMPOSE_CONFIG_PATH} ]; then + until [ -f ${CONFIG_PATH}/finished ] && [ -f ${SSL_CONFIG_PATH}/finished ] && [ -f ${COMPOSE_CONFIG_PATH}/finished ] do - echo "scp for oscm configuration file is not finished yet sleeping 5sec." + echo "scp for oscm configuration files is not finished yet - sleeping 5s" sleep 5 done - echo "scp for oscm configuration file is finished." + echo "scp for oscm configuration files is finished." fi -if [ ${SSL_CONFIG_PATH} ]; then - until [ -f $SSL_CONFIG_PATH/finished ] - do - echo "scp for oscm ssl certificate is not finished yet sleeping 5sec." - sleep 5 - done - echo "scp for oscm ssl certificate is finished." - # TODO: add certificate to a local docker volume folder. - # echo $SSL_CONFIG_PATH/somekey.pem > /var/web_cert -fi +# Enable automatic exporting of variables +set -a +# Read configuration files +source ${CONFIG_PATH}/heat-config +source ${CONFIG_PATH}/oscm-config +# Disable automatic exporting of variables +set +a + -WAIT_CURL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" -eval $WAIT_CURL +# TODO: add certificate to a local docker volume folder. +# echo $SSL_CONFIG_PATH/somekey.pem > /var/web_cert -source ${OSCM_CONFIG} +# WAIT_CURL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" +# eval $WAIT_CURL # For some reason the volume id is shortened to 27 characters when the device # link in /dev/disk/by-id/ is created. Account for this by truncating the # device name accordingly. -volume_dev_db=$(printf '%.43s\n' $DB_VOLUME_DEV) -volume_dev_app=$(printf '%.43s\n' $APP_VOLUME_DEV) +# volume_dev_db=$(printf '%.43s\n' $DB_VOLUME_DEV) +# volume_dev_app=$(printf '%.43s\n' $APP_VOLUME_DEV) # Only format volume if it hasn't already been formatted. -if ! file -Ls $volume_dev_db | grep -q "filesystem data" ; then - mkfs.ext4 "$volume_dev_db" +# if ! file -Ls $volume_dev_db | grep -q "filesystem data" ; then +# mkfs.ext4 "$volume_dev_db" +# fi +# +# if ! file -Ls $volume_dev_app | grep -q "filesystem data" ; then +# mkfs.ext4 "$volume_dev_app" +# fi +# +# mkdir -p /db +# if ! grep -qs '/db' /proc/mounts; then +# mount -t ext4 $volume_dev_db /db +# fi +# +# mkdir -p /oscm +# if ! grep -qs '/oscm' /proc/mounts; then +# mount -t ext4 $volume_dev_app /oscm +# fi + + +# DB=/db/database +# APP_PATH=/oscm/oscm-app/ +# OSCM_PATH=/oscm/oscm-core/ +# CLIENT_CRT_PATH=/oscm/client-crt/ +# +# DB_VOLUME=oscm-db-volume +# OSCM_VOLUME=oscm-core-volume +# APP_VOLUME=oscm-app-volume +# +# mkdir -p $DB +# mkdir -p $APP_PATH +# mkdir -p $OSCM_PATH +# +# OSCM_DIR=/opt/oscm +# mkdir -p $OSCM_DIR + +# ------------------------------------------------------------------------------ + +#TODO: If containers are running, stop them + +# Create Docker directories #TODO: If they do not exist yet +mkdir -p ${DOCKER_PATH}/data/oscm-db/data +mkdir -p ${DOCKER_PATH}/config/brandings +mkdir -p ${DOCKER_PATH}/config/certs + +# Copy certificate files +if [ ${GENERATE_CERTIFICATES} == "false" ] && [ -f ${SSL_CONFIG_PATH}/oscm.crt ] && [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then + cp -f ${SSL_CONFIG_PATH}/oscm.crt ${DOCKER_PATH}/config/certs + cp -f ${SSL_CONFIG_PATH}/oscm.key ${DOCKER_PATH}/config/certs + if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then + cp -f ${SSL_CONFIG_PATH}/oscm.chain ${DOCKER_PATH}/config/certs + fi +else + # TODO: Only if there are no certificate and key files yet + mkdir ${SSL_CONFIG_PATH}/generate + openssl rand -base64 48 > ${SSL_CONFIG_PATH}/generate/passphrase.txt + openssl genrsa -aes128 -passout file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key 2048 + openssl req -new -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -key ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.csr -subj "/CN=${HOST_FQDN}" + cp ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.key.pass + openssl rsa -in ${SSL_CONFIG_PATH}/generate/oscm.key.pass -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key + openssl x509 -req -days 3650 -in ${SSL_CONFIG_PATH}/generate/oscm.csr -signkey ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.crt + rm -f ${SSL_CONFIG_PATH}/generate/passphrase.txt ${SSL_CONFIG_PATH}/generate/oscm.key.pass ${SSL_CONFIG_PATH}/generate/oscm.csr + cp -f ${SSL_CONFIG_PATH}/generate/oscm.key ${DOCKER_PATH}/config/certs + cp -f ${SSL_CONFIG_PATH}/generate/oscm.crt ${DOCKER_PATH}/config/certs + rm -f ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.crt + rmdir ${SSL_CONFIG_PATH}/generate fi -if ! file -Ls $volume_dev_app | grep -q "filesystem data" ; then - mkfs.ext4 "$volume_dev_app" +# Set Docker proxy +if [ ${PROXY_ENABLED} == "true" ]; then + mkdir -p /etc/systemd/system/docker.service.d + if [ ${PROXY_AUTH} == "true" ]; then + cat < /etc/systemd/system/docker.service.d/http-proxy.conf +[Service] +Environment="HTTP_PROXY=http://${PROXY_USER}:${PROXY_PWD}@${PROXY_HTTP_HOST}:${PROXY_HTTP_PORT}/" "HTTPS_PROXY=http://${PROXY_USER}:${PROXY_PWD}@${PROXY_HTTPS_HOST}:${PROXY_HTTPS_PORT}/" "NO_PROXY=${PROXY_NOPROXY}" +EOF + else + cat < /etc/systemd/system/docker.service.d/http-proxy.conf +[Service] +Environment="HTTP_PROXY=http://${PROXY_HTTP_HOST}:${PROXY_HTTP_PORT}/" "HTTPS_PROXY=http://${PROXY_HTTPS_HOST}:${PROXY_HTTPS_PORT}/" "NO_PROXY=${PROXY_NOPROXY}" +EOF + fi + systemctl daemon-reload + systemctl restart docker + #TODO: If disabled, disable proxy fi -mkdir -p /db -if ! grep -qs '/db' /proc/mounts; then - mount -t ext4 $volume_dev_db /db +# Create Docker registry authentication info +#TODO: Change check to variable DOCKER_REGISTRY_AUTH +if [ ! -z ${DOCKER_REGISTRY_USER} ] && [ ! -z ${DOCKER_REGISTRY_PWD} ]; then + #Create auth file for Docker registry + REGISTRY_USER_PASS_BASE64=$(printf "${DOCKER_REGISTRY_USER}:${DOCKER_REGISTRY_PWD}" | base64) + mkdir -p /root/.docker + cat < /root/.docker/config.json +{ + "auths": { + "${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}": { + "auth": "${REGISTRY_USER_PASS_BASE64}" + } + } +} +EOF +#TODO: If false, remove auth file fi -mkdir -p /oscm -if ! grep -qs '/oscm' /proc/mounts; then - mount -t ext4 $volume_dev_app /oscm +# Prepare Docker image registry variable +if [ ${DOCKER_REGISTRY_DOCKERHUB} = "true" ]; then + DOCKER_IMAGES_SOURCE="${DOCKER_REGISTRY_ORGANIZATION}/" + #TODO: Remove daemon.json +else + DOCKER_IMAGES_SOURCE="${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}/${DOCKER_REGISTRY_ORGANIZATION}/" + cat < /etc/docker/daemon.json +{ + "insecure-registries" : ["${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}"] +} +EOF + systemctl restart docker fi - -OSCM_IMAGE=$REGISTRY/$OSCM_IMAGE_NAME -APP_IMAGE=$REGISTRY/$APP_IMAGE_NAME -DB_IMAGE=$REGISTRY/$DB_IMAGE_NAME -DB=/db/database -APP_PATH=/oscm/oscm-app/ -OSCM_PATH=/oscm/oscm-core/ -CLIENT_CRT_PATH=/oscm/client-crt/ - -DB_VOLUME=oscm-db-volume -OSCM_VOLUME=oscm-core-volume -APP_VOLUME=oscm-app-volume - -mkdir -p $DB -mkdir -p $APP_PATH -mkdir -p $OSCM_PATH - -OSCM_DIR=/opt/oscm -mkdir -p $OSCM_DIR - -cat > /etc/motd < ${DOCKER_PATH}/docker-compose-initdb.yml +envsubst '$IMAGE_DB $DB_VOLUME_DATA_SRC $DB_PORT $IMAGE_CORE $IMAGE_APP $IMAGE_BIRT $IMAGE_BRANDING $BRANDING_VOLUME_BRANDINGS_SRC' \ + < ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml.template \ + > ${DOCKER_PATH}/docker-compose-oscm.yml +envsubst '$SMTP_HOST $SMTP_PORT $SMTP_FROM $SMTP_USER $SMTP_PWD $SMTP_AUTH $SMTP_TLS $KEY_SECRET $HOST_FQDN $DB_PORT $DB_PWD_CORE $DB_PWD_JMS $DB_PWD_APP $DB_SUPERPWD $APP_ADMIN_MAIL_ADDRESS $CONTROLLER_ID $CONTROLLER_ORG_ID $CONTROLLER_USER_KEY $CONTROLLER_USER_NAME $CONTROLLER_USER_PASS $TOMEE_DEBUG' \ + < ${COMPOSE_CONFIG_PATH}/var.env.template \ + > ${DOCKER_PATH}/var.env + +# Pull images +egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull +egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-oscm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull + +# Initialize databases +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up -d oscm-db +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-core +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-jms +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-app +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-controller-openstack +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml stop +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f + +# Start application containers +docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml up -d From c9d89ab02798fab63c93f97e1ca56b018303f017 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 6 Oct 2017 12:00:24 +0200 Subject: [PATCH 079/181] enhanced ssl certificate deployment --- .../files/default/user-data/deploy-oscmserver | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 89a44efe05..36a191955c 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -111,28 +111,34 @@ set +a mkdir -p ${DOCKER_PATH}/data/oscm-db/data mkdir -p ${DOCKER_PATH}/config/brandings mkdir -p ${DOCKER_PATH}/config/certs +mkdir -p ${DOCKER_PATH}/config/privkey/oscm-core +mkdir -p ${DOCKER_PATH}/config/privkey/oscm-app +mkdir -p ${DOCKER_PATH}/config/privkey/oscm-birt +mkdir -p ${DOCKER_PATH}/config/privkey/oscm-branding # Copy certificate files if [ ${GENERATE_CERTIFICATES} == "false" ] && [ -f ${SSL_CONFIG_PATH}/oscm.crt ] && [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then - cp -f ${SSL_CONFIG_PATH}/oscm.crt ${DOCKER_PATH}/config/certs - cp -f ${SSL_CONFIG_PATH}/oscm.key ${DOCKER_PATH}/config/certs + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then - cp -f ${SSL_CONFIG_PATH}/oscm.chain ${DOCKER_PATH}/config/certs + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain fi else - # TODO: Only if there are no certificate and key files yet - mkdir ${SSL_CONFIG_PATH}/generate - openssl rand -base64 48 > ${SSL_CONFIG_PATH}/generate/passphrase.txt - openssl genrsa -aes128 -passout file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key 2048 - openssl req -new -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -key ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.csr -subj "/CN=${HOST_FQDN}" - cp ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.key.pass - openssl rsa -in ${SSL_CONFIG_PATH}/generate/oscm.key.pass -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key - openssl x509 -req -days 3650 -in ${SSL_CONFIG_PATH}/generate/oscm.csr -signkey ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.crt - rm -f ${SSL_CONFIG_PATH}/generate/passphrase.txt ${SSL_CONFIG_PATH}/generate/oscm.key.pass ${SSL_CONFIG_PATH}/generate/oscm.csr - cp -f ${SSL_CONFIG_PATH}/generate/oscm.key ${DOCKER_PATH}/config/certs - cp -f ${SSL_CONFIG_PATH}/generate/oscm.crt ${DOCKER_PATH}/config/certs - rm -f ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.crt - rmdir ${SSL_CONFIG_PATH}/generate + # Generate self signed certificate if none exist yet + if [ ! -f ${DOCKER_PATH}/config/privkey/oscm-core/oscm.key ] && [ ! -f ${DOCKER_PATH}/config/privkey/oscm-core/oscm.crt ]; then + mkdir ${SSL_CONFIG_PATH}/generate + openssl rand -base64 48 > ${SSL_CONFIG_PATH}/generate/passphrase.txt + openssl genrsa -aes128 -passout file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key 2048 + openssl req -new -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -key ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.csr -subj "/CN=${HOST_FQDN}" + cp ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.key.pass + openssl rsa -in ${SSL_CONFIG_PATH}/generate/oscm.key.pass -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key + openssl x509 -req -days 3650 -in ${SSL_CONFIG_PATH}/generate/oscm.csr -signkey ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.crt + rm -f ${SSL_CONFIG_PATH}/generate/passphrase.txt ${SSL_CONFIG_PATH}/generate/oscm.key.pass ${SSL_CONFIG_PATH}/generate/oscm.csr + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/generate/oscm.key + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/generate/oscm.crt + rm -f ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.crt + rmdir ${SSL_CONFIG_PATH}/generate + fi fi # Set Docker proxy From e4c8635bbb4deccb108d944fed44f6df5841615d Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 6 Oct 2017 12:03:54 +0200 Subject: [PATCH 080/181] if containers are running, stop them --- .../oscm/files/default/user-data/deploy-oscmserver | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 36a191955c..584f58531f 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -105,7 +105,11 @@ set +a # ------------------------------------------------------------------------------ -#TODO: If containers are running, stop them +# If containers are running, stop them +if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0"]; then + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f +fi # Create Docker directories #TODO: If they do not exist yet mkdir -p ${DOCKER_PATH}/data/oscm-db/data From 2adcc5f219ccdd9d054bac0c8e2f3e979bd852be Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 6 Oct 2017 12:24:15 +0200 Subject: [PATCH 081/181] added checks to make re-run and upgrade with deployment script possible --- .../files/default/user-data/deploy-oscmserver | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 584f58531f..ef37aad120 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -111,14 +111,12 @@ if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) ! docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f fi -# Create Docker directories #TODO: If they do not exist yet -mkdir -p ${DOCKER_PATH}/data/oscm-db/data -mkdir -p ${DOCKER_PATH}/config/brandings -mkdir -p ${DOCKER_PATH}/config/certs -mkdir -p ${DOCKER_PATH}/config/privkey/oscm-core -mkdir -p ${DOCKER_PATH}/config/privkey/oscm-app -mkdir -p ${DOCKER_PATH}/config/privkey/oscm-birt -mkdir -p ${DOCKER_PATH}/config/privkey/oscm-branding +# Create Docker directories if they do not exist yet +for docker_directory in ${DOCKER_PATH}/data/oscm-db/data ${DOCKER_PATH}/config/brandings ${DOCKER_PATH}/config/certs ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding; do + if [ ! -d $docker_directory ]; then + mkdir -p $docker_directory + fi +done # Copy certificate files if [ ${GENERATE_CERTIFICATES} == "false" ] && [ -f ${SSL_CONFIG_PATH}/oscm.crt ] && [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then @@ -145,9 +143,11 @@ else fi fi -# Set Docker proxy +# If http proxy is enabled, create Docker config file if [ ${PROXY_ENABLED} == "true" ]; then - mkdir -p /etc/systemd/system/docker.service.d + if [ ! -d /etc/systemd/system/docker.service.d]; then + mkdir -p /etc/systemd/system/docker.service.d + fi if [ ${PROXY_AUTH} == "true" ]; then cat < /etc/systemd/system/docker.service.d/http-proxy.conf [Service] @@ -161,12 +161,17 @@ EOF fi systemctl daemon-reload systemctl restart docker - #TODO: If disabled, disable proxy +# Otherwise remove Docker config file with proxy config +else + if [ -f /etc/systemd/system/docker.service.d/http-proxy.conf ]; then + rm -f /etc/systemd/system/docker.service.d/http-proxy.conf + systemctl daemon-reload + systemctl restart docker + fi fi -# Create Docker registry authentication info -#TODO: Change check to variable DOCKER_REGISTRY_AUTH -if [ ! -z ${DOCKER_REGISTRY_USER} ] && [ ! -z ${DOCKER_REGISTRY_PWD} ]; then +# If Docker registry authentication is enabled, create auth info file +if [ ${DOCKER_REGISTRY_AUTH} == "true" ]; then #Create auth file for Docker registry REGISTRY_USER_PASS_BASE64=$(printf "${DOCKER_REGISTRY_USER}:${DOCKER_REGISTRY_PWD}" | base64) mkdir -p /root/.docker @@ -179,15 +184,22 @@ if [ ! -z ${DOCKER_REGISTRY_USER} ] && [ ! -z ${DOCKER_REGISTRY_PWD} ]; then } } EOF -#TODO: If false, remove auth file +# Otherwise remove auth info file +else + if [ -f /root/.docker/config.json ]; then + rm -f /root/.docker/config.json + fi fi -# Prepare Docker image registry variable +# If default Docker registry (Dockerhub) is used, remove insecure registry config if [ ${DOCKER_REGISTRY_DOCKERHUB} = "true" ]; then DOCKER_IMAGES_SOURCE="${DOCKER_REGISTRY_ORGANIZATION}/" - #TODO: Remove daemon.json + if [ -f /etc/docker/daemon.json ]; then + rm -f /etc/docker/daemon.json + systemctl restart docker + fi +# Otherwise, custom registry is used, so create insecure registry config else - DOCKER_IMAGES_SOURCE="${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}/${DOCKER_REGISTRY_ORGANIZATION}/" cat < /etc/docker/daemon.json { "insecure-registries" : ["${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}"] From d016cd8b72a9634c4bbc5d18bb0baf960a37a416 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 6 Oct 2017 12:25:01 +0200 Subject: [PATCH 082/181] removed creation of docker compose files from deployment script - will be done by Barclamp --- .../oscm/files/default/user-data/deploy-oscmserver | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index ef37aad120..e6852d5ed8 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -208,18 +208,6 @@ EOF systemctl restart docker fi -# Create docker-compose files from templates -#TODO: Will be provided by Barclamp -envsubst '$IMAGE_DB $DB_VOLUME_DATA_SRC $IMAGE_INITDB' \ - < ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml.template \ - > ${DOCKER_PATH}/docker-compose-initdb.yml -envsubst '$IMAGE_DB $DB_VOLUME_DATA_SRC $DB_PORT $IMAGE_CORE $IMAGE_APP $IMAGE_BIRT $IMAGE_BRANDING $BRANDING_VOLUME_BRANDINGS_SRC' \ - < ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml.template \ - > ${DOCKER_PATH}/docker-compose-oscm.yml -envsubst '$SMTP_HOST $SMTP_PORT $SMTP_FROM $SMTP_USER $SMTP_PWD $SMTP_AUTH $SMTP_TLS $KEY_SECRET $HOST_FQDN $DB_PORT $DB_PWD_CORE $DB_PWD_JMS $DB_PWD_APP $DB_SUPERPWD $APP_ADMIN_MAIL_ADDRESS $CONTROLLER_ID $CONTROLLER_ORG_ID $CONTROLLER_USER_KEY $CONTROLLER_USER_NAME $CONTROLLER_USER_PASS $TOMEE_DEBUG' \ - < ${COMPOSE_CONFIG_PATH}/var.env.template \ - > ${DOCKER_PATH}/var.env - # Pull images egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-oscm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull From 4161a226fb5d9ad2374704ce64d0503a0faca245 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 6 Oct 2017 12:28:37 +0200 Subject: [PATCH 083/181] removed superfluous variables from deployment script --- .../files/default/user-data/deploy-oscmserver | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index e6852d5ed8..5d608f823b 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -1,6 +1,9 @@ #!/bin/sh +# Enable command traces set -x +# Exit on error set -e +# Redirect all output to log file exec &> /var/log/setup-machine.log # Variables for this script @@ -10,30 +13,6 @@ SSL_CONFIG_PATH=${CONFIG_BASE}/ssl COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose DOCKER_PATH=/docker -# Fixed values; neither from oscm-config nor heat-config -export DB_PORT=5432 # Fixed by deployment script: 5432 -export DB_PWD_JMS=jmsuser # Fixed by deployment script: jmsuser -export CONTROLLER_ORG_ID=PLATFORM_OPERATOR # Fixed by deployment script: PLATFORM_OPERATOR -export CONTROLLER_USER_KEY=1000 # Fixed by deployment script: 1000 -export CONTROLLER_USER_NAME=administrator # Fixed by deployment script: administrator -export CONTROLLER_USER_PASS=admin123 # Fixed by deployment script: admin123 -export DB_VOLUME_DATA_SRC=${DOCKER_PATH}/data/oscm-db/data -export BRANDING_VOLUME_BRANDINGS_SRC=${DOCKER_PATH}/config/brandings -export TOMEE_DEBUG=false - -# To be removed from this script -export IMAGE_DB=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-db:soc -export IMAGE_CORE=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-core:soc -export IMAGE_APP=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-app:soc -export IMAGE_BIRT=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-birt:soc -export IMAGE_BRANDING=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-branding:soc -export IMAGE_INITDB=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-initdb:soc -export KEY_SECRET=secret # Generate with Barclamp (heat-config) -export HOST_FQDN=estshimizusles.intern.est.fujitsu.com # Operator set or generated in Barclamp (oscm-config) -export DB_PWD_CORE=bssuser # Generate with Barclamp (heat-config) -export DB_PWD_APP=bssappuser # Generate with Barclamp (heat-config) -export DB_SUPERPWD=postgres # Generate with Barclamp (heat-config) - # Wait for remote copying to finish if [ ${CONFIG_PATH} ] && [ ${SSL_CONFIG_PATH} ] && [ ${COMPOSE_CONFIG_PATH} ]; then until [ -f ${CONFIG_PATH}/finished ] && [ -f ${SSL_CONFIG_PATH}/finished ] && [ -f ${COMPOSE_CONFIG_PATH}/finished ] From 75e4a4076c8f448c07ddea4bdd23c5e2a697afa4 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 9 Oct 2017 14:11:42 +0200 Subject: [PATCH 084/181] removed unnecessary wait loop from deploy script --- .../oscm/files/default/user-data/deploy-oscmserver | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 5d608f823b..13d80b4109 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -13,16 +13,6 @@ SSL_CONFIG_PATH=${CONFIG_BASE}/ssl COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose DOCKER_PATH=/docker -# Wait for remote copying to finish -if [ ${CONFIG_PATH} ] && [ ${SSL_CONFIG_PATH} ] && [ ${COMPOSE_CONFIG_PATH} ]; then - until [ -f ${CONFIG_PATH}/finished ] && [ -f ${SSL_CONFIG_PATH}/finished ] && [ -f ${COMPOSE_CONFIG_PATH}/finished ] - do - echo "scp for oscm configuration files is not finished yet - sleeping 5s" - sleep 5 - done - echo "scp for oscm configuration files is finished." -fi - # Enable automatic exporting of variables set -a # Read configuration files From d1a240e9a132f2c6d807b946ea13a56a63b689a0 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 9 Oct 2017 14:16:16 +0200 Subject: [PATCH 085/181] deploy script: copy prepared Docker Compose files --- .../oscm/files/default/user-data/deploy-oscmserver | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 13d80b4109..6891de2956 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -87,6 +87,14 @@ for docker_directory in ${DOCKER_PATH}/data/oscm-db/data ${DOCKER_PATH}/config/b fi done +# Copy Docker Compose files +if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then + cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml +fi +if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ]; then + cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ${DOCKER_PATH}/docker-compose-oscm.yml +fi + # Copy certificate files if [ ${GENERATE_CERTIFICATES} == "false" ] && [ -f ${SSL_CONFIG_PATH}/oscm.crt ] && [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key From e24a5d357a444e5b62ef3d611dc6b4f653da6261 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 10 Oct 2017 09:42:17 +0200 Subject: [PATCH 086/181] added logging configuration to deploy script --- .../files/default/user-data/deploy-oscmserver | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 6891de2956..11859787c2 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -75,18 +75,58 @@ set +a # ------------------------------------------------------------------------------ # If containers are running, stop them -if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0"]; then - docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop - docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f +if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml]; then + if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0"]; then + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f + fi fi # Create Docker directories if they do not exist yet -for docker_directory in ${DOCKER_PATH}/data/oscm-db/data ${DOCKER_PATH}/config/brandings ${DOCKER_PATH}/config/certs ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding; do - if [ ! -d $docker_directory ]; then - mkdir -p $docker_directory +for docker_directory in \ + ${DOCKER_PATH}/data/oscm-db/data \ + ${DOCKER_PATH}/config/brandings\ + ${DOCKER_PATH}/config/certs \ + ${DOCKER_PATH}/config/privkey/oscm-core \ + ${DOCKER_PATH}/config/privkey/oscm-app \ + ${DOCKER_PATH}/config/privkey/oscm-birt \ + ${DOCKER_PATH}/config/privkey/oscm-branding \ + ${DOCKER_PATH}/logs/oscm-app \ + ${DOCKER_PATH}/logs/oscm-birt \ + ${DOCKER_PATH}/logs/oscm-branding \ + ${DOCKER_PATH}/logs/oscm-core \ + ${DOCKER_PATH}/logs/oscm-db; do + if [ ! -d ${docker_directory} ]; then + mkdir -p ${docker_directory} + fi +done + +# Create Docker log files if they do not exist yet +for docker_log_file in \ + ${DOCKER_PATH}/logs/oscm-app/oscm-app.out.log \ + ${DOCKER_PATH}/logs/oscm-birt/oscm-birt.out.log \ + ${DOCKER_PATH}/logs/oscm-branding/oscm-branding.out.log \ + ${DOCKER_PATH}/logs/oscm-core/oscm-core.out.log \ + ${DOCKER_PATH}/logs/oscm-db/oscm-db.out.log; do + if [ ! -f {docker_log_file} ]; then + touch ${docker_log_file} + chmod 640 ${docker_log_file} fi done +# Create rsyslog configuration for Docker log files +if [ ! -f /etc/rsyslog.d/oscm.conf ]; then + cat < /etc/rsyslog.d/oscm.conf +local0.* /docker/logs/oscm-db/oscm-db.out.log +local1.* /docker/logs/oscm-core/oscm-core.out.log +local2.* /docker/logs/oscm-app/oscm-app.out.log +local3.* /docker/logs/oscm-birt/oscm-birt.out.log +local4.* /docker/logs/oscm-branding/oscm-branding.out.log +EOF + sed -i '/^local/d' /etc/rsyslog.conf + systemctl restart rsyslog +fi + # Copy Docker Compose files if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml @@ -122,7 +162,7 @@ fi # If http proxy is enabled, create Docker config file if [ ${PROXY_ENABLED} == "true" ]; then - if [ ! -d /etc/systemd/system/docker.service.d]; then + if [ ! -d /etc/systemd/system/docker.service.d ]; then mkdir -p /etc/systemd/system/docker.service.d fi if [ ${PROXY_AUTH} == "true" ]; then @@ -140,6 +180,7 @@ EOF systemctl restart docker # Otherwise remove Docker config file with proxy config else + echo NOPE if [ -f /etc/systemd/system/docker.service.d/http-proxy.conf ]; then rm -f /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload From b3fdbd15780ff42c9e7e1e9190d56cc7495456be Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 10 Oct 2017 11:33:59 +0200 Subject: [PATCH 087/181] SSL certs and update deploy --- chef/cookbooks/oscm/recipes/server.rb | 114 ++++++++++-------- .../default/docker-compose-initdb.yml.erb | 10 +- .../default/docker-compose-oscm.yml.erb | 30 ++++- chef/data_bags/crowbar/template-oscm.json | 3 +- chef/data_bags/crowbar/template-oscm.schema | 1 + .../barclamp/oscm/_edit_attributes.html.haml | 8 +- crowbar_framework/config/locales/oscm/en.yml | 5 +- 7 files changed, 108 insertions(+), 63 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 1e8c5c450b..8c47bec841 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -78,6 +78,20 @@ heat_insecure = CrowbarOpenStackHelper.insecure(heat_config) openstack_args_heat = heat_insecure || keystone_settings["insecure"] ? "--insecure" : "" +ruby_block "check_oscm_glance_image" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} image list -c Name -f value | egrep '^#{oscm_image}$'" + command_out = shell_out(command) + if command_out.stdout.strip != oscm_image + message = "The image with name '#{oscm_image}' is not found in glance! Please check your oscm proposal attributes or glance image registry." + raise message + end + end + action :create +end + + register_auth_hash = { user: keystone_settings["admin_user"], password: keystone_settings["admin_password"], @@ -151,21 +165,10 @@ action :add_access end -if node[:oscm][:api][:protocol] == "https" - ssl_setup "setting up ssl for oscm" do - generate_certs node[:oscm][:ssl][:generate_certs] - certfile node[:oscm][:ssl][:certfile] - keyfile node[:oscm][:ssl][:keyfile] - group oscm_group - fqdn node[:fqdn] - ca_certs node[:oscm][:ssl][:ca_certs] - end -end - execute "create_oscm_flavor" do command "#{openstack_cmd} #{openstack_args_nova} flavor create --ram #{oscm_flavor_ram} --disk #{oscm_flavor_disk} \ --vcpus #{oscm_flavor_vcpus} --private #{oscm_flavor_name}" - not_if "#{openstack_cmd} #{openstack_args_nova} flavor list --all | grep -q #{oscm_flavor_name}" + not_if "#{openstack_cmd} #{openstack_args_nova} flavor list --all -c Name -f value | egrep -q '^#{oscm_flavor_name}$'" end execute "create_oscm_flavor_access" do @@ -181,20 +184,15 @@ EOH end -execute "delete_oscm_keypair" do - command "#{openstack_cmd} #{openstack_args_nova} keypair delete #{oscm_keypair_name}" - only_if "#{openstack_cmd} #{openstack_args_nova} keypair list | grep -q #{oscm_keypair_name}" -end - execute "create_oscm_keypair" do command "#{openstack_cmd} #{openstack_args_nova} keypair create #{oscm_keypair_name} --public-key #{oscm_keypair_publickeyfile}" - not_if "#{openstack_cmd} #{openstack_args_nova} keypair list | grep -q #{oscm_keypair_name}" + not_if "#{openstack_cmd} #{openstack_args_nova} keypair list -c Name -f value | egrep -q '^#{oscm_keypair_name}$'" end directory "#{oscm_install_path}" do owner oscm_group group oscm_group - mode 0755 + mode 0640 recursive true end @@ -202,7 +200,7 @@ source "volumes.yaml" owner oscm_group group oscm_group - mode 0755 + mode 0640 action :create end @@ -210,14 +208,14 @@ source "application.yaml" owner oscm_group group oscm_group - mode 0755 + mode 0640 action :create end directory "#{oscm_install_path}/user-data" do owner oscm_group group oscm_group - mode 0755 + mode 0640 recursive true end @@ -225,7 +223,7 @@ source "user-data/heat-config" owner oscm_group group oscm_group - mode 0755 + mode 0640 action :create end @@ -233,14 +231,14 @@ source "user-data/deploy-oscmserver" owner oscm_group group oscm_group - mode 0755 + mode 0640 action :create end execute "create_oscm_volume_stack" do command "#{openstack_cmd} #{openstack_args_heat} stack create --parameter db_size=#{oscm_db_volume_size} --parameter app_size=#{oscm_app_volume_size} \ -t #{oscm_install_path}/volumes.yaml --wait #{oscm_volumestack_name}" - not_if "#{openstack_cmd} #{openstack_args_heat} stack list | grep -q #{oscm_volumestack_name}" + not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{oscm_volumestack_name}$'" end ruby_block "get_oscm_volume_ids" do @@ -261,7 +259,7 @@ Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) command = "mkdir -p '$(dirname #{oscm_keypair_crowbar_sshkey})'" command_out = shell_out(command) - command = "yes y | ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} -N ''" + command = "[ ! -f #{oscm_keypair_crowbar_sshkey}] && yes y | ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} -N ''" command_out = shell_out(command) end action :create @@ -274,7 +272,7 @@ --parameter mail_port=#{oscm_mail_port} --parameter registry_port=#{oscm_docker_port} \ --parameter-file ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub \ -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name}" } - not_if "#{openstack_cmd} #{openstack_args_heat} stack list | grep -q #{oscm_instancestack_name}" + not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{oscm_instancestack_name}$'" end ruby_block "get_oscm_floating_ip" do @@ -287,6 +285,17 @@ action :create end +if node[:oscm][:api][:protocol] == "https" + ssl_setup "setting up ssl for oscm" do + generate_certs node[:oscm][:ssl][:generate_certs] + certfile node[:oscm][:ssl][:certfile] + keyfile node[:oscm][:ssl][:keyfile] + group oscm_group + fqdn node[:oscm][:ssl][:fqdn].empty? ? node[:oscm][:openstack][:instance_stack][:ip_appserver] : node[:oscm][:ssl][:fqdn] + ca_certs node[:oscm][:ssl][:ca_certs] + end +end + ruby_block "get_oscm_secrets" do block do Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) @@ -315,7 +324,7 @@ mail: node[:oscm][:mail], docker: node[:oscm][:docker], proxy: node[:oscm][:proxy], - host_fqdn: node[:oscm][:host_fqdn], + host_fqdn: node[:oscm][:ssl][:fqdn].empty? ? node[:oscm][:openstack][:instance_stack][:ip_appserver] : node[:oscm][:ssl][:fqdn], instance: node[:oscm][:openstack][:instance_stack] ) end @@ -349,27 +358,38 @@ mail: node[:oscm][:mail], docker: node[:oscm][:docker], proxy: node[:oscm][:proxy], - host_fqdn: node[:oscm][:host_fqdn], + host_fqdn: node[:oscm][:ssl][:fqdn].empty? ? node[:oscm][:openstack][:instance_stack][:ip_appserver] : node[:oscm][:ssl][:fqdn], instance: node[:oscm][:openstack][:instance_stack] ) end -bash "inject oscm configuration and certificates" do - code <<-EOH - ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}" || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/oscm-config ${ip_appserver}:#{oscm_config_path} || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/user-data/deploy-oscmserver ${ip_appserver}:#{oscm_config_path} || true - ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/finished" - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/docker-compose-initdb.yml ${ip_appserver}:#{oscm_config_path}/docker-compose || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_install_path}/docker-compose-oscm.yml ${ip_appserver}:#{oscm_config_path}/docker-compose || true - ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/docker-compose/finished" - if [ -f #{oscm_ssl_certfile} ]; - then - ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "mkdir -p #{oscm_config_path}/ssl/" || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_certfile} ${ip_appserver}:#{oscm_config_path}/ssl/oscm.crt || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_keyfile} ${ip_appserver}:#{oscm_config_path}/ssl/oscm.key || true - scp -i #{oscm_keypair_crowbar_sshkey} #{oscm_ssl_cacerts} ${ip_appserver}:#{oscm_config_path}/ssl/oscm.chain || true - ssh -i #{oscm_keypair_crowbar_sshkey} ${ip_appserver} "touch #{oscm_config_path}/ssl/finished" - fi - EOH +ruby_block "inject_oscm_scripts" do + block do + args = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i #{oscm_keypair_crowbar_sshkey}" + ip_appserver = node[:oscm][:openstack][:instance_stack][:ip_appserver] + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_config_path}/docker-compose'" + command_out = shell_out(command) + command = "scp #{args} #{oscm_install_path}/user-data/oscm-config #{ip_appserver}:#{oscm_config_path}" + command_out = shell_out(command) + command = "scp #{args} #{oscm_install_path}/docker-compose-*.yml #{ip_appserver}:#{oscm_config_path}/docker-compose" + command_out = shell_out(command) + if node[:oscm][:api][:protocol] == "https" + command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_config_path}/ssl'" + command_out = shell_out(command) + command = "scp #{args} #{oscm_ssl_certfile} #{ip_appserver}:#{oscm_config_path}/ssl/oscm.crt" + command_out = shell_out(command) + command = "scp #{args} #{oscm_ssl_keyfile} #{ip_appserver}:#{oscm_config_path}/ssl/oscm.key" + command_out = shell_out(command) + command = "scp #{args} #{oscm_ssl_cacerts} #{ip_appserver}:#{oscm_config_path}/ssl/oscm.chain" + end + command = "scp #{args} #{oscm_install_path}/user-data/deploy-oscmserver #{ip_appserver}:#{oscm_config_path}" + command_out = shell_out(command) + command = "ssh #{args} #{ip_appserver} 'chmod 755 #{oscm_config_path}/deploy-oscmserver'" + command_out = shell_out(command) + command = "ssh #{args} #{ip_appserver} '#{oscm_config_path}/deploy-oscmserver'" + command_out = shell_out(command) + end + action :create end + diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb index 6c68bf9996..d80f2112f6 100644 --- a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb +++ b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb @@ -4,7 +4,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% end %> container_name: oscm-db volumes: @@ -14,7 +14,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-core env_file: var.env @@ -29,7 +29,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-jms env_file: var.env @@ -44,7 +44,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-app env_file: var.env @@ -59,7 +59,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-controller-openstack env_file: var.env diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb index 7b6ad66ad9..7e7b880a14 100644 --- a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb +++ b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb @@ -4,9 +4,13 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% end %> container_name: oscm-db + logging: + driver: syslog + options: + syslog-facility: "local0" volumes: - /docker/data/oscm-db/data:/var/lib/postgresql/data ports: @@ -16,9 +20,13 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-core:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> <% end %> container_name: oscm-core + logging: + driver: syslog + options: + syslog-facility: "local1" env_file: var.env environment: - JPDA_ADDRESS=8000 @@ -36,9 +44,13 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-app:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> <% end %> container_name: oscm-app + logging: + driver: syslog + options: + syslog-facility: "local2" env_file: var.env environment: - JPDA_ADDRESS=8000 @@ -56,9 +68,13 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-birt:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> <% end %> container_name: oscm-birt + logging: + driver: syslog + options: + syslog-facility: "local3" env_file: var.env volumes: - /docker/config/oscm-birt/privkey:/privkey @@ -70,9 +86,13 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-branding:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"]+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> <% end %> container_name: oscm-branding + logging: + driver: syslog + options: + syslog-facility: "local4" env_file: var.env volumes: - /docker/config/brandings:/brandings:ro diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 75399885c6..96f6dabf95 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -69,11 +69,12 @@ "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", "keyfile": "/etc/oscm/ssl/private/signing_key.pem", "generate_certs": false, + "fqdn": "", "insecure": false, "ca_certs": "/etc/oscm/ssl/certs/ca.pem" }, "api": { - "protocol": "http" + "protocol": "https" } } }, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index f7a25d72da..ccaf0b1639 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -117,6 +117,7 @@ "certfile": { "type" : "str", "required" : true }, "keyfile": { "type" : "str", "required" : true }, "generate_certs": { "type" : "bool", "required" : true }, + "fqdn": { "type" : "str", "required" : true }, "insecure": { "type" : "bool", "required" : true }, "ca_certs": { "type" : "str", "required" : true } } diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index eb54d0d918..21a9cb5f52 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -79,8 +79,6 @@ %legend = t(".ssl_header") - = string_field %w(host_fqdn) - = select_field %w(api protocol), :collection => :api_protocols_for_oscm, "data-sslprefix" => "ssl", @@ -88,8 +86,12 @@ "data-sslkey" => "/etc/oscm/ssl/private/signing_key.pem" #ssl_container + %span.help-block + = t('.ssl_hint') + = boolean_field %w(ssl generate_certs) + = string_field %w(ssl fqdn) = string_field %w(ssl certfile) = string_field %w(ssl keyfile) = boolean_field %w(ssl insecure) = string_field %w(ssl ca_certs) - + diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index f43d120608..689bc6a084 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -53,15 +53,16 @@ en: openstack: keypair: publickey: "Public Key" - keypair_hint: 'SSH public key for the openstack instance where the OSCM will be deployed' + keypair_hint: 'SSH public key of the keypair for openstack instance which hosts OSCM. It cannot be updated.' api: protocol: 'Protocol' ssl_header: 'SSL Support' - host_fqdn: 'OSCM Host FQDN (if left empty, the floating IP will be used)' ssl: generate_certs: 'Generate (self-signed) certificates (implies insecure)' + fqdn: 'Host FQDN (if left empty, the floating IP will be used)' certfile: 'SSL Certificate File' keyfile: 'SSL (Private) Key File' insecure: 'SSL Certificate is insecure (for instance, self-signed)' ca_certs: 'SSL CA Certificates File' + ssl_hint: 'Delete the SSL key pair on the given locations, in order to generate new one.' \ No newline at end of file From 7a2c8bc9fd53895caa8926c1e4a816a89a04acf0 Mon Sep 17 00:00:00 2001 From: stavreva Date: Tue, 10 Oct 2017 16:10:32 +0200 Subject: [PATCH 088/181] Rename volumes --- .../oscm/files/default/application.yaml | 28 ++++++++-------- .../oscm/files/default/user-data/heat-config | 4 +-- .../cookbooks/oscm/files/default/volumes.yaml | 32 +++++++++---------- chef/cookbooks/oscm/recipes/server.rb | 18 +++++------ .../oscm/templates/default/oscm.conf.erb | 5 +-- .../oscm/templates/default/var.env.erb | 5 --- chef/data_bags/crowbar/template-oscm.json | 4 +-- chef/data_bags/crowbar/template-oscm.schema | 4 +-- 8 files changed, 46 insertions(+), 54 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/oscm/files/default/application.yaml index 4e9446f3ed..78a5e5e898 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/oscm/files/default/application.yaml @@ -1,4 +1,4 @@ -# heat stack-create --poll -f application.yaml -P app_volume_id=$app_volume_id -P db_volume_id=$db_volume_id -P key_name=default -P flavor=d1.tiny oscm +# heat stack-create --poll -f application.yaml -P logs_volume_id=$logs_volume_id -P data_volume_id=$data_volume_id -P key_name=default -P flavor=d1.tiny oscm heat_template_version: 2015-10-15 description: | @@ -7,11 +7,11 @@ description: | heat stack-create --poll -f volumes.yaml oscm # 2) Retrieve outputs from oscm: - app_volume_id=$(heat output-show oscm app_volume_id | sed 's/"//g') - db_volume_id=$(heat output-show oscm db_volume_id | sed 's/"//g') + logs_volume_id=$(heat output-show oscm logs_volume_id | sed 's/"//g') + data_volume_id=$(heat output-show oscm data_volume_id | sed 's/"//g') # 3) Create application stack: - heat stack-create --poll -f application.yaml -P app_volume_id=$app_volume_id -P db_volume_id=$db_volume_id -P key_name= oscm + heat stack-create --poll -f application.yaml -P logs_volume_id=$logs_volume_id -P data_volume_id=$data_volume_id -P key_name= oscm @@ -32,10 +32,10 @@ parameters: type: string default: oscm description: Keypair name - app_volume_id: + logs_volume_id: type: string description: The Cinder volume to attach to the application server - db_volume_id: + data_volume_id: type: string description: The Cinder volume to attach to the database server registry_port: @@ -151,16 +151,16 @@ resources: str_replace: template: {get_file: user-data/heat-config} params: - $APP_VOLUME_DEV: + $LOGS_VOLUME_DEV: list_join: - "" - - "/dev/disk/by-id/virtio-" - - { get_param: app_volume_id } - $DB_VOLUME_DEV: + - { get_param: logs_volume_id } + $DATA_VOLUME_DEV: list_join: - "" - - "/dev/disk/by-id/virtio-" - - { get_param: db_volume_id } + - { get_param: data_volume_id } $SSH_CERT: { get_param: ssh_cert } # User data payload for appserver @@ -200,17 +200,17 @@ resources: floating_network: get_param: floating_network - app_volume_attachment: + logs_volume_attachment: type: OS::Cinder::VolumeAttachment properties: instance_uuid: { get_resource: appserver } - volume_id: { get_param: app_volume_id } + volume_id: { get_param: logs_volume_id } - db_volume_attachment: + data_volume_attachment: type: OS::Cinder::VolumeAttachment properties: instance_uuid: { get_resource: appserver } - volume_id: { get_param: db_volume_id } + volume_id: { get_param: data_volume_id } outputs: # Retrieval after stack creation (presuming stack is named `mystack`): diff --git a/chef/cookbooks/oscm/files/default/user-data/heat-config b/chef/cookbooks/oscm/files/default/user-data/heat-config index 835b27b5ed..95500af31c 100644 --- a/chef/cookbooks/oscm/files/default/user-data/heat-config +++ b/chef/cookbooks/oscm/files/default/user-data/heat-config @@ -6,8 +6,8 @@ touch /etc/oscm/config/heat-config chmod 400 /etc/oscm/config/heat-config cat > /etc/oscm/config/heat-config < DOCKER_OSCM_TAG=<%= @docker["tag"] %> #host -<% if @host_fqdn.strip.empty? -%> -HOST_FQDN=<%= @instance["ip_appserver"] %> -<% else %> HOST_FQDN=<%= @host_fqdn %> -<% end %> + \ No newline at end of file diff --git a/chef/cookbooks/oscm/templates/default/var.env.erb b/chef/cookbooks/oscm/templates/default/var.env.erb index 2b45afbd22..41e5519a69 100644 --- a/chef/cookbooks/oscm/templates/default/var.env.erb +++ b/chef/cookbooks/oscm/templates/default/var.env.erb @@ -6,12 +6,7 @@ SMTP_PWD=<%= @mail["password"] %> SMTP_FROM=<%= @mail["from"] %> SMTP_AUTH=<%= @mail["auth"] %> - -<% if @host_fqdn.strip.empty? -%> -HOST_FQDN=<%= @instance["ip_appserver"] %> -<% else %> HOST_FQDN=<%= @host_fqdn %> -<% end %> DB_PORT_CORE=5432 DB_PORT_JMS=5432 diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 96f6dabf95..365a8e95e3 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -27,8 +27,8 @@ }, "volume_stack": { "stack_name": "oscm-volumes", - "db_volume_size": 1, - "app_volume_size": 1 + "data_volume_size": 1, + "logs_volume_size": 1 }, "instance_stack": { "stack_name": "oscm-instances" diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index ccaf0b1639..87fa13f853 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -54,8 +54,8 @@ "required" : true, "mapping": { "stack_name": { "type": "str", "required": true }, - "db_volume_size": { "type": "int", "required": true }, - "app_volume_size": { "type": "int", "required": true } + "data_volume_size": { "type": "int", "required": true }, + "logs_volume_size": { "type": "int", "required": true } } }, "instance_stack" : { From d9df8ac518a6a603eb1b6f092db2f0c623fc8fd1 Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 11 Oct 2017 12:24:01 +0200 Subject: [PATCH 089/181] Add missing settings --- chef/cookbooks/oscm/recipes/server.rb | 22 +++---------------- chef/data_bags/crowbar/template-oscm.json | 3 ++- chef/data_bags/crowbar/template-oscm.schema | 3 ++- .../barclamp/oscm/_edit_attributes.html.haml | 1 + crowbar_framework/config/locales/oscm/en.yml | 1 + 5 files changed, 9 insertions(+), 21 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 219d04877f..592647aa52 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -34,24 +34,7 @@ oscm_data_volume_size = node[:oscm][:openstack][:volume_stack][:data_volume_size] oscm_logs_volume_size = node[:oscm][:openstack][:volume_stack][:logs_volume_size] oscm_image = node[:oscm][:openstack][:image] -oscm_docker_host = node[:oscm][:docker][:host] -oscm_docker_port = node[:oscm][:docker][:port] -oscm_docker_user = node[:oscm][:docker][:user] -oscm_docker_pwd = node[:oscm][:docker][:password] -oscm_docker_tag = node[:oscm][:docker][:tag] -oscm_proxy_httphost = node[:oscm][:proxy][:http_host] -oscm_proxy_httpport = node[:oscm][:proxy][:http_port] -oscm_proxy_httpshost = node[:oscm][:proxy][:https_host] -oscm_proxy_httpsport = node[:oscm][:proxy][:https_port] -oscm_proxy_user = node[:oscm][:proxy][:user] -oscm_proxy_pwd = node[:oscm][:proxy][:password] -oscm_mail_host = node[:oscm][:mail][:host] -oscm_mail_port = node[:oscm][:mail][:port] -oscm_mail_tls = node[:oscm][:mail][:tls] -oscm_mail_from = node[:oscm][:mail][:from] -oscm_mail_auth = node[:oscm][:mail][:auth] -oscm_mail_user = node[:oscm][:mail][:user] -oscm_mail_pwd = node[:oscm][:mail][:password] +oscm_floating_network = node[:oscm][:openstack][:floating_network] oscm_keypair_crowbar_sshkey = "/etc/oscm/install/oscm_ssh.key" oscm_group = "root" @@ -269,7 +252,8 @@ command lazy { "#{openstack_cmd} #{openstack_args_heat} stack create --parameter logs_volume_id=#{node[:oscm][:openstack][:volume_stack][:logs_volume_id]} \ --parameter data_volume_id=#{node[:oscm][:openstack][:volume_stack][:data_volume_id]} \ --parameter image=#{oscm_image} --parameter flavor=#{oscm_flavor_name} \ - --parameter mail_port=#{oscm_mail_port} --parameter registry_port=#{oscm_docker_port} \ + --parameter key_name=#{oscm_keypair_name} --parameter floating_network=#{oscm_floating_network} \ + --parameter mail_port=#{node[:oscm][:mail][:port]} --parameter registry_port=#{node[:oscm][:docker][:port]} \ --parameter-file ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub \ -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name}" } not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{oscm_instancestack_name}$'" diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-oscm.json index 365a8e95e3..6fa79a8620 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-oscm.json @@ -32,7 +32,8 @@ }, "instance_stack": { "stack_name": "oscm-instances" - } + }, + "floating_network": "floating" }, "mail": { "auth": true, diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index 87fa13f853..a2e8234275 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -64,7 +64,8 @@ "mapping": { "stack_name": { "type": "str", "required": true } } - } + }, + "floating-network" : { "type": "str", "required": true } } }, "mail" : { diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml index 21a9cb5f52..373955de41 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml @@ -32,6 +32,7 @@ #dockerhub_container = string_field %w(docker host) = integer_field %w(docker port) + = string_field %w(docker organization) = boolean_field %w(docker auth), "data-showit" => "true", diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/oscm/en.yml index 689bc6a084..e0c4b5f6a0 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/oscm/en.yml @@ -32,6 +32,7 @@ en: docker_settings: 'Docker Registry' docker: dockerhub: 'Use Docker Hub' + organization: 'Organization' auth: 'Authentication Required' host: 'Registry Host' port: 'Registry Port' From 7ba045c74dd12a136ed266e62b4dbc9bb8b6caf0 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 11 Oct 2017 13:29:35 +0200 Subject: [PATCH 090/181] fixed error in deploy script --- chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 11859787c2..0684a54f25 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -75,7 +75,7 @@ set +a # ------------------------------------------------------------------------------ # If containers are running, stop them -if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml]; then +if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0"]; then docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f From e6707be9c6ad573b524572e239ef2236f6333d7b Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 11 Oct 2017 13:32:53 +0200 Subject: [PATCH 091/181] remove ssl certificate generation from deploy script --- .../files/default/user-data/deploy-oscmserver | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 0684a54f25..913addd08a 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -136,28 +136,14 @@ if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ]; then fi # Copy certificate files -if [ ${GENERATE_CERTIFICATES} == "false" ] && [ -f ${SSL_CONFIG_PATH}/oscm.crt ] && [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then +if [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key +fi +if [ -f ${SSL_CONFIG_PATH}/oscm.crt ]; then echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt - if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain - fi -else - # Generate self signed certificate if none exist yet - if [ ! -f ${DOCKER_PATH}/config/privkey/oscm-core/oscm.key ] && [ ! -f ${DOCKER_PATH}/config/privkey/oscm-core/oscm.crt ]; then - mkdir ${SSL_CONFIG_PATH}/generate - openssl rand -base64 48 > ${SSL_CONFIG_PATH}/generate/passphrase.txt - openssl genrsa -aes128 -passout file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key 2048 - openssl req -new -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -key ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.csr -subj "/CN=${HOST_FQDN}" - cp ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.key.pass - openssl rsa -in ${SSL_CONFIG_PATH}/generate/oscm.key.pass -passin file:${SSL_CONFIG_PATH}/generate/passphrase.txt -out ${SSL_CONFIG_PATH}/generate/oscm.key - openssl x509 -req -days 3650 -in ${SSL_CONFIG_PATH}/generate/oscm.csr -signkey ${SSL_CONFIG_PATH}/generate/oscm.key -out ${SSL_CONFIG_PATH}/generate/oscm.crt - rm -f ${SSL_CONFIG_PATH}/generate/passphrase.txt ${SSL_CONFIG_PATH}/generate/oscm.key.pass ${SSL_CONFIG_PATH}/generate/oscm.csr - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/generate/oscm.key - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/generate/oscm.crt - rm -f ${SSL_CONFIG_PATH}/generate/oscm.key ${SSL_CONFIG_PATH}/generate/oscm.crt - rmdir ${SSL_CONFIG_PATH}/generate - fi +fi +if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain fi # If http proxy is enabled, create Docker config file From 70848895229d4f60417e4a0d8108636d6ad66a40 Mon Sep 17 00:00:00 2001 From: stavreva Date: Wed, 11 Oct 2017 13:34:18 +0200 Subject: [PATCH 092/181] correct parameter name --- chef/data_bags/crowbar/template-oscm.schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-oscm.schema index a2e8234275..087895cd47 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-oscm.schema @@ -65,7 +65,7 @@ "stack_name": { "type": "str", "required": true } } }, - "floating-network" : { "type": "str", "required": true } + "floating_network" : { "type": "str", "required": true } } }, "mail" : { From 9574c63134989612e8b04258fed4463eec028013 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 12 Oct 2017 08:50:49 +0200 Subject: [PATCH 093/181] Change path for deployment script --- chef/cookbooks/oscm/recipes/server.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index 592647aa52..dc699b2f99 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -28,7 +28,7 @@ oscm_keypair_publickey = node[:oscm][:openstack][:keypair][:publickey] oscm_keypair_publickeyfile = "/etc/oscm/install/openstack_keypair_public.pem" oscm_install_path = "/etc/oscm/install" -oscm_config_path = "/etc/oscm/config" +oscm_path = "/etc/oscm" oscm_volumestack_name = node[:oscm][:openstack][:volume_stack][:stack_name] oscm_instancestack_name = node[:oscm][:openstack][:instance_stack][:stack_name] oscm_data_volume_size = node[:oscm][:openstack][:volume_stack][:data_volume_size] @@ -352,26 +352,26 @@ args = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i #{oscm_keypair_crowbar_sshkey}" ip_appserver = node[:oscm][:openstack][:instance_stack][:ip_appserver] Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_config_path}/docker-compose'" + command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_path}/docker-compose'" command_out = shell_out(command) - command = "scp #{args} #{oscm_install_path}/user-data/oscm-config #{ip_appserver}:#{oscm_config_path}" + command = "scp #{args} #{oscm_install_path}/user-data/oscm-config #{ip_appserver}:#{oscm_path}/config" command_out = shell_out(command) - command = "scp #{args} #{oscm_install_path}/docker-compose-*.yml #{ip_appserver}:#{oscm_config_path}/docker-compose" + command = "scp #{args} #{oscm_install_path}/docker-compose-*.yml #{ip_appserver}:#{oscm_path}/docker-compose" command_out = shell_out(command) if node[:oscm][:api][:protocol] == "https" - command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_config_path}/ssl'" + command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_path}/ssl'" command_out = shell_out(command) - command = "scp #{args} #{oscm_ssl_certfile} #{ip_appserver}:#{oscm_config_path}/ssl/oscm.crt" + command = "scp #{args} #{oscm_ssl_certfile} #{ip_appserver}:#{oscm_path}/ssl/oscm.crt" command_out = shell_out(command) - command = "scp #{args} #{oscm_ssl_keyfile} #{ip_appserver}:#{oscm_config_path}/ssl/oscm.key" + command = "scp #{args} #{oscm_ssl_keyfile} #{ip_appserver}:#{oscm_path}/ssl/oscm.key" command_out = shell_out(command) - command = "scp #{args} #{oscm_ssl_cacerts} #{ip_appserver}:#{oscm_config_path}/ssl/oscm.chain" + command = "scp #{args} #{oscm_ssl_cacerts} #{ip_appserver}:#{oscm_path}/ssl/oscm.chain" end - command = "scp #{args} #{oscm_install_path}/user-data/deploy-oscmserver #{ip_appserver}:#{oscm_config_path}" + command = "scp #{args} #{oscm_install_path}/user-data/deploy-oscmserver #{ip_appserver}:#{oscm_path}/config" command_out = shell_out(command) - command = "ssh #{args} #{ip_appserver} 'chmod 755 #{oscm_config_path}/deploy-oscmserver'" + command = "ssh #{args} #{ip_appserver} 'chmod 755 #{oscm_path}/config/deploy-oscmserver'" command_out = shell_out(command) - command = "ssh #{args} #{ip_appserver} '#{oscm_config_path}/deploy-oscmserver'" + command = "ssh #{args} #{ip_appserver} '#{oscm_path}/config/deploy-oscmserver'" command_out = shell_out(command) end action :create From b998a552099b3c12579a8bf82c1c93e6d2300b0c Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 12 Oct 2017 09:31:05 +0200 Subject: [PATCH 094/181] Correct container name --- .../oscm/templates/default/docker-compose-initdb.yml.erb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb index d80f2112f6..eb5bc5babd 100644 --- a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb +++ b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb @@ -14,7 +14,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-core env_file: var.env @@ -29,7 +29,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-jms env_file: var.env @@ -44,7 +44,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-app env_file: var.env @@ -59,7 +59,7 @@ services: <% if @docker["dockerhub"] == true %> image: <%= @docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> <% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> <% end %> container_name: oscm-initdb-controller-openstack env_file: var.env From 313a67183f05b906a0d7702e82f3f2225b4e4758 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 12 Oct 2017 09:45:33 +0200 Subject: [PATCH 095/181] Add var.env --- chef/cookbooks/oscm/recipes/server.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index dc699b2f99..afcfa4ba8b 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -358,6 +358,8 @@ command_out = shell_out(command) command = "scp #{args} #{oscm_install_path}/docker-compose-*.yml #{ip_appserver}:#{oscm_path}/docker-compose" command_out = shell_out(command) + command = "scp #{args} #{oscm_install_path}/var.env #{ip_appserver}:#{oscm_path}/docker-compose" + command_out = shell_out(command) if node[:oscm][:api][:protocol] == "https" command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_path}/ssl'" command_out = shell_out(command) From d3743620048e38426926abd58ad844bf3d1df7aa Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 12 Oct 2017 09:56:57 +0200 Subject: [PATCH 096/181] add copying of var.env to deploy script --- chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 913addd08a..cd9c6c6b0f 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -134,6 +134,9 @@ fi if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ]; then cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ${DOCKER_PATH}/docker-compose-oscm.yml fi +if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then + cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env +fi # Copy certificate files if [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then From 86876bde47d8e56b69023d01a526a8af34b43ef8 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 12 Oct 2017 12:37:09 +0200 Subject: [PATCH 097/181] remove debug output from deploy script --- chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver | 1 - 1 file changed, 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index cd9c6c6b0f..88b3d6c1c3 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -169,7 +169,6 @@ EOF systemctl restart docker # Otherwise remove Docker config file with proxy config else - echo NOPE if [ -f /etc/systemd/system/docker.service.d/http-proxy.conf ]; then rm -f /etc/systemd/system/docker.service.d/http-proxy.conf systemctl daemon-reload From 3de883d19ac7dceb8b16d25e2809849244b29447 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 12 Oct 2017 13:11:46 +0200 Subject: [PATCH 098/181] fix typo in deploy script --- chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index 88b3d6c1c3..f9754d4706 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -76,7 +76,7 @@ set +a # If containers are running, stop them if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then - if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0"]; then + if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0" ]; then docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f fi From 96f5746865739e59eaafa4e70275aee0a6ad7469 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 12 Oct 2017 13:13:51 +0200 Subject: [PATCH 099/181] Set temp postgres pwd --- chef/cookbooks/oscm/recipes/server.rb | 2 +- chef/cookbooks/oscm/templates/default/var.env.erb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index afcfa4ba8b..b4b2a19187 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -373,7 +373,7 @@ command_out = shell_out(command) command = "ssh #{args} #{ip_appserver} 'chmod 755 #{oscm_path}/config/deploy-oscmserver'" command_out = shell_out(command) - command = "ssh #{args} #{ip_appserver} '#{oscm_path}/config/deploy-oscmserver'" + command = "ssh #{args} #{ip_appserver} '#{oscm_path}/config/deploy-oscmserver &'" command_out = shell_out(command) end action :create diff --git a/chef/cookbooks/oscm/templates/default/var.env.erb b/chef/cookbooks/oscm/templates/default/var.env.erb index 41e5519a69..af2165b7d8 100644 --- a/chef/cookbooks/oscm/templates/default/var.env.erb +++ b/chef/cookbooks/oscm/templates/default/var.env.erb @@ -14,7 +14,8 @@ DB_PORT_APP=5432 DB_PWD_CORE=<%= @instance["db_core_password"] %> DB_PWD_APP=<%= @instance["db_app_password"] %> -DB_SUPERPWD=<%= @instance["db_password"] %> +#DB_SUPERPWD=<%= @instance["db_password"] %> +DB_SUPERPWD=postgres KEY_SECRET=<%= @instance["key_secret"] %> APP_ADMIN_MAIL_ADDRESS=<%= @mail["from"] %> From 73f9ba140c653bb7467805b933fe7641e1e290a6 Mon Sep 17 00:00:00 2001 From: stavreva Date: Thu, 12 Oct 2017 13:51:20 +0200 Subject: [PATCH 100/181] Remove temp pwd --- chef/cookbooks/oscm/templates/default/var.env.erb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chef/cookbooks/oscm/templates/default/var.env.erb b/chef/cookbooks/oscm/templates/default/var.env.erb index af2165b7d8..41e5519a69 100644 --- a/chef/cookbooks/oscm/templates/default/var.env.erb +++ b/chef/cookbooks/oscm/templates/default/var.env.erb @@ -14,8 +14,7 @@ DB_PORT_APP=5432 DB_PWD_CORE=<%= @instance["db_core_password"] %> DB_PWD_APP=<%= @instance["db_app_password"] %> -#DB_SUPERPWD=<%= @instance["db_password"] %> -DB_SUPERPWD=postgres +DB_SUPERPWD=<%= @instance["db_password"] %> KEY_SECRET=<%= @instance["key_secret"] %> APP_ADMIN_MAIL_ADDRESS=<%= @mail["from"] %> From 9be6da4b1a8ce854ef112f4ba90ab0827869d7ca Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 12 Oct 2017 15:39:42 +0200 Subject: [PATCH 101/181] change ports of oscm CORE to standard --- .../oscm/templates/default/docker-compose-oscm.yml.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb index 7e7b880a14..4834234720 100644 --- a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb +++ b/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb @@ -37,8 +37,8 @@ services: - /docker/config/oscm-core/privkey:/privkey - /docker/config/certs:/certs ports: - - 10080:8080 - - 10081:8081 + - 8080:8080 + - 8081:8081 oscm-app: <% if @docker["dockerhub"] == true %> From 8a5386a4e240a0fc9c6fcccef6140c1219acd235 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 12 Oct 2017 17:21:01 +0200 Subject: [PATCH 102/181] add handling of cinder volumes to deploy script --- .../files/default/user-data/deploy-oscmserver | 83 +++++++------------ 1 file changed, 30 insertions(+), 53 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index f9754d4706..c79e360754 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -21,59 +21,6 @@ source ${CONFIG_PATH}/oscm-config # Disable automatic exporting of variables set +a - -# TODO: add certificate to a local docker volume folder. -# echo $SSL_CONFIG_PATH/somekey.pem > /var/web_cert - -# WAIT_CURL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" -# eval $WAIT_CURL - -# For some reason the volume id is shortened to 27 characters when the device -# link in /dev/disk/by-id/ is created. Account for this by truncating the -# device name accordingly. - -# volume_dev_db=$(printf '%.43s\n' $DB_VOLUME_DEV) -# volume_dev_app=$(printf '%.43s\n' $APP_VOLUME_DEV) - -# Only format volume if it hasn't already been formatted. - -# if ! file -Ls $volume_dev_db | grep -q "filesystem data" ; then -# mkfs.ext4 "$volume_dev_db" -# fi -# -# if ! file -Ls $volume_dev_app | grep -q "filesystem data" ; then -# mkfs.ext4 "$volume_dev_app" -# fi -# -# mkdir -p /db -# if ! grep -qs '/db' /proc/mounts; then -# mount -t ext4 $volume_dev_db /db -# fi -# -# mkdir -p /oscm -# if ! grep -qs '/oscm' /proc/mounts; then -# mount -t ext4 $volume_dev_app /oscm -# fi - - -# DB=/db/database -# APP_PATH=/oscm/oscm-app/ -# OSCM_PATH=/oscm/oscm-core/ -# CLIENT_CRT_PATH=/oscm/client-crt/ -# -# DB_VOLUME=oscm-db-volume -# OSCM_VOLUME=oscm-core-volume -# APP_VOLUME=oscm-app-volume -# -# mkdir -p $DB -# mkdir -p $APP_PATH -# mkdir -p $OSCM_PATH -# -# OSCM_DIR=/opt/oscm -# mkdir -p $OSCM_DIR - -# ------------------------------------------------------------------------------ - # If containers are running, stop them if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0" ]; then @@ -82,6 +29,36 @@ if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then fi fi +# The Cinder volume ids are truncated to 27 characters when the device +# link in /dev/disk/by-id/ is created. Account for this by truncating the +# device name accordingly. +volume_dev_logs=$(printf '%.43s\n' ${LOGS_VOLUME_DEV}) +volume_dev_data=$(printf '%.43s\n' ${DATA_VOLUME_DEV}) + +# Create a file system on the Cinder volumes +if ! file -Ls ${volume_dev_logs} | grep -q "filesystem data" ; then + mkfs.ext4 "${volume_dev_logs}" +fi + +if ! file -Ls ${volume_dev_data} | grep -q "filesystem data" ; then + mkfs.ext4 "${volume_dev_data}" +fi + +# Mount the Cinder volumes if they are not mounted yet +if ! grep -qs "${DOCKER_PATH}/logs" /proc/mounts; then + if [ ! -d ${DOCKER_PATH}/logs ]; then + mkdir -p ${DOCKER_PATH}/logs + fi + mount -t ext4 ${volume_dev_logs} ${DOCKER_PATH}/logs +fi + +if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then + if [ ! -d ${DOCKER_PATH}/data ]; then + mkdir -p ${DOCKER_PATH}/data + fi + mount -t ext4 ${volume_dev_data} ${DOCKER_PATH}/data +fi + # Create Docker directories if they do not exist yet for docker_directory in \ ${DOCKER_PATH}/data/oscm-db/data \ From 40373335913d173a2e69acf71a82f486ce4e736b Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 13 Oct 2017 10:30:53 +0200 Subject: [PATCH 103/181] Barclamp should not wait for deployment script. --- chef/cookbooks/oscm/recipes/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb index b4b2a19187..dbf1703f5a 100644 --- a/chef/cookbooks/oscm/recipes/server.rb +++ b/chef/cookbooks/oscm/recipes/server.rb @@ -373,7 +373,7 @@ command_out = shell_out(command) command = "ssh #{args} #{ip_appserver} 'chmod 755 #{oscm_path}/config/deploy-oscmserver'" command_out = shell_out(command) - command = "ssh #{args} #{ip_appserver} '#{oscm_path}/config/deploy-oscmserver &'" + command = "ssh #{args} #{ip_appserver} '#{oscm_path}/config/deploy-oscmserver' &" command_out = shell_out(command) end action :create From df6bbdb0ad0c6861a0b4cce9167cb1b68dd20a72 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 13 Oct 2017 11:12:34 +0200 Subject: [PATCH 104/181] move cinder volume preparation to heat-config and add fstab --- .../files/default/user-data/deploy-oscmserver | 31 ----------- .../oscm/files/default/user-data/heat-config | 53 +++++++++++++++---- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver index c79e360754..fec8f08e01 100644 --- a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver +++ b/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver @@ -16,7 +16,6 @@ DOCKER_PATH=/docker # Enable automatic exporting of variables set -a # Read configuration files -source ${CONFIG_PATH}/heat-config source ${CONFIG_PATH}/oscm-config # Disable automatic exporting of variables set +a @@ -29,36 +28,6 @@ if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then fi fi -# The Cinder volume ids are truncated to 27 characters when the device -# link in /dev/disk/by-id/ is created. Account for this by truncating the -# device name accordingly. -volume_dev_logs=$(printf '%.43s\n' ${LOGS_VOLUME_DEV}) -volume_dev_data=$(printf '%.43s\n' ${DATA_VOLUME_DEV}) - -# Create a file system on the Cinder volumes -if ! file -Ls ${volume_dev_logs} | grep -q "filesystem data" ; then - mkfs.ext4 "${volume_dev_logs}" -fi - -if ! file -Ls ${volume_dev_data} | grep -q "filesystem data" ; then - mkfs.ext4 "${volume_dev_data}" -fi - -# Mount the Cinder volumes if they are not mounted yet -if ! grep -qs "${DOCKER_PATH}/logs" /proc/mounts; then - if [ ! -d ${DOCKER_PATH}/logs ]; then - mkdir -p ${DOCKER_PATH}/logs - fi - mount -t ext4 ${volume_dev_logs} ${DOCKER_PATH}/logs -fi - -if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then - if [ ! -d ${DOCKER_PATH}/data ]; then - mkdir -p ${DOCKER_PATH}/data - fi - mount -t ext4 ${volume_dev_data} ${DOCKER_PATH}/data -fi - # Create Docker directories if they do not exist yet for docker_directory in \ ${DOCKER_PATH}/data/oscm-db/data \ diff --git a/chef/cookbooks/oscm/files/default/user-data/heat-config b/chef/cookbooks/oscm/files/default/user-data/heat-config index 95500af31c..c849d9e7bd 100644 --- a/chef/cookbooks/oscm/files/default/user-data/heat-config +++ b/chef/cookbooks/oscm/files/default/user-data/heat-config @@ -1,14 +1,47 @@ #!/bin/sh -# Writes the configuration parameters passed in by Heat/cloud-init to /etc/heatconfig. -mkdir -p /etc/oscm/config -touch /etc/oscm/config/heat-config -chmod 400 /etc/oscm/config/heat-config - -cat > /etc/oscm/config/heat-config <> /root/.ssh/authorized_keys \ No newline at end of file +echo $CERT_STR >> /root/.ssh/authorized_keys + +# Set up Cinder storage +# The Cinder volume ids are truncated to 27 characters when the device +# link in /dev/disk/by-id/ is created. Account for this by truncating the +# device name accordingly. +volume_dev_logs=$(printf '%.43s\n' ${LOGS_VOLUME_DEV}) +volume_dev_data=$(printf '%.43s\n' ${DATA_VOLUME_DEV}) + +# Create a file system on the Cinder volumes +if ! file -Ls ${volume_dev_logs} | grep -q "filesystem data" ; then + mkfs.ext4 "${volume_dev_logs}" +fi + +if ! file -Ls ${volume_dev_data} | grep -q "filesystem data" ; then + mkfs.ext4 "${volume_dev_data}" +fi + +# Create fstab entries +if ! grep -qs "${volume_dev_logs}" /etc/fstab; then + echo "${volume_dev_logs} ${DOCKER_PATH}/logs ext4 defaults 1 1" >> /etc/fstab +fi + +if ! grep -qs "${volume_dev_data}" /etc/fstab; then + echo "${volume_dev_data} ${DOCKER_PATH}/logs ext4 defaults 1 1" >> /etc/fstab +fi + +# Mount the Cinder volumes if they are not mounted yet +if ! grep -qs "${DOCKER_PATH}/logs" /proc/mounts; then + if [ ! -d ${DOCKER_PATH}/logs ]; then + mkdir -p ${DOCKER_PATH}/logs + fi + mount ${volume_dev_logs} +fi + +if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then + if [ ! -d ${DOCKER_PATH}/data ]; then + mkdir -p ${DOCKER_PATH}/data + fi + mount ${volume_dev_data} +fi From b0b6a072061eb50724cd5d75663a79cfebf5c365 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 13 Oct 2017 11:26:51 +0200 Subject: [PATCH 105/181] Add env file --- .../oscm/templates/default/docker-compose-initdb.yml.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb index eb5bc5babd..e172da8088 100644 --- a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb +++ b/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb @@ -7,6 +7,7 @@ services: image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% end %> container_name: oscm-db + env_file: var.env volumes: - /docker/data/oscm-db/data:/var/lib/postgresql/data From 3dcece045a85e1937f1b12dad2e0ccdedc059b54 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 13 Oct 2017 11:55:55 +0200 Subject: [PATCH 106/181] Rename oscm to escm --- bin/{crowbar_oscm => crowbar_escm} | 2 +- chef/cookbooks/escm/README.md | 1 + .../{oscm => escm}/attributes/default.rb | 10 +- .../files/default/application.yaml | 40 +- .../files/default/user-data/deploy-oscmserver | 0 .../files/default/user-data/heat-config | 0 .../{oscm => escm}/files/default/volumes.yaml | 10 +- chef/cookbooks/{oscm => escm}/metadata.rb | 4 +- .../escm/recipes/role_oscm_server.rb | 3 + chef/cookbooks/escm/recipes/server.rb | 381 ++++++++++++++++++ .../default/docker-compose-initdb.yml.erb | 0 .../default/docker-compose-oscm.yml.erb | 0 .../templates/default/oscm.conf.erb | 1 - .../templates/default/var.env.erb | 0 chef/cookbooks/oscm/README.md | 1 - .../oscm/recipes/role_oscm_server.rb | 3 - chef/cookbooks/oscm/recipes/server.rb | 381 ------------------ ...{template-oscm.json => template-escm.json} | 38 +- ...plate-oscm.schema => template-escm.schema} | 4 +- chef/roles/escm-server.rb | 5 + chef/roles/oscm-server.rb | 5 - ...{oscm_controller.rb => escm_controller.rb} | 6 +- .../{oscm_helper.rb => escm_helper.rb} | 4 +- .../{oscm_service.rb => escm_service.rb} | 14 +- .../{oscm => escm}/_edit_attributes.html.haml | 6 +- .../config/locales/{oscm => escm}/en.yml | 6 +- oscm.yml => escm.yml | 6 +- 27 files changed, 465 insertions(+), 466 deletions(-) rename bin/{crowbar_oscm => crowbar_escm} (97%) create mode 100644 chef/cookbooks/escm/README.md rename chef/cookbooks/{oscm => escm}/attributes/default.rb (68%) rename chef/cookbooks/{oscm => escm}/files/default/application.yaml (88%) rename chef/cookbooks/{oscm => escm}/files/default/user-data/deploy-oscmserver (100%) rename chef/cookbooks/{oscm => escm}/files/default/user-data/heat-config (100%) rename chef/cookbooks/{oscm => escm}/files/default/volumes.yaml (75%) rename chef/cookbooks/{oscm => escm}/metadata.rb (79%) create mode 100644 chef/cookbooks/escm/recipes/role_oscm_server.rb create mode 100644 chef/cookbooks/escm/recipes/server.rb rename chef/cookbooks/{oscm => escm}/templates/default/docker-compose-initdb.yml.erb (100%) rename chef/cookbooks/{oscm => escm}/templates/default/docker-compose-oscm.yml.erb (100%) rename chef/cookbooks/{oscm => escm}/templates/default/oscm.conf.erb (95%) rename chef/cookbooks/{oscm => escm}/templates/default/var.env.erb (100%) delete mode 100644 chef/cookbooks/oscm/README.md delete mode 100644 chef/cookbooks/oscm/recipes/role_oscm_server.rb delete mode 100644 chef/cookbooks/oscm/recipes/server.rb rename chef/data_bags/crowbar/{template-oscm.json => template-escm.json} (73%) rename chef/data_bags/crowbar/{template-oscm.schema => template-escm.schema} (99%) create mode 100644 chef/roles/escm-server.rb delete mode 100644 chef/roles/oscm-server.rb rename crowbar_framework/app/controllers/{oscm_controller.rb => escm_controller.rb} (84%) rename crowbar_framework/app/helpers/barclamp/{oscm_helper.rb => escm_helper.rb} (92%) rename crowbar_framework/app/models/{oscm_service.rb => escm_service.rb} (86%) rename crowbar_framework/app/views/barclamp/{oscm => escm}/_edit_attributes.html.haml (93%) rename crowbar_framework/config/locales/{oscm => escm}/en.yml (96%) rename oscm.yml => escm.yml (89%) diff --git a/bin/crowbar_oscm b/bin/crowbar_escm similarity index 97% rename from bin/crowbar_oscm rename to bin/crowbar_escm index 83f0b39dd9..ac0fb596f0 100644 --- a/bin/crowbar_oscm +++ b/bin/crowbar_escm @@ -16,7 +16,7 @@ # require File.join(File.expand_path(File.dirname(__FILE__)), "barclamp_lib") -@barclamp = "oscm" +@barclamp = "escm" @timeout = 3600 main diff --git a/chef/cookbooks/escm/README.md b/chef/cookbooks/escm/README.md new file mode 100644 index 0000000000..32c486aeee --- /dev/null +++ b/chef/cookbooks/escm/README.md @@ -0,0 +1 @@ +Chef Cookbook to install and configure ESCM \ No newline at end of file diff --git a/chef/cookbooks/oscm/attributes/default.rb b/chef/cookbooks/escm/attributes/default.rb similarity index 68% rename from chef/cookbooks/oscm/attributes/default.rb rename to chef/cookbooks/escm/attributes/default.rb index b7dc13629c..67a24cd687 100644 --- a/chef/cookbooks/oscm/attributes/default.rb +++ b/chef/cookbooks/escm/attributes/default.rb @@ -14,8 +14,8 @@ # limitations under the License. # -default[:oscm][:ssl][:certfile] = "/etc/oscm/ssl/certs/signing_cert.pem" -default[:oscm][:ssl][:keyfile] = "/etc/oscm/ssl/private/signing_key.pem" -default[:oscm][:ssl][:generate_certs] = false -default[:oscm][:ssl][:insecure] = false -default[:oscm][:ssl][:ca_certs] = "/etc/oscm/ssl/certs/ca.pem" +default[:escm][:ssl][:certfile] = "/etc/escm/ssl/certs/signing_cert.pem" +default[:escm][:ssl][:keyfile] = "/etc/escm/ssl/private/signing_key.pem" +default[:escm][:ssl][:generate_certs] = false +default[:escm][:ssl][:insecure] = false +default[:escm][:ssl][:ca_certs] = "/etc/escm/ssl/certs/ca.pem" diff --git a/chef/cookbooks/oscm/files/default/application.yaml b/chef/cookbooks/escm/files/default/application.yaml similarity index 88% rename from chef/cookbooks/oscm/files/default/application.yaml rename to chef/cookbooks/escm/files/default/application.yaml index 78a5e5e898..782c6188ef 100644 --- a/chef/cookbooks/oscm/files/default/application.yaml +++ b/chef/cookbooks/escm/files/default/application.yaml @@ -1,17 +1,17 @@ -# heat stack-create --poll -f application.yaml -P logs_volume_id=$logs_volume_id -P data_volume_id=$data_volume_id -P key_name=default -P flavor=d1.tiny oscm +# heat stack-create --poll -f application.yaml -P logs_volume_id=$logs_volume_id -P data_volume_id=$data_volume_id -P key_name=default -P flavor=d1.tiny escm heat_template_version: 2015-10-15 description: | # This is how you deploy the whole thing: # 1) Instantiate volumes.yaml as follows - heat stack-create --poll -f volumes.yaml oscm + heat stack-create --poll -f volumes.yaml escm - # 2) Retrieve outputs from oscm: - logs_volume_id=$(heat output-show oscm logs_volume_id | sed 's/"//g') - data_volume_id=$(heat output-show oscm data_volume_id | sed 's/"//g') + # 2) Retrieve outputs from escm: + logs_volume_id=$(heat output-show escm logs_volume_id | sed 's/"//g') + data_volume_id=$(heat output-show escm data_volume_id | sed 's/"//g') # 3) Create application stack: - heat stack-create --poll -f application.yaml -P logs_volume_id=$logs_volume_id -P data_volume_id=$data_volume_id -P key_name= oscm + heat stack-create --poll -f application.yaml -P logs_volume_id=$logs_volume_id -P data_volume_id=$data_volume_id -P key_name= escm @@ -23,21 +23,21 @@ parameters: image: type: string default: sles12-sp1 - description: Glance image to use for servers + description: Glance image to use for server flavor: type: string - default: oscm.medium - description: Nova flavor to use for servers + default: escm.medium + description: Nova flavor to use for server key_name: type: string - default: oscm + default: escm description: Keypair name logs_volume_id: type: string - description: The Cinder volume to attach to the application server + description: The Cinder volume for logs data_volume_id: type: string - description: The Cinder volume to attach to the database server + description: The Cinder volume for data registry_port: default: "" type: string @@ -66,18 +66,18 @@ resources: ### Network infrastructure ### - oscm_network: + escm_network: type: OS::Neutron::Net properties: - name: oscm + name: escm - oscm_subnet: + escm_subnet: type: OS::Neutron::Subnet properties: cidr: 10.0.0.1/24 - name: oscm + name: escm network: - get_resource: oscm_network + get_resource: escm_network router: type: OS::Neutron::Router @@ -91,13 +91,13 @@ resources: type: OS::Neutron::RouterInterface properties: router: { get_resource: router } - subnet: { get_resource: oscm_subnet } + subnet: { get_resource: escm_subnet } allow_inbound: type: OS::Neutron::SecurityGroup properties: description: "Allow inbound SSH and HTTP traffic" - name: oscm + name: escm rules: - direction: ingress remote_ip_prefix: 0.0.0.0/0 @@ -189,7 +189,7 @@ resources: type: OS::Neutron::Port properties: network: - get_resource: oscm_network + get_resource: escm_network security_groups: - get_resource: allow_inbound diff --git a/chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/escm/files/default/user-data/deploy-oscmserver similarity index 100% rename from chef/cookbooks/oscm/files/default/user-data/deploy-oscmserver rename to chef/cookbooks/escm/files/default/user-data/deploy-oscmserver diff --git a/chef/cookbooks/oscm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config similarity index 100% rename from chef/cookbooks/oscm/files/default/user-data/heat-config rename to chef/cookbooks/escm/files/default/user-data/heat-config diff --git a/chef/cookbooks/oscm/files/default/volumes.yaml b/chef/cookbooks/escm/files/default/volumes.yaml similarity index 75% rename from chef/cookbooks/oscm/files/default/volumes.yaml rename to chef/cookbooks/escm/files/default/volumes.yaml index 4c788a040c..a8cb50fc95 100644 --- a/chef/cookbooks/oscm/files/default/volumes.yaml +++ b/chef/cookbooks/escm/files/default/volumes.yaml @@ -1,4 +1,4 @@ -# Example invocation: heat stack-create --poll -f volumes.yaml oscm +# Example invocation: heat stack-create --poll -f volumes.yaml escm heat_template_version: 2015-10-15 @@ -36,12 +36,12 @@ resources: size: { get_param: data_size } outputs: - # Retrieval after stack creation (presuming stack is named `oscm`): - # heat output-show oscm logs_volume_id + # Retrieval after stack creation (presuming stack is named `escm`): + # heat output-show escm logs_volume_id logs_volume_id: value: { get_resource: logs_volume } - # Retrieval after stack creation (presuming stack is named `oscm`): - # heat output-show oscm data_volume_id + # Retrieval after stack creation (presuming stack is named `escm`): + # heat output-show escm data_volume_id data_volume_id: value: { get_resource: data_volume } diff --git a/chef/cookbooks/oscm/metadata.rb b/chef/cookbooks/escm/metadata.rb similarity index 79% rename from chef/cookbooks/oscm/metadata.rb rename to chef/cookbooks/escm/metadata.rb index fdb8f4ffda..612eb99b33 100644 --- a/chef/cookbooks/oscm/metadata.rb +++ b/chef/cookbooks/escm/metadata.rb @@ -1,8 +1,8 @@ -name "oscm" +name "escm" maintainer "EST" maintainer_email "arkadiusz.kowalczyk@ts.fujistu.com" license "Apache 2.0" -description "Installs/Configures OSCM" +description "Installs/Configures ESCM" long_description IO.read(File.join(File.dirname(__FILE__), "README.md")) version "0.1" diff --git a/chef/cookbooks/escm/recipes/role_oscm_server.rb b/chef/cookbooks/escm/recipes/role_oscm_server.rb new file mode 100644 index 0000000000..f334e59351 --- /dev/null +++ b/chef/cookbooks/escm/recipes/role_oscm_server.rb @@ -0,0 +1,3 @@ +if CrowbarRoleRecipe.node_state_valid_for_role?(node, "escm", "escm-server") + include_recipe "#{@cookbook_name}::server" +end diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb new file mode 100644 index 0000000000..728faf1639 --- /dev/null +++ b/chef/cookbooks/escm/recipes/server.rb @@ -0,0 +1,381 @@ +# +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +escm_project = node[:escm][:keystone][:project] +escm_user = node[:escm][:keystone][:user] +escm_password = node[:escm][:keystone][:password] +escm_ssl_certfile = node[:escm][:ssl][:certfile] +escm_ssl_keyfile = node[:escm][:ssl][:keyfile] +escm_ssl_cacerts = node[:escm][:ssl][:ca_certs] +escm_flavor_name = node[:escm][:openstack][:flavor][:name] +escm_flavor_ram = node[:escm][:openstack][:flavor][:ram] +escm_flavor_vcpus = node[:escm][:openstack][:flavor][:vcpus] +escm_flavor_disk = node[:escm][:openstack][:flavor][:disk] +escm_keypair_name = node[:escm][:openstack][:keypair][:name] +escm_keypair_publickey = node[:escm][:openstack][:keypair][:publickey] +escm_keypair_publickeyfile = "/etc/escm/install/openstack_keypair_public.pem" +escm_install_path = "/etc/escm/install" +escm_path = "/etc/escm" +escm_volumestack_name = node[:escm][:openstack][:volume_stack][:stack_name] +escm_instancestack_name = node[:escm][:openstack][:instance_stack][:stack_name] +escm_data_volume_size = node[:escm][:openstack][:volume_stack][:data_volume_size] +escm_logs_volume_size = node[:escm][:openstack][:volume_stack][:logs_volume_size] +escm_image = node[:escm][:openstack][:image] +escm_floating_network = node[:escm][:openstack][:floating_network] +escm_keypair_crowbar_sshkey = "/etc/escm/install/escm_ssh.key" +escm_group = "root" + +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) + +env = "OS_USERNAME='#{escm_user}' " +env << "OS_PASSWORD='#{escm_password}' " +env << "OS_PROJECT_NAME='#{escm_project}' " +env << "OS_AUTH_URL='#{keystone_settings["internal_auth_url"]}' " +env << "OS_INTERFACE=internal " +env << "OS_IDENTITY_API_VERSION='#{keystone_settings["api_version"]}' " +env << "OS_USER_DOMAIN_NAME='Default' " +env << "OS_PROJECT_DOMAIN_NAME='Default'" + +openstack_cmd = "#{env} openstack" + +openstack_args_keystone = keystone_settings["insecure"] ? "--insecure" : "" + +nova_config = Barclamp::Config.load("openstack", "nova", node[:escm][:nova_instance]) +nova_insecure = CrowbarOpenStackHelper.insecure(nova_config) +openstack_args_nova = nova_insecure || keystone_settings["insecure"] ? "--insecure" : "" + +heat_config = Barclamp::Config.load("openstack", "heat", node[:escm][:heat_instance]) +heat_insecure = CrowbarOpenStackHelper.insecure(heat_config) +openstack_args_heat = heat_insecure || keystone_settings["insecure"] ? "--insecure" : "" + +ruby_block "check_escm_glance_image" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} image list -c Name -f value | egrep '^#{escm_image}$'" + command_out = shell_out(command) + if command_out.stdout.strip != escm_image + message = "The image with name '#{escm_image}' is not found in glance! Please check your escm proposal attributes or glance image registry." + raise message + end + end + action :create +end + + +register_auth_hash = { + user: keystone_settings["admin_user"], + password: keystone_settings["admin_password"], + tenant: keystone_settings["admin_tenant"] +} + +keystone_register "escm wakeup keystone" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + action :wakeup +end + +keystone_register "escm create project" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + tenant_name escm_project + action :add_tenant +end + +keystone_register "escm register user" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name escm_user + user_password escm_password + tenant_name escm_project + action :add_user +end + +keystone_register "escm give user admin role" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name escm_user + tenant_name escm_project + role_name "admin" + action :add_access +end + +keystone_register "escm give user member role" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name escm_user + tenant_name escm_project + role_name "Member" + action :add_access +end + +keystone_register "escm give user _member_ role" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name escm_user + tenant_name escm_project + role_name "_member_" + action :add_access +end + +execute "create_escm_flavor" do + command "#{openstack_cmd} #{openstack_args_nova} flavor create --ram #{escm_flavor_ram} --disk #{escm_flavor_disk} \ + --vcpus #{escm_flavor_vcpus} --private #{escm_flavor_name}" + not_if "#{openstack_cmd} #{openstack_args_nova} flavor list --all -c Name -f value | egrep -q '^#{escm_flavor_name}$'" +end + +execute "create_escm_flavor_access" do + command "#{openstack_cmd} #{openstack_args_nova} flavor set --project #{escm_project} #{escm_flavor_name}" + ignore_failure true +end + +bash "create_escm_keypair_file" do + code <<-EOH + publickey="#{escm_keypair_publickey}" + mkdir -p "$(dirname "#{escm_keypair_publickeyfile}")" &> /dev/null + echo "${publickey}" > "#{escm_keypair_publickeyfile}" +EOH +end + +execute "create_escm_keypair" do + command "#{openstack_cmd} #{openstack_args_nova} keypair create #{escm_keypair_name} --public-key #{escm_keypair_publickeyfile}" + not_if "#{openstack_cmd} #{openstack_args_nova} keypair list -c Name -f value | egrep -q '^#{escm_keypair_name}$'" +end + +directory "#{escm_install_path}" do + owner escm_group + group escm_group + mode 0640 + recursive true +end + +cookbook_file "#{escm_install_path}/volumes.yaml" do + source "volumes.yaml" + owner escm_group + group escm_group + mode 0640 + action :create +end + +cookbook_file "#{escm_install_path}/application.yaml" do + source "application.yaml" + owner escm_group + group escm_group + mode 0640 + action :create +end + +directory "#{escm_install_path}/user-data" do + owner escm_group + group escm_group + mode 0640 + recursive true +end + +cookbook_file "#{escm_install_path}/user-data/heat-config" do + source "user-data/heat-config" + owner escm_group + group escm_group + mode 0640 + action :create +end + +cookbook_file "#{escm_install_path}/user-data/deploy-escmserver" do + source "user-data/deploy-escmserver" + owner escm_group + group escm_group + mode 0640 + action :create +end + +execute "create_escm_volume_stack" do + command "#{openstack_cmd} #{openstack_args_heat} stack create --parameter data_size=#{escm_data_volume_size} --parameter logs_size=#{escm_logs_volume_size} \ + -t #{escm_install_path}/volumes.yaml --wait #{escm_volumestack_name}" + not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{escm_volumestack_name}$'" +end + +ruby_block "get_escm_volume_ids" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell #{escm_volumestack_name} data_volume_id | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:escm][:openstack][:volume_stack][:data_volume_id] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell #{escm_volumestack_name} logs_volume_id | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:escm][:openstack][:volume_stack][:logs_volume_id] = command_out.stdout.strip + end + action :create +end + +ruby_block "generate_escm_crowbar_ssh_keys" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "mkdir -p '$(dirname #{escm_keypair_crowbar_sshkey})'" + command_out = shell_out(command) + command = "[ ! -f #{escm_keypair_crowbar_sshkey}] && yes y | ssh-keygen -t rsa -f #{escm_keypair_crowbar_sshkey} -N ''" + command_out = shell_out(command) + end + action :create +end + +execute "create_escm_instance_stack" do + command lazy { "#{openstack_cmd} #{openstack_args_heat} stack create --parameter logs_volume_id=#{node[:escm][:openstack][:volume_stack][:logs_volume_id]} \ + --parameter data_volume_id=#{node[:escm][:openstack][:volume_stack][:data_volume_id]} \ + --parameter image=#{escm_image} --parameter flavor=#{escm_flavor_name} \ + --parameter key_name=#{escm_keypair_name} --parameter floating_network=#{escm_floating_network} \ + --parameter mail_port=#{node[:escm][:mail][:port]} --parameter registry_port=#{node[:escm][:docker][:port]} \ + --parameter-file ssh_cert=#{escm_keypair_crowbar_sshkey}.pub \ + -t #{escm_install_path}/application.yaml --wait #{escm_instancestack_name}" } + not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{escm_instancestack_name}$'" +end + +ruby_block "get_escm_floating_ip" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{escm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:escm][:openstack][:instance_stack][:ip_appserver] = command_out.stdout.strip + end + action :create +end + +if node[:escm][:api][:protocol] == "https" + ssl_setup "setting up ssl for escm" do + generate_certs node[:escm][:ssl][:generate_certs] + certfile node[:escm][:ssl][:certfile] + keyfile node[:escm][:ssl][:keyfile] + group escm_group + fqdn node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] + ca_certs node[:escm][:ssl][:ca_certs] + end +end + +ruby_block "get_escm_secrets" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{escm_instancestack_name} db_password | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:escm][:openstack][:instance_stack][:db_password] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{escm_instancestack_name} db_core_password | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:escm][:openstack][:instance_stack][:db_core_password] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{escm_instancestack_name} db_app_password | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:escm][:openstack][:instance_stack][:db_app_password] = command_out.stdout.strip + command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{escm_instancestack_name} key_secret | grep -Po '(?<=^output_value=\")[^\"]*'" + command_out = shell_out(command) + node[:escm][:openstack][:instance_stack][:key_secret] = command_out.stdout.strip + end + action :create +end + +template "#{escm_install_path}/user-data/escm-config" do + source "escm.conf.erb" + owner escm_group + group escm_group + mode 0640 + variables( + mail: node[:escm][:mail], + docker: node[:escm][:docker], + proxy: node[:escm][:proxy], + host_fqdn: node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn], + instance: node[:escm][:openstack][:instance_stack] + ) +end + +template "#{escm_install_path}/docker-compose-initdb.yml" do + source "docker-compose-initdb.yml.erb" + owner escm_group + group escm_group + mode 0640 + variables( + docker: node[:escm][:docker] + ) +end + +template "#{escm_install_path}/docker-compose-escm.yml" do + source "docker-compose-escm.yml.erb" + owner escm_group + group escm_group + mode 0640 + variables( + docker: node[:escm][:docker] + ) +end + +template "#{escm_install_path}/var.env" do + source "var.env.erb" + owner escm_group + group escm_group + mode 0640 + variables( + mail: node[:escm][:mail], + docker: node[:escm][:docker], + proxy: node[:escm][:proxy], + host_fqdn: node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn], + instance: node[:escm][:openstack][:instance_stack] + ) +end + +ruby_block "inject_escm_scripts" do + block do + args = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i #{escm_keypair_crowbar_sshkey}" + ip_appserver = node[:escm][:openstack][:instance_stack][:ip_appserver] + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/docker-compose'" + command_out = shell_out(command) + command = "scp #{args} #{escm_install_path}/user-data/escm-config #{ip_appserver}:#{escm_path}/config" + command_out = shell_out(command) + command = "scp #{args} #{escm_install_path}/docker-compose-*.yml #{ip_appserver}:#{escm_path}/docker-compose" + command_out = shell_out(command) + command = "scp #{args} #{escm_install_path}/var.env #{ip_appserver}:#{escm_path}/docker-compose" + command_out = shell_out(command) + if node[:escm][:api][:protocol] == "https" + command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/ssl'" + command_out = shell_out(command) + command = "scp #{args} #{escm_ssl_certfile} #{ip_appserver}:#{escm_path}/ssl/escm.crt" + command_out = shell_out(command) + command = "scp #{args} #{escm_ssl_keyfile} #{ip_appserver}:#{escm_path}/ssl/escm.key" + command_out = shell_out(command) + command = "scp #{args} #{escm_ssl_cacerts} #{ip_appserver}:#{escm_path}/ssl/escm.chain" + end + command = "scp #{args} #{escm_install_path}/user-data/deploy-escmserver #{ip_appserver}:#{escm_path}/config" + command_out = shell_out(command) + command = "ssh #{args} #{ip_appserver} 'chmod 755 #{escm_path}/config/deploy-escmserver'" + command_out = shell_out(command) + command = "ssh #{args} #{ip_appserver} '#{escm_path}/config/deploy-escmserver' &" + command_out = shell_out(command) + end + action :create +end + diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-initdb.yml.erb similarity index 100% rename from chef/cookbooks/oscm/templates/default/docker-compose-initdb.yml.erb rename to chef/cookbooks/escm/templates/default/docker-compose-initdb.yml.erb diff --git a/chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-oscm.yml.erb similarity index 100% rename from chef/cookbooks/oscm/templates/default/docker-compose-oscm.yml.erb rename to chef/cookbooks/escm/templates/default/docker-compose-oscm.yml.erb diff --git a/chef/cookbooks/oscm/templates/default/oscm.conf.erb b/chef/cookbooks/escm/templates/default/oscm.conf.erb similarity index 95% rename from chef/cookbooks/oscm/templates/default/oscm.conf.erb rename to chef/cookbooks/escm/templates/default/oscm.conf.erb index 1e9a462f78..7cc8f39bed 100644 --- a/chef/cookbooks/oscm/templates/default/oscm.conf.erb +++ b/chef/cookbooks/escm/templates/default/oscm.conf.erb @@ -17,7 +17,6 @@ DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> DOCKER_REGISTRY_USER=<%= @docker["user"] %> DOCKER_REGISTRY_PWD=<%= @docker["password"] %> -DOCKER_OSCM_TAG=<%= @docker["tag"] %> #host HOST_FQDN=<%= @host_fqdn %> diff --git a/chef/cookbooks/oscm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb similarity index 100% rename from chef/cookbooks/oscm/templates/default/var.env.erb rename to chef/cookbooks/escm/templates/default/var.env.erb diff --git a/chef/cookbooks/oscm/README.md b/chef/cookbooks/oscm/README.md deleted file mode 100644 index a413e4b084..0000000000 --- a/chef/cookbooks/oscm/README.md +++ /dev/null @@ -1 +0,0 @@ -Chef Cookbook to install and configure OSCM \ No newline at end of file diff --git a/chef/cookbooks/oscm/recipes/role_oscm_server.rb b/chef/cookbooks/oscm/recipes/role_oscm_server.rb deleted file mode 100644 index dc5978f152..0000000000 --- a/chef/cookbooks/oscm/recipes/role_oscm_server.rb +++ /dev/null @@ -1,3 +0,0 @@ -if CrowbarRoleRecipe.node_state_valid_for_role?(node, "oscm", "oscm-server") - include_recipe "#{@cookbook_name}::server" -end diff --git a/chef/cookbooks/oscm/recipes/server.rb b/chef/cookbooks/oscm/recipes/server.rb deleted file mode 100644 index dbf1703f5a..0000000000 --- a/chef/cookbooks/oscm/recipes/server.rb +++ /dev/null @@ -1,381 +0,0 @@ -# -# Copyright 2017, SUSE LINUX GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -oscm_project = node[:oscm][:keystone][:project] -oscm_user = node[:oscm][:keystone][:user] -oscm_password = node[:oscm][:keystone][:password] -oscm_ssl_certfile = node[:oscm][:ssl][:certfile] -oscm_ssl_keyfile = node[:oscm][:ssl][:keyfile] -oscm_ssl_cacerts = node[:oscm][:ssl][:ca_certs] -oscm_flavor_name = node[:oscm][:openstack][:flavor][:name] -oscm_flavor_ram = node[:oscm][:openstack][:flavor][:ram] -oscm_flavor_vcpus = node[:oscm][:openstack][:flavor][:vcpus] -oscm_flavor_disk = node[:oscm][:openstack][:flavor][:disk] -oscm_keypair_name = node[:oscm][:openstack][:keypair][:name] -oscm_keypair_publickey = node[:oscm][:openstack][:keypair][:publickey] -oscm_keypair_publickeyfile = "/etc/oscm/install/openstack_keypair_public.pem" -oscm_install_path = "/etc/oscm/install" -oscm_path = "/etc/oscm" -oscm_volumestack_name = node[:oscm][:openstack][:volume_stack][:stack_name] -oscm_instancestack_name = node[:oscm][:openstack][:instance_stack][:stack_name] -oscm_data_volume_size = node[:oscm][:openstack][:volume_stack][:data_volume_size] -oscm_logs_volume_size = node[:oscm][:openstack][:volume_stack][:logs_volume_size] -oscm_image = node[:oscm][:openstack][:image] -oscm_floating_network = node[:oscm][:openstack][:floating_network] -oscm_keypair_crowbar_sshkey = "/etc/oscm/install/oscm_ssh.key" -oscm_group = "root" - -keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) - -env = "OS_USERNAME='#{oscm_user}' " -env << "OS_PASSWORD='#{oscm_password}' " -env << "OS_PROJECT_NAME='#{oscm_project}' " -env << "OS_AUTH_URL='#{keystone_settings["internal_auth_url"]}' " -env << "OS_INTERFACE=internal " -env << "OS_IDENTITY_API_VERSION='#{keystone_settings["api_version"]}' " -env << "OS_USER_DOMAIN_NAME='Default' " -env << "OS_PROJECT_DOMAIN_NAME='Default'" - -openstack_cmd = "#{env} openstack" - -openstack_args_keystone = keystone_settings["insecure"] ? "--insecure" : "" - -nova_config = Barclamp::Config.load("openstack", "nova", node[:oscm][:nova_instance]) -nova_insecure = CrowbarOpenStackHelper.insecure(nova_config) -openstack_args_nova = nova_insecure || keystone_settings["insecure"] ? "--insecure" : "" - -heat_config = Barclamp::Config.load("openstack", "heat", node[:oscm][:heat_instance]) -heat_insecure = CrowbarOpenStackHelper.insecure(heat_config) -openstack_args_heat = heat_insecure || keystone_settings["insecure"] ? "--insecure" : "" - -ruby_block "check_oscm_glance_image" do - block do - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "#{openstack_cmd} #{openstack_args_heat} image list -c Name -f value | egrep '^#{oscm_image}$'" - command_out = shell_out(command) - if command_out.stdout.strip != oscm_image - message = "The image with name '#{oscm_image}' is not found in glance! Please check your oscm proposal attributes or glance image registry." - raise message - end - end - action :create -end - - -register_auth_hash = { - user: keystone_settings["admin_user"], - password: keystone_settings["admin_password"], - tenant: keystone_settings["admin_tenant"] -} - -keystone_register "oscm wakeup keystone" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - action :wakeup -end - -keystone_register "oscm create project" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - tenant_name oscm_project - action :add_tenant -end - -keystone_register "oscm register user" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - user_name oscm_user - user_password oscm_password - tenant_name oscm_project - action :add_user -end - -keystone_register "oscm give user admin role" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - user_name oscm_user - tenant_name oscm_project - role_name "admin" - action :add_access -end - -keystone_register "oscm give user member role" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - user_name oscm_user - tenant_name oscm_project - role_name "Member" - action :add_access -end - -keystone_register "oscm give user _member_ role" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - user_name oscm_user - tenant_name oscm_project - role_name "_member_" - action :add_access -end - -execute "create_oscm_flavor" do - command "#{openstack_cmd} #{openstack_args_nova} flavor create --ram #{oscm_flavor_ram} --disk #{oscm_flavor_disk} \ - --vcpus #{oscm_flavor_vcpus} --private #{oscm_flavor_name}" - not_if "#{openstack_cmd} #{openstack_args_nova} flavor list --all -c Name -f value | egrep -q '^#{oscm_flavor_name}$'" -end - -execute "create_oscm_flavor_access" do - command "#{openstack_cmd} #{openstack_args_nova} flavor set --project #{oscm_project} #{oscm_flavor_name}" - ignore_failure true -end - -bash "create_oscm_keypair_file" do - code <<-EOH - publickey="#{oscm_keypair_publickey}" - mkdir -p "$(dirname "#{oscm_keypair_publickeyfile}")" &> /dev/null - echo "${publickey}" > "#{oscm_keypair_publickeyfile}" -EOH -end - -execute "create_oscm_keypair" do - command "#{openstack_cmd} #{openstack_args_nova} keypair create #{oscm_keypair_name} --public-key #{oscm_keypair_publickeyfile}" - not_if "#{openstack_cmd} #{openstack_args_nova} keypair list -c Name -f value | egrep -q '^#{oscm_keypair_name}$'" -end - -directory "#{oscm_install_path}" do - owner oscm_group - group oscm_group - mode 0640 - recursive true -end - -cookbook_file "#{oscm_install_path}/volumes.yaml" do - source "volumes.yaml" - owner oscm_group - group oscm_group - mode 0640 - action :create -end - -cookbook_file "#{oscm_install_path}/application.yaml" do - source "application.yaml" - owner oscm_group - group oscm_group - mode 0640 - action :create -end - -directory "#{oscm_install_path}/user-data" do - owner oscm_group - group oscm_group - mode 0640 - recursive true -end - -cookbook_file "#{oscm_install_path}/user-data/heat-config" do - source "user-data/heat-config" - owner oscm_group - group oscm_group - mode 0640 - action :create -end - -cookbook_file "#{oscm_install_path}/user-data/deploy-oscmserver" do - source "user-data/deploy-oscmserver" - owner oscm_group - group oscm_group - mode 0640 - action :create -end - -execute "create_oscm_volume_stack" do - command "#{openstack_cmd} #{openstack_args_heat} stack create --parameter data_size=#{oscm_data_volume_size} --parameter logs_size=#{oscm_logs_volume_size} \ - -t #{oscm_install_path}/volumes.yaml --wait #{oscm_volumestack_name}" - not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{oscm_volumestack_name}$'" -end - -ruby_block "get_oscm_volume_ids" do - block do - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell #{oscm_volumestack_name} data_volume_id | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:volume_stack][:data_volume_id] = command_out.stdout.strip - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell #{oscm_volumestack_name} logs_volume_id | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:volume_stack][:logs_volume_id] = command_out.stdout.strip - end - action :create -end - -ruby_block "generate_oscm_crowbar_ssh_keys" do - block do - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "mkdir -p '$(dirname #{oscm_keypair_crowbar_sshkey})'" - command_out = shell_out(command) - command = "[ ! -f #{oscm_keypair_crowbar_sshkey}] && yes y | ssh-keygen -t rsa -f #{oscm_keypair_crowbar_sshkey} -N ''" - command_out = shell_out(command) - end - action :create -end - -execute "create_oscm_instance_stack" do - command lazy { "#{openstack_cmd} #{openstack_args_heat} stack create --parameter logs_volume_id=#{node[:oscm][:openstack][:volume_stack][:logs_volume_id]} \ - --parameter data_volume_id=#{node[:oscm][:openstack][:volume_stack][:data_volume_id]} \ - --parameter image=#{oscm_image} --parameter flavor=#{oscm_flavor_name} \ - --parameter key_name=#{oscm_keypair_name} --parameter floating_network=#{oscm_floating_network} \ - --parameter mail_port=#{node[:oscm][:mail][:port]} --parameter registry_port=#{node[:oscm][:docker][:port]} \ - --parameter-file ssh_cert=#{oscm_keypair_crowbar_sshkey}.pub \ - -t #{oscm_install_path}/application.yaml --wait #{oscm_instancestack_name}" } - not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{oscm_instancestack_name}$'" -end - -ruby_block "get_oscm_floating_ip" do - block do - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} ip_appserver | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:instance_stack][:ip_appserver] = command_out.stdout.strip - end - action :create -end - -if node[:oscm][:api][:protocol] == "https" - ssl_setup "setting up ssl for oscm" do - generate_certs node[:oscm][:ssl][:generate_certs] - certfile node[:oscm][:ssl][:certfile] - keyfile node[:oscm][:ssl][:keyfile] - group oscm_group - fqdn node[:oscm][:ssl][:fqdn].empty? ? node[:oscm][:openstack][:instance_stack][:ip_appserver] : node[:oscm][:ssl][:fqdn] - ca_certs node[:oscm][:ssl][:ca_certs] - end -end - -ruby_block "get_oscm_secrets" do - block do - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} db_password | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:instance_stack][:db_password] = command_out.stdout.strip - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} db_core_password | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:instance_stack][:db_core_password] = command_out.stdout.strip - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} db_app_password | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:instance_stack][:db_app_password] = command_out.stdout.strip - command = "#{openstack_cmd} #{openstack_args_heat} stack output show -f shell --variable output_value #{oscm_instancestack_name} key_secret | grep -Po '(?<=^output_value=\")[^\"]*'" - command_out = shell_out(command) - node[:oscm][:openstack][:instance_stack][:key_secret] = command_out.stdout.strip - end - action :create -end - -template "#{oscm_install_path}/user-data/oscm-config" do - source "oscm.conf.erb" - owner oscm_group - group oscm_group - mode 0640 - variables( - mail: node[:oscm][:mail], - docker: node[:oscm][:docker], - proxy: node[:oscm][:proxy], - host_fqdn: node[:oscm][:ssl][:fqdn].empty? ? node[:oscm][:openstack][:instance_stack][:ip_appserver] : node[:oscm][:ssl][:fqdn], - instance: node[:oscm][:openstack][:instance_stack] - ) -end - -template "#{oscm_install_path}/docker-compose-initdb.yml" do - source "docker-compose-initdb.yml.erb" - owner oscm_group - group oscm_group - mode 0640 - variables( - docker: node[:oscm][:docker] - ) -end - -template "#{oscm_install_path}/docker-compose-oscm.yml" do - source "docker-compose-oscm.yml.erb" - owner oscm_group - group oscm_group - mode 0640 - variables( - docker: node[:oscm][:docker] - ) -end - -template "#{oscm_install_path}/var.env" do - source "var.env.erb" - owner oscm_group - group oscm_group - mode 0640 - variables( - mail: node[:oscm][:mail], - docker: node[:oscm][:docker], - proxy: node[:oscm][:proxy], - host_fqdn: node[:oscm][:ssl][:fqdn].empty? ? node[:oscm][:openstack][:instance_stack][:ip_appserver] : node[:oscm][:ssl][:fqdn], - instance: node[:oscm][:openstack][:instance_stack] - ) -end - -ruby_block "inject_oscm_scripts" do - block do - args = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i #{oscm_keypair_crowbar_sshkey}" - ip_appserver = node[:oscm][:openstack][:instance_stack][:ip_appserver] - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_path}/docker-compose'" - command_out = shell_out(command) - command = "scp #{args} #{oscm_install_path}/user-data/oscm-config #{ip_appserver}:#{oscm_path}/config" - command_out = shell_out(command) - command = "scp #{args} #{oscm_install_path}/docker-compose-*.yml #{ip_appserver}:#{oscm_path}/docker-compose" - command_out = shell_out(command) - command = "scp #{args} #{oscm_install_path}/var.env #{ip_appserver}:#{oscm_path}/docker-compose" - command_out = shell_out(command) - if node[:oscm][:api][:protocol] == "https" - command = "ssh #{args} #{ip_appserver} 'mkdir -p #{oscm_path}/ssl'" - command_out = shell_out(command) - command = "scp #{args} #{oscm_ssl_certfile} #{ip_appserver}:#{oscm_path}/ssl/oscm.crt" - command_out = shell_out(command) - command = "scp #{args} #{oscm_ssl_keyfile} #{ip_appserver}:#{oscm_path}/ssl/oscm.key" - command_out = shell_out(command) - command = "scp #{args} #{oscm_ssl_cacerts} #{ip_appserver}:#{oscm_path}/ssl/oscm.chain" - end - command = "scp #{args} #{oscm_install_path}/user-data/deploy-oscmserver #{ip_appserver}:#{oscm_path}/config" - command_out = shell_out(command) - command = "ssh #{args} #{ip_appserver} 'chmod 755 #{oscm_path}/config/deploy-oscmserver'" - command_out = shell_out(command) - command = "ssh #{args} #{ip_appserver} '#{oscm_path}/config/deploy-oscmserver' &" - command_out = shell_out(command) - end - action :create -end - diff --git a/chef/data_bags/crowbar/template-oscm.json b/chef/data_bags/crowbar/template-escm.json similarity index 73% rename from chef/data_bags/crowbar/template-oscm.json rename to chef/data_bags/crowbar/template-escm.json index 6fa79a8620..5a2abcb3f9 100644 --- a/chef/data_bags/crowbar/template-oscm.json +++ b/chef/data_bags/crowbar/template-escm.json @@ -1,37 +1,37 @@ { - "id": "template-oscm", - "description": "OSCM", + "id": "template-escm", + "description": "ESCM", "attributes": { - "oscm": { + "escm": { "timeout": 300, "debug": false, "keystone_instance": "none", "nova_instance": "none", "heat_instance": "none", "keystone": { - "project": "oscm", - "user": "oscm", - "password": "oscm" + "project": "escm", + "user": "escm", + "password": "escm" }, "openstack": { - "image": "sles12-sp1", + "image": "sles12-sp3", "flavor": { - "name": "oscm.small", - "ram": 2048, + "name": "escm.small", + "ram": 8192, "vcpus": 1, "disk": 20 }, "keypair": { - "name": "oscm", + "name": "escm", "publickey": "" }, "volume_stack": { - "stack_name": "oscm-volumes", + "stack_name": "escm-volumes", "data_volume_size": 1, "logs_volume_size": 1 }, "instance_stack": { - "stack_name": "oscm-instances" + "stack_name": "escm-instances" }, "floating_network": "floating" }, @@ -67,12 +67,12 @@ }, "host_fqdn": "", "ssl": { - "certfile": "/etc/oscm/ssl/certs/signing_cert.pem", - "keyfile": "/etc/oscm/ssl/private/signing_key.pem", + "certfile": "/etc/escm/ssl/certs/signing_cert.pem", + "keyfile": "/etc/escm/ssl/private/signing_key.pem", "generate_certs": false, "fqdn": "", "insecure": false, - "ca_certs": "/etc/oscm/ssl/certs/ca.pem" + "ca_certs": "/etc/escm/ssl/certs/ca.pem" }, "api": { "protocol": "https" @@ -80,19 +80,19 @@ } }, "deployment": { - "oscm": { + "escm": { "crowbar-revision": 0, "crowbar-applied": false, "schema-revision": 100, "element_states": { - "oscm-server": [ "readying", "ready", "applying" ] + "escm-server": [ "readying", "ready", "applying" ] }, "element_order": [ - [ "oscm-server" ] + [ "escm-server" ] ], "elements": {}, "element_run_list_order": { - "oscm-server": 110 + "escm-server": 110 }, "config": { "mode": "full", diff --git a/chef/data_bags/crowbar/template-oscm.schema b/chef/data_bags/crowbar/template-escm.schema similarity index 99% rename from chef/data_bags/crowbar/template-oscm.schema rename to chef/data_bags/crowbar/template-escm.schema index 087895cd47..d09d45b353 100644 --- a/chef/data_bags/crowbar/template-oscm.schema +++ b/chef/data_bags/crowbar/template-escm.schema @@ -8,7 +8,7 @@ "type": "map", "required": true, "mapping": { - "oscm": { + "escm": { "type": "map", "required": true, "mapping": { @@ -139,7 +139,7 @@ "type": "map", "required": true, "mapping": { - "oscm": { + "escm": { "type": "map", "required": true, "mapping": { diff --git a/chef/roles/escm-server.rb b/chef/roles/escm-server.rb new file mode 100644 index 0000000000..fc9d320930 --- /dev/null +++ b/chef/roles/escm-server.rb @@ -0,0 +1,5 @@ +name "escm-server" +description "ESCM Server Role" +run_list("recipe[escm::role_escm_server]") +default_attributes +override_attributes diff --git a/chef/roles/oscm-server.rb b/chef/roles/oscm-server.rb deleted file mode 100644 index 3948922df6..0000000000 --- a/chef/roles/oscm-server.rb +++ /dev/null @@ -1,5 +0,0 @@ -name "oscm-server" -description "OSCM Server Role" -run_list("recipe[oscm::role_oscm_server]") -default_attributes -override_attributes diff --git a/crowbar_framework/app/controllers/oscm_controller.rb b/crowbar_framework/app/controllers/escm_controller.rb similarity index 84% rename from crowbar_framework/app/controllers/oscm_controller.rb rename to crowbar_framework/app/controllers/escm_controller.rb index ef16e99827..e2dabd8a62 100644 --- a/crowbar_framework/app/controllers/oscm_controller.rb +++ b/crowbar_framework/app/controllers/escm_controller.rb @@ -14,12 +14,12 @@ # limitations under the License. # -class OscmController < BarclampController - # Controller for Oscm barclamp +class EscmController < BarclampController + # Controller for ESCM barclamp protected def initialize_service - @service_object = OscmService.new logger + @service_object = EscmService.new logger end end diff --git a/crowbar_framework/app/helpers/barclamp/oscm_helper.rb b/crowbar_framework/app/helpers/barclamp/escm_helper.rb similarity index 92% rename from crowbar_framework/app/helpers/barclamp/oscm_helper.rb rename to crowbar_framework/app/helpers/barclamp/escm_helper.rb index a593352cdd..c6b253331e 100644 --- a/crowbar_framework/app/helpers/barclamp/oscm_helper.rb +++ b/crowbar_framework/app/helpers/barclamp/escm_helper.rb @@ -15,8 +15,8 @@ # module Barclamp - module OscmHelper - def api_protocols_for_oscm(selected) + module EscmHelper + def api_protocols_for_escm(selected) options_for_select( [ ["HTTP", "http"], diff --git a/crowbar_framework/app/models/oscm_service.rb b/crowbar_framework/app/models/escm_service.rb similarity index 86% rename from crowbar_framework/app/models/oscm_service.rb rename to crowbar_framework/app/models/escm_service.rb index e82f77d19c..2270950d80 100644 --- a/crowbar_framework/app/models/oscm_service.rb +++ b/crowbar_framework/app/models/escm_service.rb @@ -14,9 +14,9 @@ # limitations under the License. # -class OscmService < PacemakerServiceObject +class EscmService < PacemakerServiceObject def initialize(thelogger) - @bc_name = "oscm" + @bc_name = "escm" @logger = thelogger end @@ -28,7 +28,7 @@ def self.allow_multiple_proposals? def role_constraints { - "oscm-server" => { + "escm-server" => { "unique" => false, "count" => 1, "admin" => false, @@ -51,7 +51,7 @@ def proposal_dependencies(role) end def create_proposal - @logger.debug("Oscm create_proposal: entering") + @logger.debug("Escm create_proposal: entering") base = super nodes = NodeObject.all @@ -59,18 +59,18 @@ def create_proposal server_nodes = [nodes.first] if server_nodes.empty? base["deployment"][@bc_name]["elements"] = { - "oscm-server" => [server_nodes.first.name] + "escm-server" => [server_nodes.first.name] } unless server_nodes.nil? base["attributes"][@bc_name]["keystone_instance"] = find_dep_proposal("keystone") base["attributes"][@bc_name]["heat_instance"] = find_dep_proposal("heat") - @logger.debug("Oscm create_proposal: exiting") + @logger.debug("Escm create_proposal: exiting") base end def validate_proposal_after_save(proposal) - validate_one_for_role proposal, "oscm-server" + validate_one_for_role proposal, "escm-server" super end diff --git a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml similarity index 93% rename from crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml rename to crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml index 373955de41..4fd259c284 100644 --- a/crowbar_framework/app/views/barclamp/oscm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml @@ -81,10 +81,10 @@ = t(".ssl_header") = select_field %w(api protocol), - :collection => :api_protocols_for_oscm, + :collection => :api_protocols_for_escm, "data-sslprefix" => "ssl", - "data-sslcert" => "/etc/oscm/ssl/certs/signing_cert.pem", - "data-sslkey" => "/etc/oscm/ssl/private/signing_key.pem" + "data-sslcert" => "/etc/escm/ssl/certs/signing_cert.pem", + "data-sslkey" => "/etc/escm/ssl/private/signing_key.pem" #ssl_container %span.help-block diff --git a/crowbar_framework/config/locales/oscm/en.yml b/crowbar_framework/config/locales/escm/en.yml similarity index 96% rename from crowbar_framework/config/locales/oscm/en.yml rename to crowbar_framework/config/locales/escm/en.yml index e0c4b5f6a0..afb259997c 100644 --- a/crowbar_framework/config/locales/oscm/en.yml +++ b/crowbar_framework/config/locales/escm/en.yml @@ -16,7 +16,7 @@ en: barclamp: - oscm: + escm: edit_attributes: keystone_instance: 'Keystone' heat_instance: 'Heat' @@ -28,7 +28,7 @@ en: tls: 'Enable TLS' user: 'User' password: 'Password' - from: 'Oscm Email Address' + from: 'ESCM Email Address' docker_settings: 'Docker Registry' docker: dockerhub: 'Use Docker Hub' @@ -54,7 +54,7 @@ en: openstack: keypair: publickey: "Public Key" - keypair_hint: 'SSH public key of the keypair for openstack instance which hosts OSCM. It cannot be updated.' + keypair_hint: 'SSH public key of the keypair for openstack instance which hosts ESCM. It cannot be updated.' api: protocol: 'Protocol' ssl_header: 'SSL Support' diff --git a/oscm.yml b/escm.yml similarity index 89% rename from oscm.yml rename to escm.yml index 19baba9eb4..0405479dd6 100644 --- a/oscm.yml +++ b/escm.yml @@ -15,9 +15,9 @@ # barclamp: - name: 'oscm' - display: 'OSCM' - description: 'Open Service Catalog Manager' + name: 'escm' + display: 'ESCM' + description: 'Enterprise Service Catalog Manager' version: 1 member: - 'openstack' From a253475be937e4a11da118d18901d9d65cb1c264 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 13 Oct 2017 12:39:59 +0200 Subject: [PATCH 107/181] create config directory in heat-config --- chef/cookbooks/escm/files/default/user-data/heat-config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index c849d9e7bd..ad90b097e8 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -2,6 +2,11 @@ DOCKER_PATH=/docker +# Create the config directory if it does not exist yet +if [ ! -d /etc/oscm/config ]; then + mkdir -p /etc/oscm/config +fi + # Write the SSH public key to root's trusted keys file CERT_STR="$SSH_CERT" echo $CERT_STR >> /root/.ssh/authorized_keys From 419f5df68bf3c87705b09aa48a5de3bcac0bbe62 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 13 Oct 2017 13:55:35 +0200 Subject: [PATCH 108/181] Rename template --- .../escm/templates/default/escm.conf.erb | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 chef/cookbooks/escm/templates/default/escm.conf.erb diff --git a/chef/cookbooks/escm/templates/default/escm.conf.erb b/chef/cookbooks/escm/templates/default/escm.conf.erb new file mode 100644 index 0000000000..7cc8f39bed --- /dev/null +++ b/chef/cookbooks/escm/templates/default/escm.conf.erb @@ -0,0 +1,24 @@ +#proxy +PROXY_ENABLED=<%= @proxy["use_proxy"] %> +PROXY_HTTP_HOST=<%= @proxy["http_host"] %> +PROXY_HTTP_PORT=<%= @proxy["http_port"] %> +PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> +PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> +PROXY_NOPROXY=<%= @proxy["no_proxy"] %> +PROXY_AUTH=<%= @proxy["auth"] %> +PROXY_USER=<%= @proxy["user"] %> +PROXY_PWD=<%= @proxy["password"] %> + +#docker +DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> +DOCKER_REGISTRY_HOST=<%= @docker["host"] %> +DOCKER_REGISTRY_PORT=<%= @docker["port"] %> +DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> +DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> +DOCKER_REGISTRY_USER=<%= @docker["user"] %> +DOCKER_REGISTRY_PWD=<%= @docker["password"] %> + +#host +HOST_FQDN=<%= @host_fqdn %> + + \ No newline at end of file From ab58e9408df8e20f50cec5d87e1e49688ab6653e Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 13 Oct 2017 13:55:46 +0200 Subject: [PATCH 109/181] Rename template --- .../escm/templates/default/oscm.conf.erb | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 chef/cookbooks/escm/templates/default/oscm.conf.erb diff --git a/chef/cookbooks/escm/templates/default/oscm.conf.erb b/chef/cookbooks/escm/templates/default/oscm.conf.erb deleted file mode 100644 index 7cc8f39bed..0000000000 --- a/chef/cookbooks/escm/templates/default/oscm.conf.erb +++ /dev/null @@ -1,24 +0,0 @@ -#proxy -PROXY_ENABLED=<%= @proxy["use_proxy"] %> -PROXY_HTTP_HOST=<%= @proxy["http_host"] %> -PROXY_HTTP_PORT=<%= @proxy["http_port"] %> -PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> -PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> -PROXY_NOPROXY=<%= @proxy["no_proxy"] %> -PROXY_AUTH=<%= @proxy["auth"] %> -PROXY_USER=<%= @proxy["user"] %> -PROXY_PWD=<%= @proxy["password"] %> - -#docker -DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> -DOCKER_REGISTRY_HOST=<%= @docker["host"] %> -DOCKER_REGISTRY_PORT=<%= @docker["port"] %> -DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> -DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> -DOCKER_REGISTRY_USER=<%= @docker["user"] %> -DOCKER_REGISTRY_PWD=<%= @docker["password"] %> - -#host -HOST_FQDN=<%= @host_fqdn %> - - \ No newline at end of file From d8ebc59cf334d6646e59b9f7c1d5e39f1da25d6b Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 13 Oct 2017 14:11:23 +0200 Subject: [PATCH 110/181] no_proxy setting --- chef/cookbooks/escm/attributes/default.rb | 3 +++ chef/cookbooks/escm/templates/default/escm.conf.erb | 6 +++++- chef/data_bags/crowbar/template-escm.json | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/attributes/default.rb b/chef/cookbooks/escm/attributes/default.rb index 67a24cd687..819bd01324 100644 --- a/chef/cookbooks/escm/attributes/default.rb +++ b/chef/cookbooks/escm/attributes/default.rb @@ -14,8 +14,11 @@ # limitations under the License. # +default[:escm][:proxy][:no_proxy_default] = "localhost,127.0.0.1" + default[:escm][:ssl][:certfile] = "/etc/escm/ssl/certs/signing_cert.pem" default[:escm][:ssl][:keyfile] = "/etc/escm/ssl/private/signing_key.pem" default[:escm][:ssl][:generate_certs] = false default[:escm][:ssl][:insecure] = false default[:escm][:ssl][:ca_certs] = "/etc/escm/ssl/certs/ca.pem" + diff --git a/chef/cookbooks/escm/templates/default/escm.conf.erb b/chef/cookbooks/escm/templates/default/escm.conf.erb index 7cc8f39bed..8ad75ed102 100644 --- a/chef/cookbooks/escm/templates/default/escm.conf.erb +++ b/chef/cookbooks/escm/templates/default/escm.conf.erb @@ -4,7 +4,11 @@ PROXY_HTTP_HOST=<%= @proxy["http_host"] %> PROXY_HTTP_PORT=<%= @proxy["http_port"] %> PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> -PROXY_NOPROXY=<%= @proxy["no_proxy"] %> +<% if @proxy["no_proxy"].empty? %> +PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> +<% else %> +PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> +<% end%> PROXY_AUTH=<%= @proxy["auth"] %> PROXY_USER=<%= @proxy["user"] %> PROXY_PWD=<%= @proxy["password"] %> diff --git a/chef/data_bags/crowbar/template-escm.json b/chef/data_bags/crowbar/template-escm.json index 5a2abcb3f9..99936a7aaf 100644 --- a/chef/data_bags/crowbar/template-escm.json +++ b/chef/data_bags/crowbar/template-escm.json @@ -60,7 +60,7 @@ "http_port": 0, "https_host": "", "https_port": 0, - "no_proxy": "localhost,127.0.0.1", + "no_proxy": "", "auth": false, "user": "", "password": "" From 59d8e1006d3eae7a09a94a85d4ad78a2beb31233 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 09:53:38 +0200 Subject: [PATCH 111/181] Rename oscm to escm --- chef/cookbooks/escm/recipes/role_escm_server.rb | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 chef/cookbooks/escm/recipes/role_escm_server.rb diff --git a/chef/cookbooks/escm/recipes/role_escm_server.rb b/chef/cookbooks/escm/recipes/role_escm_server.rb new file mode 100644 index 0000000000..f334e59351 --- /dev/null +++ b/chef/cookbooks/escm/recipes/role_escm_server.rb @@ -0,0 +1,3 @@ +if CrowbarRoleRecipe.node_state_valid_for_role?(node, "escm", "escm-server") + include_recipe "#{@cookbook_name}::server" +end From 4dd1c1a35eb2005b57cfe5d31cb891f9ed553d5b Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 09:53:56 +0200 Subject: [PATCH 112/181] Rename oscm to escm --- chef/cookbooks/escm/recipes/role_oscm_server.rb | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 chef/cookbooks/escm/recipes/role_oscm_server.rb diff --git a/chef/cookbooks/escm/recipes/role_oscm_server.rb b/chef/cookbooks/escm/recipes/role_oscm_server.rb deleted file mode 100644 index f334e59351..0000000000 --- a/chef/cookbooks/escm/recipes/role_oscm_server.rb +++ /dev/null @@ -1,3 +0,0 @@ -if CrowbarRoleRecipe.node_state_valid_for_role?(node, "escm", "escm-server") - include_recipe "#{@cookbook_name}::server" -end From 0e0c1e43e396f09d2cc7f57346233318e8c9f4ae Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 10:11:07 +0200 Subject: [PATCH 113/181] Check the glance image for existence --- chef/cookbooks/escm/recipes/server.rb | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 728faf1639..e25c53f041 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -61,20 +61,6 @@ heat_insecure = CrowbarOpenStackHelper.insecure(heat_config) openstack_args_heat = heat_insecure || keystone_settings["insecure"] ? "--insecure" : "" -ruby_block "check_escm_glance_image" do - block do - Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "#{openstack_cmd} #{openstack_args_heat} image list -c Name -f value | egrep '^#{escm_image}$'" - command_out = shell_out(command) - if command_out.stdout.strip != escm_image - message = "The image with name '#{escm_image}' is not found in glance! Please check your escm proposal attributes or glance image registry." - raise message - end - end - action :create -end - - register_auth_hash = { user: keystone_settings["admin_user"], password: keystone_settings["admin_password"], @@ -148,6 +134,19 @@ action :add_access end +ruby_block "check_escm_glance_image" do + block do + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "#{openstack_cmd} #{openstack_args_heat} image list -c Name -f value | egrep '^#{escm_image}$'" + command_out = shell_out(command) + if command_out.stdout.strip != escm_image + message = "The image with name '#{escm_image}' is not found in glance! Please check your escm proposal attributes or glance image registry." + raise message + end + end + action :create +end + execute "create_escm_flavor" do command "#{openstack_cmd} #{openstack_args_nova} flavor create --ram #{escm_flavor_ram} --disk #{escm_flavor_disk} \ --vcpus #{escm_flavor_vcpus} --private #{escm_flavor_name}" From 64a35a23e1edccab803ba0f6745fcd24071c1538 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 10:23:11 +0200 Subject: [PATCH 114/181] Rename --- .../files/default/user-data/deploy-escmserver | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 chef/cookbooks/escm/files/default/user-data/deploy-escmserver diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver new file mode 100644 index 0000000000..fec8f08e01 --- /dev/null +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -0,0 +1,177 @@ +#!/bin/sh +# Enable command traces +set -x +# Exit on error +set -e +# Redirect all output to log file +exec &> /var/log/setup-machine.log + +# Variables for this script +CONFIG_BASE=/etc/oscm +CONFIG_PATH=${CONFIG_BASE}/config +SSL_CONFIG_PATH=${CONFIG_BASE}/ssl +COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose +DOCKER_PATH=/docker + +# Enable automatic exporting of variables +set -a +# Read configuration files +source ${CONFIG_PATH}/oscm-config +# Disable automatic exporting of variables +set +a + +# If containers are running, stop them +if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then + if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0" ]; then + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f + fi +fi + +# Create Docker directories if they do not exist yet +for docker_directory in \ + ${DOCKER_PATH}/data/oscm-db/data \ + ${DOCKER_PATH}/config/brandings\ + ${DOCKER_PATH}/config/certs \ + ${DOCKER_PATH}/config/privkey/oscm-core \ + ${DOCKER_PATH}/config/privkey/oscm-app \ + ${DOCKER_PATH}/config/privkey/oscm-birt \ + ${DOCKER_PATH}/config/privkey/oscm-branding \ + ${DOCKER_PATH}/logs/oscm-app \ + ${DOCKER_PATH}/logs/oscm-birt \ + ${DOCKER_PATH}/logs/oscm-branding \ + ${DOCKER_PATH}/logs/oscm-core \ + ${DOCKER_PATH}/logs/oscm-db; do + if [ ! -d ${docker_directory} ]; then + mkdir -p ${docker_directory} + fi +done + +# Create Docker log files if they do not exist yet +for docker_log_file in \ + ${DOCKER_PATH}/logs/oscm-app/oscm-app.out.log \ + ${DOCKER_PATH}/logs/oscm-birt/oscm-birt.out.log \ + ${DOCKER_PATH}/logs/oscm-branding/oscm-branding.out.log \ + ${DOCKER_PATH}/logs/oscm-core/oscm-core.out.log \ + ${DOCKER_PATH}/logs/oscm-db/oscm-db.out.log; do + if [ ! -f {docker_log_file} ]; then + touch ${docker_log_file} + chmod 640 ${docker_log_file} + fi +done + +# Create rsyslog configuration for Docker log files +if [ ! -f /etc/rsyslog.d/oscm.conf ]; then + cat < /etc/rsyslog.d/oscm.conf +local0.* /docker/logs/oscm-db/oscm-db.out.log +local1.* /docker/logs/oscm-core/oscm-core.out.log +local2.* /docker/logs/oscm-app/oscm-app.out.log +local3.* /docker/logs/oscm-birt/oscm-birt.out.log +local4.* /docker/logs/oscm-branding/oscm-branding.out.log +EOF + sed -i '/^local/d' /etc/rsyslog.conf + systemctl restart rsyslog +fi + +# Copy Docker Compose files +if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then + cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml +fi +if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ]; then + cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ${DOCKER_PATH}/docker-compose-oscm.yml +fi +if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then + cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env +fi + +# Copy certificate files +if [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key +fi +if [ -f ${SSL_CONFIG_PATH}/oscm.crt ]; then + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt +fi +if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then + echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain +fi + +# If http proxy is enabled, create Docker config file +if [ ${PROXY_ENABLED} == "true" ]; then + if [ ! -d /etc/systemd/system/docker.service.d ]; then + mkdir -p /etc/systemd/system/docker.service.d + fi + if [ ${PROXY_AUTH} == "true" ]; then + cat < /etc/systemd/system/docker.service.d/http-proxy.conf +[Service] +Environment="HTTP_PROXY=http://${PROXY_USER}:${PROXY_PWD}@${PROXY_HTTP_HOST}:${PROXY_HTTP_PORT}/" "HTTPS_PROXY=http://${PROXY_USER}:${PROXY_PWD}@${PROXY_HTTPS_HOST}:${PROXY_HTTPS_PORT}/" "NO_PROXY=${PROXY_NOPROXY}" +EOF + else + cat < /etc/systemd/system/docker.service.d/http-proxy.conf +[Service] +Environment="HTTP_PROXY=http://${PROXY_HTTP_HOST}:${PROXY_HTTP_PORT}/" "HTTPS_PROXY=http://${PROXY_HTTPS_HOST}:${PROXY_HTTPS_PORT}/" "NO_PROXY=${PROXY_NOPROXY}" +EOF + fi + systemctl daemon-reload + systemctl restart docker +# Otherwise remove Docker config file with proxy config +else + if [ -f /etc/systemd/system/docker.service.d/http-proxy.conf ]; then + rm -f /etc/systemd/system/docker.service.d/http-proxy.conf + systemctl daemon-reload + systemctl restart docker + fi +fi + +# If Docker registry authentication is enabled, create auth info file +if [ ${DOCKER_REGISTRY_AUTH} == "true" ]; then + #Create auth file for Docker registry + REGISTRY_USER_PASS_BASE64=$(printf "${DOCKER_REGISTRY_USER}:${DOCKER_REGISTRY_PWD}" | base64) + mkdir -p /root/.docker + cat < /root/.docker/config.json +{ + "auths": { + "${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}": { + "auth": "${REGISTRY_USER_PASS_BASE64}" + } + } +} +EOF +# Otherwise remove auth info file +else + if [ -f /root/.docker/config.json ]; then + rm -f /root/.docker/config.json + fi +fi + +# If default Docker registry (Dockerhub) is used, remove insecure registry config +if [ ${DOCKER_REGISTRY_DOCKERHUB} = "true" ]; then + DOCKER_IMAGES_SOURCE="${DOCKER_REGISTRY_ORGANIZATION}/" + if [ -f /etc/docker/daemon.json ]; then + rm -f /etc/docker/daemon.json + systemctl restart docker + fi +# Otherwise, custom registry is used, so create insecure registry config +else + cat < /etc/docker/daemon.json +{ + "insecure-registries" : ["${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}"] +} +EOF + systemctl restart docker +fi + +# Pull images +egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull +egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-oscm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull + +# Initialize databases +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up -d oscm-db +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-core +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-jms +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-app +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-controller-openstack +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml stop +docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f + +# Start application containers +docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml up -d From fe68186f409217d97d8f30c9a89b6ea6a70bacfc Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 10:23:21 +0200 Subject: [PATCH 115/181] Rename --- .../files/default/user-data/deploy-oscmserver | 177 ------------------ 1 file changed, 177 deletions(-) delete mode 100644 chef/cookbooks/escm/files/default/user-data/deploy-oscmserver diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-oscmserver b/chef/cookbooks/escm/files/default/user-data/deploy-oscmserver deleted file mode 100644 index fec8f08e01..0000000000 --- a/chef/cookbooks/escm/files/default/user-data/deploy-oscmserver +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/sh -# Enable command traces -set -x -# Exit on error -set -e -# Redirect all output to log file -exec &> /var/log/setup-machine.log - -# Variables for this script -CONFIG_BASE=/etc/oscm -CONFIG_PATH=${CONFIG_BASE}/config -SSL_CONFIG_PATH=${CONFIG_BASE}/ssl -COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose -DOCKER_PATH=/docker - -# Enable automatic exporting of variables -set -a -# Read configuration files -source ${CONFIG_PATH}/oscm-config -# Disable automatic exporting of variables -set +a - -# If containers are running, stop them -if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then - if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0" ]; then - docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop - docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f - fi -fi - -# Create Docker directories if they do not exist yet -for docker_directory in \ - ${DOCKER_PATH}/data/oscm-db/data \ - ${DOCKER_PATH}/config/brandings\ - ${DOCKER_PATH}/config/certs \ - ${DOCKER_PATH}/config/privkey/oscm-core \ - ${DOCKER_PATH}/config/privkey/oscm-app \ - ${DOCKER_PATH}/config/privkey/oscm-birt \ - ${DOCKER_PATH}/config/privkey/oscm-branding \ - ${DOCKER_PATH}/logs/oscm-app \ - ${DOCKER_PATH}/logs/oscm-birt \ - ${DOCKER_PATH}/logs/oscm-branding \ - ${DOCKER_PATH}/logs/oscm-core \ - ${DOCKER_PATH}/logs/oscm-db; do - if [ ! -d ${docker_directory} ]; then - mkdir -p ${docker_directory} - fi -done - -# Create Docker log files if they do not exist yet -for docker_log_file in \ - ${DOCKER_PATH}/logs/oscm-app/oscm-app.out.log \ - ${DOCKER_PATH}/logs/oscm-birt/oscm-birt.out.log \ - ${DOCKER_PATH}/logs/oscm-branding/oscm-branding.out.log \ - ${DOCKER_PATH}/logs/oscm-core/oscm-core.out.log \ - ${DOCKER_PATH}/logs/oscm-db/oscm-db.out.log; do - if [ ! -f {docker_log_file} ]; then - touch ${docker_log_file} - chmod 640 ${docker_log_file} - fi -done - -# Create rsyslog configuration for Docker log files -if [ ! -f /etc/rsyslog.d/oscm.conf ]; then - cat < /etc/rsyslog.d/oscm.conf -local0.* /docker/logs/oscm-db/oscm-db.out.log -local1.* /docker/logs/oscm-core/oscm-core.out.log -local2.* /docker/logs/oscm-app/oscm-app.out.log -local3.* /docker/logs/oscm-birt/oscm-birt.out.log -local4.* /docker/logs/oscm-branding/oscm-branding.out.log -EOF - sed -i '/^local/d' /etc/rsyslog.conf - systemctl restart rsyslog -fi - -# Copy Docker Compose files -if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then - cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml -fi -if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ]; then - cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ${DOCKER_PATH}/docker-compose-oscm.yml -fi -if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then - cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env -fi - -# Copy certificate files -if [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key -fi -if [ -f ${SSL_CONFIG_PATH}/oscm.crt ]; then - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt -fi -if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain -fi - -# If http proxy is enabled, create Docker config file -if [ ${PROXY_ENABLED} == "true" ]; then - if [ ! -d /etc/systemd/system/docker.service.d ]; then - mkdir -p /etc/systemd/system/docker.service.d - fi - if [ ${PROXY_AUTH} == "true" ]; then - cat < /etc/systemd/system/docker.service.d/http-proxy.conf -[Service] -Environment="HTTP_PROXY=http://${PROXY_USER}:${PROXY_PWD}@${PROXY_HTTP_HOST}:${PROXY_HTTP_PORT}/" "HTTPS_PROXY=http://${PROXY_USER}:${PROXY_PWD}@${PROXY_HTTPS_HOST}:${PROXY_HTTPS_PORT}/" "NO_PROXY=${PROXY_NOPROXY}" -EOF - else - cat < /etc/systemd/system/docker.service.d/http-proxy.conf -[Service] -Environment="HTTP_PROXY=http://${PROXY_HTTP_HOST}:${PROXY_HTTP_PORT}/" "HTTPS_PROXY=http://${PROXY_HTTPS_HOST}:${PROXY_HTTPS_PORT}/" "NO_PROXY=${PROXY_NOPROXY}" -EOF - fi - systemctl daemon-reload - systemctl restart docker -# Otherwise remove Docker config file with proxy config -else - if [ -f /etc/systemd/system/docker.service.d/http-proxy.conf ]; then - rm -f /etc/systemd/system/docker.service.d/http-proxy.conf - systemctl daemon-reload - systemctl restart docker - fi -fi - -# If Docker registry authentication is enabled, create auth info file -if [ ${DOCKER_REGISTRY_AUTH} == "true" ]; then - #Create auth file for Docker registry - REGISTRY_USER_PASS_BASE64=$(printf "${DOCKER_REGISTRY_USER}:${DOCKER_REGISTRY_PWD}" | base64) - mkdir -p /root/.docker - cat < /root/.docker/config.json -{ - "auths": { - "${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}": { - "auth": "${REGISTRY_USER_PASS_BASE64}" - } - } -} -EOF -# Otherwise remove auth info file -else - if [ -f /root/.docker/config.json ]; then - rm -f /root/.docker/config.json - fi -fi - -# If default Docker registry (Dockerhub) is used, remove insecure registry config -if [ ${DOCKER_REGISTRY_DOCKERHUB} = "true" ]; then - DOCKER_IMAGES_SOURCE="${DOCKER_REGISTRY_ORGANIZATION}/" - if [ -f /etc/docker/daemon.json ]; then - rm -f /etc/docker/daemon.json - systemctl restart docker - fi -# Otherwise, custom registry is used, so create insecure registry config -else - cat < /etc/docker/daemon.json -{ - "insecure-registries" : ["${DOCKER_REGISTRY_HOST}:${DOCKER_REGISTRY_PORT}"] -} -EOF - systemctl restart docker -fi - -# Pull images -egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull -egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-oscm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull - -# Initialize databases -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up -d oscm-db -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-core -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-jms -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-app -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-controller-openstack -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml stop -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f - -# Start application containers -docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml up -d From 5c0ce7cd1d17a859ff263a6fa9e4343c8dfd86c1 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 12:47:31 +0200 Subject: [PATCH 116/181] rename --- .../default/docker-compose-escm.yml.erb | 102 ++++++++++++++++++ .../escm/templates/default/escm.conf.erb | 11 +- 2 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb diff --git a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb new file mode 100644 index 0000000000..4834234720 --- /dev/null +++ b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb @@ -0,0 +1,102 @@ +version: "3" +services: + oscm-db: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> +<% else %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> +<% end %> + container_name: oscm-db + logging: + driver: syslog + options: + syslog-facility: "local0" + volumes: + - /docker/data/oscm-db/data:/var/lib/postgresql/data + ports: + - 5432:5432 + + oscm-core: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-core:"+@docker["tag"] %> +<% else %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> +<% end %> + container_name: oscm-core + logging: + driver: syslog + options: + syslog-facility: "local1" + env_file: var.env + environment: + - JPDA_ADDRESS=8000 + - JPDA_TRANSPORT=dt_socket + links: + - oscm-db:oscm-db + volumes: + - /docker/config/oscm-core/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 8080:8080 + - 8081:8081 + + oscm-app: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-app:"+@docker["tag"] %> +<% else %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> +<% end %> + container_name: oscm-app + logging: + driver: syslog + options: + syslog-facility: "local2" + env_file: var.env + environment: + - JPDA_ADDRESS=8000 + - JPDA_TRANSPORT=dt_socket + links: + - oscm-db:oscm-db + volumes: + - /docker/config/oscm-app/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 8880:8880 + - 8881:8881 + + oscm-birt: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-birt:"+@docker["tag"] %> +<% else %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> +<% end %> + container_name: oscm-birt + logging: + driver: syslog + options: + syslog-facility: "local3" + env_file: var.env + volumes: + - /docker/config/oscm-birt/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 8181:8181 + + oscm-branding: +<% if @docker["dockerhub"] == true %> + image: <%= @docker["organization"]+"/oscm-branding:"+@docker["tag"] %> +<% else %> + image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> +<% end %> + container_name: oscm-branding + logging: + driver: syslog + options: + syslog-facility: "local4" + env_file: var.env + volumes: + - /docker/config/brandings:/brandings:ro + - /docker/config/oscm-branding/privkey:/privkey + - /docker/config/certs:/certs + ports: + - 4443:443 diff --git a/chef/cookbooks/escm/templates/default/escm.conf.erb b/chef/cookbooks/escm/templates/default/escm.conf.erb index 8ad75ed102..9363ae77e2 100644 --- a/chef/cookbooks/escm/templates/default/escm.conf.erb +++ b/chef/cookbooks/escm/templates/default/escm.conf.erb @@ -4,11 +4,12 @@ PROXY_HTTP_HOST=<%= @proxy["http_host"] %> PROXY_HTTP_PORT=<%= @proxy["http_port"] %> PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> -<% if @proxy["no_proxy"].empty? %> -PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> -<% else %> -PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> -<% end%> +#<% if @proxy["no_proxy"].empty? %> +#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> +#<% else %> +#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> +#<% end %> +PROXY_NOPROXY=<%= @proxy["no_proxy"] %> PROXY_AUTH=<%= @proxy["auth"] %> PROXY_USER=<%= @proxy["user"] %> PROXY_PWD=<%= @proxy["password"] %> From 1ea5490a012dc05347cb33635e490084fc9befca Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 12:49:44 +0200 Subject: [PATCH 117/181] rename --- .../default/docker-compose-oscm.yml.erb | 102 ------------------ 1 file changed, 102 deletions(-) delete mode 100644 chef/cookbooks/escm/templates/default/docker-compose-oscm.yml.erb diff --git a/chef/cookbooks/escm/templates/default/docker-compose-oscm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-oscm.yml.erb deleted file mode 100644 index 4834234720..0000000000 --- a/chef/cookbooks/escm/templates/default/docker-compose-oscm.yml.erb +++ /dev/null @@ -1,102 +0,0 @@ -version: "3" -services: - oscm-db: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> -<% end %> - container_name: oscm-db - logging: - driver: syslog - options: - syslog-facility: "local0" - volumes: - - /docker/data/oscm-db/data:/var/lib/postgresql/data - ports: - - 5432:5432 - - oscm-core: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-core:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> -<% end %> - container_name: oscm-core - logging: - driver: syslog - options: - syslog-facility: "local1" - env_file: var.env - environment: - - JPDA_ADDRESS=8000 - - JPDA_TRANSPORT=dt_socket - links: - - oscm-db:oscm-db - volumes: - - /docker/config/oscm-core/privkey:/privkey - - /docker/config/certs:/certs - ports: - - 8080:8080 - - 8081:8081 - - oscm-app: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-app:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> -<% end %> - container_name: oscm-app - logging: - driver: syslog - options: - syslog-facility: "local2" - env_file: var.env - environment: - - JPDA_ADDRESS=8000 - - JPDA_TRANSPORT=dt_socket - links: - - oscm-db:oscm-db - volumes: - - /docker/config/oscm-app/privkey:/privkey - - /docker/config/certs:/certs - ports: - - 8880:8880 - - 8881:8881 - - oscm-birt: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-birt:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> -<% end %> - container_name: oscm-birt - logging: - driver: syslog - options: - syslog-facility: "local3" - env_file: var.env - volumes: - - /docker/config/oscm-birt/privkey:/privkey - - /docker/config/certs:/certs - ports: - - 8181:8181 - - oscm-branding: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-branding:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> -<% end %> - container_name: oscm-branding - logging: - driver: syslog - options: - syslog-facility: "local4" - env_file: var.env - volumes: - - /docker/config/brandings:/brandings:ro - - /docker/config/oscm-branding/privkey:/privkey - - /docker/config/certs:/certs - ports: - - 4443:443 From d82f0691e3e1ee2cbbdc373c5175f2efd54cf268 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 13:10:10 +0200 Subject: [PATCH 118/181] Rename --- .../files/default/user-data/deploy-escmserver | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index fec8f08e01..b9b984b0fd 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -7,7 +7,7 @@ set -e exec &> /var/log/setup-machine.log # Variables for this script -CONFIG_BASE=/etc/oscm +CONFIG_BASE=/etc/escm CONFIG_PATH=${CONFIG_BASE}/config SSL_CONFIG_PATH=${CONFIG_BASE}/ssl COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose @@ -16,15 +16,15 @@ DOCKER_PATH=/docker # Enable automatic exporting of variables set -a # Read configuration files -source ${CONFIG_PATH}/oscm-config +source ${CONFIG_PATH}/escm-config # Disable automatic exporting of variables set +a # If containers are running, stop them -if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then - if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0" ]; then - docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop - docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f +if [ -f ${DOCKER_PATH}/docker-compose-escm.yml ]; then + if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml ps -q | wc -l) != "0" ]; then + docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml stop + docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml rm -f fi fi @@ -77,8 +77,8 @@ fi if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml fi -if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ]; then - cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-oscm.yml ${DOCKER_PATH}/docker-compose-oscm.yml +if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ]; then + cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ${DOCKER_PATH}/docker-compose-escm.yml fi if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env @@ -162,7 +162,7 @@ fi # Pull images egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull -egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-oscm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull +egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-escm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull # Initialize databases docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up -d oscm-db @@ -174,4 +174,4 @@ docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml stop docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f # Start application containers -docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml up -d +docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml up -d From d2d4b905aa9b50e44100cf55854492b36e6b6718 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Mon, 16 Oct 2017 13:24:16 +0200 Subject: [PATCH 119/181] Rename --- chef/cookbooks/escm/files/default/user-data/heat-config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index ad90b097e8..bb4e17a284 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -3,8 +3,8 @@ DOCKER_PATH=/docker # Create the config directory if it does not exist yet -if [ ! -d /etc/oscm/config ]; then - mkdir -p /etc/oscm/config +if [ ! -d /etc/escm/config ]; then + mkdir -p /etc/escm/config fi # Write the SSH public key to root's trusted keys file From 1f37ee97cb30b25c8256a1710e180b432b2f6fbc Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 16 Oct 2017 14:21:07 +0200 Subject: [PATCH 120/181] add debug log for heat-config --- chef/cookbooks/escm/files/default/user-data/heat-config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index ad90b097e8..9474d799c3 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -1,5 +1,12 @@ #!/bin/sh - +# Enable command traces +set -x +# Exit on error +set -e +# Redirect all output to log file +exec &> /var/log/setup-cloud.log + +# Variables for this script DOCKER_PATH=/docker # Create the config directory if it does not exist yet From 8eedc5b34064a6b9de45b44fc08c96ed81dd93f4 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 16 Oct 2017 14:51:54 +0200 Subject: [PATCH 121/181] change format of heat variables in heat-config --- chef/cookbooks/escm/files/default/user-data/heat-config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index 646d3e4e1c..99931606f1 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -22,8 +22,8 @@ echo $CERT_STR >> /root/.ssh/authorized_keys # The Cinder volume ids are truncated to 27 characters when the device # link in /dev/disk/by-id/ is created. Account for this by truncating the # device name accordingly. -volume_dev_logs=$(printf '%.43s\n' ${LOGS_VOLUME_DEV}) -volume_dev_data=$(printf '%.43s\n' ${DATA_VOLUME_DEV}) +volume_dev_logs=$(printf '%.43s\n' $LOGS_VOLUME_DEV) +volume_dev_data=$(printf '%.43s\n' $DATA_VOLUME_DEV) # Create a file system on the Cinder volumes if ! file -Ls ${volume_dev_logs} | grep -q "filesystem data" ; then From 752e6492ac822c12dbbdffc76d4db6fc962b3c8d Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 16 Oct 2017 18:01:18 +0200 Subject: [PATCH 122/181] change directory structure --- .../files/default/user-data/deploy-escmserver | 16 ++++++++-------- .../default/docker-compose-escm.yml.erb | 18 +++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index b9b984b0fd..55c127bb29 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -31,12 +31,12 @@ fi # Create Docker directories if they do not exist yet for docker_directory in \ ${DOCKER_PATH}/data/oscm-db/data \ - ${DOCKER_PATH}/config/brandings\ ${DOCKER_PATH}/config/certs \ - ${DOCKER_PATH}/config/privkey/oscm-core \ - ${DOCKER_PATH}/config/privkey/oscm-app \ - ${DOCKER_PATH}/config/privkey/oscm-birt \ - ${DOCKER_PATH}/config/privkey/oscm-branding \ + ${DOCKER_PATH}/config/oscm-branding/brandings \ + ${DOCKER_PATH}/config/oscm-core/privkey \ + ${DOCKER_PATH}/config/oscm-app/privkey \ + ${DOCKER_PATH}/config/oscm-birt/privkey \ + ${DOCKER_PATH}/config/oscm-branding/privkey \ ${DOCKER_PATH}/logs/oscm-app \ ${DOCKER_PATH}/logs/oscm-birt \ ${DOCKER_PATH}/logs/oscm-branding \ @@ -86,13 +86,13 @@ fi # Copy certificate files if [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key + echo ${DOCKER_PATH}/config/oscm-core/privkey ${DOCKER_PATH}/config/oscm-app/privkey ${DOCKER_PATH}/config/oscm-birt/privkey ${DOCKER_PATH}/config/oscm-branding/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key fi if [ -f ${SSL_CONFIG_PATH}/oscm.crt ]; then - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt + echo ${DOCKER_PATH}/config/oscm-core/privkey ${DOCKER_PATH}/config/oscm-app/privkey ${DOCKER_PATH}/config/oscm-birt/privkey ${DOCKER_PATH}/config/oscm-branding/privkey ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt fi if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then - echo ${DOCKER_PATH}/config/privkey/oscm-core ${DOCKER_PATH}/config/privkey/oscm-app ${DOCKER_PATH}/config/privkey/oscm-birt ${DOCKER_PATH}/config/privkey/oscm-branding | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain + echo ${DOCKER_PATH}/config/oscm-core/privkey ${DOCKER_PATH}/config/oscm-app/privkey ${DOCKER_PATH}/config/oscm-birt/privkey ${DOCKER_PATH}/config/oscm-branding/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain fi # If http proxy is enabled, create Docker config file diff --git a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb index 4834234720..aac0e1be29 100644 --- a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb +++ b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb @@ -34,8 +34,8 @@ services: links: - oscm-db:oscm-db volumes: - - /docker/config/oscm-core/privkey:/privkey - - /docker/config/certs:/certs + - /docker/config/oscm-core/privkey:/import/privkey + - /docker/config/certs:/import/certs ports: - 8080:8080 - 8081:8081 @@ -58,8 +58,8 @@ services: links: - oscm-db:oscm-db volumes: - - /docker/config/oscm-app/privkey:/privkey - - /docker/config/certs:/certs + - /docker/config/oscm-app/privkey:/import/privkey + - /docker/config/certs:/import/certs ports: - 8880:8880 - 8881:8881 @@ -77,8 +77,8 @@ services: syslog-facility: "local3" env_file: var.env volumes: - - /docker/config/oscm-birt/privkey:/privkey - - /docker/config/certs:/certs + - /docker/config/oscm-birt/privkey:/import/privkey + - /docker/config/certs:/import/certs ports: - 8181:8181 @@ -95,8 +95,8 @@ services: syslog-facility: "local4" env_file: var.env volumes: - - /docker/config/brandings:/brandings:ro - - /docker/config/oscm-branding/privkey:/privkey - - /docker/config/certs:/certs + - /docker/config/brandings:/import:ro + - /docker/config/oscm-branding/privkey:/import/privkey + - /docker/config/certs:/import/certs ports: - 4443:443 From de77359fdd0e21a3c726c24cbe1feba2c6355b4d Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 17 Oct 2017 11:25:36 +0200 Subject: [PATCH 123/181] fix typo in heat-config --- chef/cookbooks/escm/files/default/user-data/heat-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index 99931606f1..4775bd1e38 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -40,7 +40,7 @@ if ! grep -qs "${volume_dev_logs}" /etc/fstab; then fi if ! grep -qs "${volume_dev_data}" /etc/fstab; then - echo "${volume_dev_data} ${DOCKER_PATH}/logs ext4 defaults 1 1" >> /etc/fstab + echo "${volume_dev_data} ${DOCKER_PATH}/data ext4 defaults 1 1" >> /etc/fstab fi # Mount the Cinder volumes if they are not mounted yet From 91168af62d887e2b5540ae2244c9727de10c3203 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Tue, 17 Oct 2017 15:55:48 +0200 Subject: [PATCH 124/181] Fix error --- chef/cookbooks/escm/recipes/server.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index e25c53f041..3b030d6ed5 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -161,7 +161,7 @@ bash "create_escm_keypair_file" do code <<-EOH publickey="#{escm_keypair_publickey}" - mkdir -p "$(dirname "#{escm_keypair_publickeyfile}")" &> /dev/null + mkdir -p $(dirname #{escm_keypair_publickeyfile}) echo "${publickey}" > "#{escm_keypair_publickeyfile}" EOH end @@ -241,7 +241,7 @@ Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) command = "mkdir -p '$(dirname #{escm_keypair_crowbar_sshkey})'" command_out = shell_out(command) - command = "[ ! -f #{escm_keypair_crowbar_sshkey}] && yes y | ssh-keygen -t rsa -f #{escm_keypair_crowbar_sshkey} -N ''" + command = "[ ! -f #{escm_keypair_crowbar_sshkey} ] && yes y | ssh-keygen -t rsa -f #{escm_keypair_crowbar_sshkey} -N ''" command_out = shell_out(command) end action :create @@ -351,6 +351,8 @@ args = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i #{escm_keypair_crowbar_sshkey}" ip_appserver = node[:escm][:openstack][:instance_stack][:ip_appserver] Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/config'" + command_out = shell_out(command) command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/docker-compose'" command_out = shell_out(command) command = "scp #{args} #{escm_install_path}/user-data/escm-config #{ip_appserver}:#{escm_path}/config" @@ -368,7 +370,7 @@ command_out = shell_out(command) command = "scp #{args} #{escm_ssl_cacerts} #{ip_appserver}:#{escm_path}/ssl/escm.chain" end - command = "scp #{args} #{escm_install_path}/user-data/deploy-escmserver #{ip_appserver}:#{escm_path}/config" + command = "scp #{args} #{escm_install_path}/user-data/deploy-escmserver #{ip_appserver}:#{escm_path}/config/" command_out = shell_out(command) command = "ssh #{args} #{ip_appserver} 'chmod 755 #{escm_path}/config/deploy-escmserver'" command_out = shell_out(command) From 0c204a3ab41609063a3285d845cff07a47a98174 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Wed, 18 Oct 2017 15:13:17 +0200 Subject: [PATCH 125/181] Change the birt container ports --- chef/cookbooks/escm/files/default/application.yaml | 10 ++++++++++ .../escm/templates/default/docker-compose-escm.yml.erb | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/application.yaml b/chef/cookbooks/escm/files/default/application.yaml index 782c6188ef..e3ff99cff5 100644 --- a/chef/cookbooks/escm/files/default/application.yaml +++ b/chef/cookbooks/escm/files/default/application.yaml @@ -131,6 +131,16 @@ resources: protocol: tcp port_range_min: 8881 port_range_max: 8881 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8680 + port_range_max: 8680 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8681 + port_range_max: 8681 - direction: egress remote_ip_prefix: 0.0.0.0/0 protocol: tcp diff --git a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb index aac0e1be29..80e0dab6b3 100644 --- a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb +++ b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb @@ -80,7 +80,8 @@ services: - /docker/config/oscm-birt/privkey:/import/privkey - /docker/config/certs:/import/certs ports: - - 8181:8181 + - 8680:8680 + - 8681:8681 oscm-branding: <% if @docker["dockerhub"] == true %> From 20edef8ba8005f8730859349002836fbd8418d60 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 18 Oct 2017 15:15:57 +0200 Subject: [PATCH 126/181] re-introduce success message in heat-config --- chef/cookbooks/escm/files/default/user-data/heat-config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index 4775bd1e38..88e74310fb 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -57,3 +57,7 @@ if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then fi mount ${volume_dev_data} fi + +# Signal to OpenStack that we're finished and ready +WAIT_CURL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" +eval $WAIT_CURL From 7e6a0b8b3fb6c62ffdc0ac56a73d52146100c953 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 18 Oct 2017 15:45:03 +0200 Subject: [PATCH 127/181] brandings: fix directory structure --- .../escm/templates/default/docker-compose-escm.yml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb index 80e0dab6b3..6ae732d0a3 100644 --- a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb +++ b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb @@ -96,7 +96,7 @@ services: syslog-facility: "local4" env_file: var.env volumes: - - /docker/config/brandings:/import:ro + - /docker/config/brandings:/import/brandings:ro - /docker/config/oscm-branding/privkey:/import/privkey - /docker/config/certs:/import/certs ports: From 02696330b770279297a61a0594b03819ed83c086 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Wed, 18 Oct 2017 16:09:50 +0200 Subject: [PATCH 128/181] Add synchronization --- .../escm/files/default/application.yaml | 101 ++++++++---------- chef/cookbooks/escm/recipes/server.rb | 1 - 2 files changed, 42 insertions(+), 60 deletions(-) diff --git a/chef/cookbooks/escm/files/default/application.yaml b/chef/cookbooks/escm/files/default/application.yaml index e3ff99cff5..05a68fdb99 100644 --- a/chef/cookbooks/escm/files/default/application.yaml +++ b/chef/cookbooks/escm/files/default/application.yaml @@ -38,20 +38,38 @@ parameters: data_volume_id: type: string description: The Cinder volume for data - registry_port: - default: "" - type: string - description: Docker registry port - mail_port: - default: 25 - type: string ssh_cert: default: "" type: string description: SSH key to add to servers' /root/.ssh/authorized_keys + ports: + type: comma_delimited_list + label: ports + default: "22,80,8080,8081,8880,8881,8680,8681" + protocols: + type: comma_delimited_list + label: protocols + default: "tcp,udp,icmp" + ethertypes: + type: comma_delimited_list + label: ethertypes + default: "IPv4,IPv6" + wait_condition_timeout: + default: 1800 + type: number resources: + wait_handle: + type: OS::Heat::WaitConditionHandle + + wait_condition: + type: OS::Heat::WaitCondition + depends_on: appserver + properties: + handle: { get_resource: wait_handle } + timeout: { get_param: wait_condition_timeout } + db_password: type: OS::Heat::RandomString @@ -99,58 +117,22 @@ resources: description: "Allow inbound SSH and HTTP traffic" name: escm rules: - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 22 - port_range_max: 22 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 80 - port_range_max: 80 - - remote_ip_prefix: 0.0.0.0/0 - protocol: icmp - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8080 - port_range_max: 8080 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8880 - port_range_max: 8880 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8081 - port_range_max: 8081 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8881 - port_range_max: 8881 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8680 - port_range_max: 8680 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: 8681 - port_range_max: 8681 - - direction: egress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: { get_param: registry_port } - port_range_max: { get_param: registry_port } - - direction: egress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - port_range_min: { get_param: mail_port } - port_range_max: { get_param: mail_port } + repeat: + for_each: + <%port%>: { get_param: ports } + template: + direction: ingress + protocol: tcp + port_range_min: <%port%> + port_range_max: <%port%> + repeat: + for_each: + <%protocol%>: { get_param: protocols } + <%ethertype%>: { get_param: ethertype } + template: + direction: egress + protocol: <%protocol%> + ethertype: <%ethertype%> # Parameters that will be available to both user data scripts user_data_params: @@ -172,6 +154,7 @@ resources: - - "/dev/disk/by-id/virtio-" - { get_param: data_volume_id } $SSH_CERT: { get_param: ssh_cert } + $WAIT_CURL: { get_attr: [ wait_handle, curl_cli ] } # User data payload for appserver userdata_appserver: diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 3b030d6ed5..54b1342598 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -252,7 +252,6 @@ --parameter data_volume_id=#{node[:escm][:openstack][:volume_stack][:data_volume_id]} \ --parameter image=#{escm_image} --parameter flavor=#{escm_flavor_name} \ --parameter key_name=#{escm_keypair_name} --parameter floating_network=#{escm_floating_network} \ - --parameter mail_port=#{node[:escm][:mail][:port]} --parameter registry_port=#{node[:escm][:docker][:port]} \ --parameter-file ssh_cert=#{escm_keypair_crowbar_sshkey}.pub \ -t #{escm_install_path}/application.yaml --wait #{escm_instancestack_name}" } not_if "#{openstack_cmd} #{openstack_args_heat} stack list -c 'Stack Name' -f value | egrep -q '^#{escm_instancestack_name}$'" From d1c056ce2ad6700f6245c01166db36a3b3d65064 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 18 Oct 2017 16:44:04 +0200 Subject: [PATCH 129/181] fix variable name in heat-config --- chef/cookbooks/escm/files/default/user-data/heat-config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index 88e74310fb..2955c833db 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -59,5 +59,5 @@ if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then fi # Signal to OpenStack that we're finished and ready -WAIT_CURL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" -eval $WAIT_CURL +WAIT_CURL_FULL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" +eval ${WAIT_CURL_FULL} From 3ace36262df7b8f82bf280822437b104ffd94eb8 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Wed, 18 Oct 2017 16:45:13 +0200 Subject: [PATCH 130/181] Correct paramter --- chef/cookbooks/escm/files/default/application.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/application.yaml b/chef/cookbooks/escm/files/default/application.yaml index 05a68fdb99..777b3c345c 100644 --- a/chef/cookbooks/escm/files/default/application.yaml +++ b/chef/cookbooks/escm/files/default/application.yaml @@ -123,12 +123,13 @@ resources: template: direction: ingress protocol: tcp + ethertype: IPv4 port_range_min: <%port%> port_range_max: <%port%> repeat: for_each: <%protocol%>: { get_param: protocols } - <%ethertype%>: { get_param: ethertype } + <%ethertype%>: { get_param: ethertypes } template: direction: egress protocol: <%protocol%> From 1634a1e0707acf83c29797775ffa0c219ce4ebd5 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 19 Oct 2017 10:24:55 +0200 Subject: [PATCH 131/181] add missing variable in docker compose environment file --- chef/cookbooks/escm/templates/default/var.env.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 41e5519a69..6a762ca0dd 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -12,6 +12,8 @@ DB_PORT_CORE=5432 DB_PORT_JMS=5432 DB_PORT_APP=5432 +REPORT_ENGINEURL=https://<%= @host_fqdn %>:8681/birt/frameset?__report=${reportname}.rptdesign&SessionId=${sessionid}&__locale=${locale}&WSDLURL=${wsdlurl}&SOAPEndPoint=${soapendpoint}&wsname=Report&wsport=ReportPort + DB_PWD_CORE=<%= @instance["db_core_password"] %> DB_PWD_APP=<%= @instance["db_app_password"] %> DB_SUPERPWD=<%= @instance["db_password"] %> From 4c344d5529a15ff9c5f5200ce94c3257889c29bf Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Thu, 19 Oct 2017 13:18:48 +0200 Subject: [PATCH 132/181] Remove repeat --- .../escm/files/default/application.yaml | 95 +++++++++++++------ 1 file changed, 66 insertions(+), 29 deletions(-) diff --git a/chef/cookbooks/escm/files/default/application.yaml b/chef/cookbooks/escm/files/default/application.yaml index 777b3c345c..1803f90ffa 100644 --- a/chef/cookbooks/escm/files/default/application.yaml +++ b/chef/cookbooks/escm/files/default/application.yaml @@ -42,18 +42,6 @@ parameters: default: "" type: string description: SSH key to add to servers' /root/.ssh/authorized_keys - ports: - type: comma_delimited_list - label: ports - default: "22,80,8080,8081,8880,8881,8680,8681" - protocols: - type: comma_delimited_list - label: protocols - default: "tcp,udp,icmp" - ethertypes: - type: comma_delimited_list - label: ethertypes - default: "IPv4,IPv6" wait_condition_timeout: default: 1800 type: number @@ -117,23 +105,72 @@ resources: description: "Allow inbound SSH and HTTP traffic" name: escm rules: - repeat: - for_each: - <%port%>: { get_param: ports } - template: - direction: ingress - protocol: tcp - ethertype: IPv4 - port_range_min: <%port%> - port_range_max: <%port%> - repeat: - for_each: - <%protocol%>: { get_param: protocols } - <%ethertype%>: { get_param: ethertypes } - template: - direction: egress - protocol: <%protocol%> - ethertype: <%ethertype%> + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 22 + port_range_max: 22 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 80 + port_range_max: 80 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 8080 + port_range_max: 8080 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 8880 + port_range_max: 8880 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 8081 + port_range_max: 8081 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 8881 + port_range_max: 8881 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 8680 + port_range_max: 8680 + - direction: ingress + remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + ethertype: IPv4 + port_range_min: 8681 + port_range_max: 8681 + - direction: egress + protocol: tcp + ethertype: IPv4 + - direction: egress + protocol: tcp + ethertype: IPv6 + - direction: egress + protocol: udp + ethertype: IPv4 + - direction: egress + protocol: udp + ethertype: IPv6 + - direction: egress + protocol: icmp + ethertype: IPv4 + - direction: egress + protocol: icmp + ethertype: IPv6 # Parameters that will be available to both user data scripts user_data_params: From 79e567682c5205cbb6e0ad1f10a290a491402d64 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Thu, 19 Oct 2017 16:42:52 +0200 Subject: [PATCH 133/181] One variables file --- .../files/default/user-data/deploy-escmserver | 3 +-- chef/cookbooks/escm/recipes/server.rb | 18 ------------- .../escm/templates/default/var.env.erb | 26 +++++++++++++++++++ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 55c127bb29..0d0330b9b7 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -8,7 +8,6 @@ exec &> /var/log/setup-machine.log # Variables for this script CONFIG_BASE=/etc/escm -CONFIG_PATH=${CONFIG_BASE}/config SSL_CONFIG_PATH=${CONFIG_BASE}/ssl COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose DOCKER_PATH=/docker @@ -16,7 +15,7 @@ DOCKER_PATH=/docker # Enable automatic exporting of variables set -a # Read configuration files -source ${CONFIG_PATH}/escm-config +source ${COMPOSE_CONFIG_PATH}/var.env # Disable automatic exporting of variables set +a diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 54b1342598..ab9d968164 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -297,20 +297,6 @@ action :create end -template "#{escm_install_path}/user-data/escm-config" do - source "escm.conf.erb" - owner escm_group - group escm_group - mode 0640 - variables( - mail: node[:escm][:mail], - docker: node[:escm][:docker], - proxy: node[:escm][:proxy], - host_fqdn: node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn], - instance: node[:escm][:openstack][:instance_stack] - ) -end - template "#{escm_install_path}/docker-compose-initdb.yml" do source "docker-compose-initdb.yml.erb" owner escm_group @@ -350,12 +336,8 @@ args = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i #{escm_keypair_crowbar_sshkey}" ip_appserver = node[:escm][:openstack][:instance_stack][:ip_appserver] Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) - command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/config'" - command_out = shell_out(command) command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/docker-compose'" command_out = shell_out(command) - command = "scp #{args} #{escm_install_path}/user-data/escm-config #{ip_appserver}:#{escm_path}/config" - command_out = shell_out(command) command = "scp #{args} #{escm_install_path}/docker-compose-*.yml #{ip_appserver}:#{escm_path}/docker-compose" command_out = shell_out(command) command = "scp #{args} #{escm_install_path}/var.env #{ip_appserver}:#{escm_path}/docker-compose" diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 6a762ca0dd..0ad3f4764a 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -1,3 +1,29 @@ +#proxy +PROXY_ENABLED=<%= @proxy["use_proxy"] %> +PROXY_HTTP_HOST=<%= @proxy["http_host"] %> +PROXY_HTTP_PORT=<%= @proxy["http_port"] %> +PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> +PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> +#<% if @proxy["no_proxy"].empty? %> +#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> +#<% else %> +#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> +#<% end %> +PROXY_NOPROXY=<%= @proxy["no_proxy"] %> +PROXY_AUTH=<%= @proxy["auth"] %> +PROXY_USER=<%= @proxy["user"] %> +PROXY_PWD=<%= @proxy["password"] %> + +#docker +DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> +DOCKER_REGISTRY_HOST=<%= @docker["host"] %> +DOCKER_REGISTRY_PORT=<%= @docker["port"] %> +DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> +DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> +DOCKER_REGISTRY_USER=<%= @docker["user"] %> +DOCKER_REGISTRY_PWD=<%= @docker["password"] %> + +#mail SMTP_HOST=<%= @mail["host"] %> SMTP_PORT=<%= @mail["port"] %> SMTP_TLS=<%= @mail["tls"] %> From 80a7e2af6a7569bfc8cf2f6f2dd47aa277fbc1a8 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 19 Oct 2017 18:35:21 +0200 Subject: [PATCH 134/181] change locations of ssl files --- .../files/default/user-data/deploy-escmserver | 22 +++++++++++++------ .../default/docker-compose-escm.yml.erb | 16 ++++++++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 0d0330b9b7..80d7120dde 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -32,10 +32,18 @@ for docker_directory in \ ${DOCKER_PATH}/data/oscm-db/data \ ${DOCKER_PATH}/config/certs \ ${DOCKER_PATH}/config/oscm-branding/brandings \ - ${DOCKER_PATH}/config/oscm-core/privkey \ - ${DOCKER_PATH}/config/oscm-app/privkey \ - ${DOCKER_PATH}/config/oscm-birt/privkey \ - ${DOCKER_PATH}/config/oscm-branding/privkey \ + ${DOCKER_PATH}/config/oscm-core/ssl/privkey \ + ${DOCKER_PATH}/config/oscm-core/ssl/cert \ + ${DOCKER_PATH}/config/oscm-core/ssl/chain \ + ${DOCKER_PATH}/config/oscm-app/ssl/privkey \ + ${DOCKER_PATH}/config/oscm-app/ssl/cert \ + ${DOCKER_PATH}/config/oscm-app/ssl/chain \ + ${DOCKER_PATH}/config/oscm-birt/ssl/privkey \ + ${DOCKER_PATH}/config/oscm-birt/ssl/cert \ + ${DOCKER_PATH}/config/oscm-birt/ssl/chain \ + ${DOCKER_PATH}/config/oscm-branding/ssl/privkey \ + ${DOCKER_PATH}/config/oscm-branding/ssl/cert \ + ${DOCKER_PATH}/config/oscm-branding/ssl/chain \ ${DOCKER_PATH}/logs/oscm-app \ ${DOCKER_PATH}/logs/oscm-birt \ ${DOCKER_PATH}/logs/oscm-branding \ @@ -85,13 +93,13 @@ fi # Copy certificate files if [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then - echo ${DOCKER_PATH}/config/oscm-core/privkey ${DOCKER_PATH}/config/oscm-app/privkey ${DOCKER_PATH}/config/oscm-birt/privkey ${DOCKER_PATH}/config/oscm-branding/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key + echo ${DOCKER_PATH}/config/oscm-core/ssl/privkey ${DOCKER_PATH}/config/oscm-app/ssl/privkey ${DOCKER_PATH}/config/oscm-birt/ssl/privkey ${DOCKER_PATH}/config/oscm-branding/ssl/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key fi if [ -f ${SSL_CONFIG_PATH}/oscm.crt ]; then - echo ${DOCKER_PATH}/config/oscm-core/privkey ${DOCKER_PATH}/config/oscm-app/privkey ${DOCKER_PATH}/config/oscm-birt/privkey ${DOCKER_PATH}/config/oscm-branding/privkey ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt + echo ${DOCKER_PATH}/config/oscm-core/ssl/cert ${DOCKER_PATH}/config/oscm-app/ssl/cert ${DOCKER_PATH}/config/oscm-birt/ssl/cert ${DOCKER_PATH}/config/oscm-branding/ssl/cert ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt fi if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then - echo ${DOCKER_PATH}/config/oscm-core/privkey ${DOCKER_PATH}/config/oscm-app/privkey ${DOCKER_PATH}/config/oscm-birt/privkey ${DOCKER_PATH}/config/oscm-branding/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain + echo ${DOCKER_PATH}/config/oscm-core/ssl/chain ${DOCKER_PATH}/config/oscm-app/ssl/chain ${DOCKER_PATH}/config/oscm-birt/ssl/chain ${DOCKER_PATH}/config/oscm-branding/ssl/chain | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain fi # If http proxy is enabled, create Docker config file diff --git a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb index 6ae732d0a3..e97c6b1b52 100644 --- a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb +++ b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb @@ -34,7 +34,9 @@ services: links: - oscm-db:oscm-db volumes: - - /docker/config/oscm-core/privkey:/import/privkey + - /docker/config/oscm-core/ssl/privkey:/import/privkey + - /docker/config/oscm-core/ssl/cert:/import/privkey + - /docker/config/oscm-core/ssl/chain:/import/privkey - /docker/config/certs:/import/certs ports: - 8080:8080 @@ -58,7 +60,9 @@ services: links: - oscm-db:oscm-db volumes: - - /docker/config/oscm-app/privkey:/import/privkey + - /docker/config/oscm-app/ssl/privkey:/import/privkey + - /docker/config/oscm-app/ssl/cert:/import/privkey + - /docker/config/oscm-app/ssl/chain:/import/privkey - /docker/config/certs:/import/certs ports: - 8880:8880 @@ -77,7 +81,9 @@ services: syslog-facility: "local3" env_file: var.env volumes: - - /docker/config/oscm-birt/privkey:/import/privkey + - /docker/config/oscm-birt/ssl/privkey:/import/privkey + - /docker/config/oscm-birt/ssl/cert:/import/privkey + - /docker/config/oscm-birt/ssl/chain:/import/privkey - /docker/config/certs:/import/certs ports: - 8680:8680 @@ -97,7 +103,9 @@ services: env_file: var.env volumes: - /docker/config/brandings:/import/brandings:ro - - /docker/config/oscm-branding/privkey:/import/privkey + - /docker/config/oscm-branding/ssl/privkey:/import/privkey + - /docker/config/oscm-branding/ssl/cert:/import/privkey + - /docker/config/oscm-branding/ssl/chain:/import/privkey - /docker/config/certs:/import/certs ports: - 4443:443 From 50fd7200336ca1c9405ce5fcdc60ee92799a9228 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 20 Oct 2017 09:05:23 +0200 Subject: [PATCH 135/181] Check if update necessary --- .../escm/files/default/user-data/deploy-escmserver | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 80d7120dde..242616b5d6 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -12,6 +12,13 @@ SSL_CONFIG_PATH=${CONFIG_BASE}/ssl COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose DOCKER_PATH=/docker +# Check if update is necessary +if [ -f ${DOCKER_PATH}/var.env.current ]; then + if [ $(diff ${DOCKER_PATH}/var.env.current ${COMPOSE_CONFIG_PATH}/var.env | wc -l) == "0" ]; then + exit + fi +fi + # Enable automatic exporting of variables set -a # Read configuration files @@ -182,3 +189,7 @@ docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f # Start application containers docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml up -d + +# Backup the current configuration for update check by the next run +cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.current + From a41b5ff05b1377fa3bac0942a7a68f5bd2b39d40 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 20 Oct 2017 10:28:09 +0200 Subject: [PATCH 136/181] Rename cert files --- .../escm/files/default/user-data/deploy-escmserver | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 242616b5d6..130c3e587a 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -99,14 +99,14 @@ if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then fi # Copy certificate files -if [ -f ${SSL_CONFIG_PATH}/oscm.key ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/privkey ${DOCKER_PATH}/config/oscm-app/ssl/privkey ${DOCKER_PATH}/config/oscm-birt/ssl/privkey ${DOCKER_PATH}/config/oscm-branding/ssl/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.key +if [ -f ${SSL_CONFIG_PATH}/escm.key ]; then + echo ${DOCKER_PATH}/config/oscm-core/ssl/privkey ${DOCKER_PATH}/config/oscm-app/ssl/privkey ${DOCKER_PATH}/config/oscm-birt/ssl/privkey ${DOCKER_PATH}/config/oscm-branding/ssl/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.key fi -if [ -f ${SSL_CONFIG_PATH}/oscm.crt ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/cert ${DOCKER_PATH}/config/oscm-app/ssl/cert ${DOCKER_PATH}/config/oscm-birt/ssl/cert ${DOCKER_PATH}/config/oscm-branding/ssl/cert ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.crt +if [ -f ${SSL_CONFIG_PATH}/escm.crt ]; then + echo ${DOCKER_PATH}/config/oscm-core/ssl/cert ${DOCKER_PATH}/config/oscm-app/ssl/cert ${DOCKER_PATH}/config/oscm-birt/ssl/cert ${DOCKER_PATH}/config/oscm-branding/ssl/cert ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.crt fi -if [ -f ${SSL_CONFIG_PATH}/oscm.chain ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/chain ${DOCKER_PATH}/config/oscm-app/ssl/chain ${DOCKER_PATH}/config/oscm-birt/ssl/chain ${DOCKER_PATH}/config/oscm-branding/ssl/chain | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/oscm.chain +if [ -f ${SSL_CONFIG_PATH}/escm.chain ]; then + echo ${DOCKER_PATH}/config/oscm-core/ssl/chain ${DOCKER_PATH}/config/oscm-app/ssl/chain ${DOCKER_PATH}/config/oscm-birt/ssl/chain ${DOCKER_PATH}/config/oscm-branding/ssl/chain | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.chain fi # If http proxy is enabled, create Docker config file From bb8a5d071b5ad3609ca6722096c9a8c9d7590a15 Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 20 Oct 2017 10:45:55 +0200 Subject: [PATCH 137/181] Include docker tag in variables template --- chef/cookbooks/escm/templates/default/var.env.erb | 1 + chef/data_bags/crowbar/template-escm.json | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 0ad3f4764a..f1416001ac 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -22,6 +22,7 @@ DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> DOCKER_REGISTRY_USER=<%= @docker["user"] %> DOCKER_REGISTRY_PWD=<%= @docker["password"] %> +DOCKER_TAG=<%= @docker["tag"] %> #mail SMTP_HOST=<%= @mail["host"] %> diff --git a/chef/data_bags/crowbar/template-escm.json b/chef/data_bags/crowbar/template-escm.json index 99936a7aaf..d5731906a7 100644 --- a/chef/data_bags/crowbar/template-escm.json +++ b/chef/data_bags/crowbar/template-escm.json @@ -57,9 +57,9 @@ "proxy": { "use_proxy": false, "http_host": "", - "http_port": 0, + "http_port": 80, "https_host": "", - "https_port": 0, + "https_port": 80, "no_proxy": "", "auth": false, "user": "", From bf6c319037effd6550634e73891631afc901f38b Mon Sep 17 00:00:00 2001 From: StavrevaS Date: Fri, 20 Oct 2017 11:00:46 +0200 Subject: [PATCH 138/181] Adapt container ssl paths --- .../default/docker-compose-escm.yml.erb | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb index e97c6b1b52..4395102015 100644 --- a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb +++ b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb @@ -34,9 +34,9 @@ services: links: - oscm-db:oscm-db volumes: - - /docker/config/oscm-core/ssl/privkey:/import/privkey - - /docker/config/oscm-core/ssl/cert:/import/privkey - - /docker/config/oscm-core/ssl/chain:/import/privkey + - /docker/config/oscm-core/ssl/privkey:/import/ssl/privkey + - /docker/config/oscm-core/ssl/cert:/import/ssl/cert + - /docker/config/oscm-core/ssl/chain:/import/ssl/chain - /docker/config/certs:/import/certs ports: - 8080:8080 @@ -60,9 +60,9 @@ services: links: - oscm-db:oscm-db volumes: - - /docker/config/oscm-app/ssl/privkey:/import/privkey - - /docker/config/oscm-app/ssl/cert:/import/privkey - - /docker/config/oscm-app/ssl/chain:/import/privkey + - /docker/config/oscm-app/ssl/privkey:/import/ssl/privkey + - /docker/config/oscm-app/ssl/cert:/import/ssl/cert + - /docker/config/oscm-app/ssl/chain:/import/ssl/chain - /docker/config/certs:/import/certs ports: - 8880:8880 @@ -81,9 +81,9 @@ services: syslog-facility: "local3" env_file: var.env volumes: - - /docker/config/oscm-birt/ssl/privkey:/import/privkey - - /docker/config/oscm-birt/ssl/cert:/import/privkey - - /docker/config/oscm-birt/ssl/chain:/import/privkey + - /docker/config/oscm-birt/ssl/privkey:/import/ssl/privkey + - /docker/config/oscm-birt/ssl/cert:/import/ssl/cert + - /docker/config/oscm-birt/ssl/chain:/import/ssl/chain - /docker/config/certs:/import/certs ports: - 8680:8680 @@ -103,9 +103,9 @@ services: env_file: var.env volumes: - /docker/config/brandings:/import/brandings:ro - - /docker/config/oscm-branding/ssl/privkey:/import/privkey - - /docker/config/oscm-branding/ssl/cert:/import/privkey - - /docker/config/oscm-branding/ssl/chain:/import/privkey + - /docker/config/oscm-branding/ssl/privkey:/import/ssl/privkey + - /docker/config/oscm-branding/ssl/cert:/import/ssl/cert + - /docker/config/oscm-branding/ssl/chain:/import/ssl/chain - /docker/config/certs:/import/certs ports: - 4443:443 From 4c9adb584df77af4b1efcbd293b025ae33f4ab71 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 20 Oct 2017 11:33:16 +0200 Subject: [PATCH 139/181] Fixes #17 make logging persistent and append --- .../escm/files/default/user-data/deploy-escmserver | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 130c3e587a..08dbed2b87 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -3,8 +3,6 @@ set -x # Exit on error set -e -# Redirect all output to log file -exec &> /var/log/setup-machine.log # Variables for this script CONFIG_BASE=/etc/escm @@ -12,6 +10,11 @@ SSL_CONFIG_PATH=${CONFIG_BASE}/ssl COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose DOCKER_PATH=/docker +# Redirect all output to log file +exec &>> ${DOCKER_PATH}/setup-machine-$(date '+%Y-%m-%d').log + +echo "$(date '+%Y-%m-%d %H:%M:%S') starting setup..." + # Check if update is necessary if [ -f ${DOCKER_PATH}/var.env.current ]; then if [ $(diff ${DOCKER_PATH}/var.env.current ${COMPOSE_CONFIG_PATH}/var.env | wc -l) == "0" ]; then @@ -193,3 +196,4 @@ docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml up -d # Backup the current configuration for update check by the next run cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.current +echo "$(date '+%Y-%m-%d %H:%M:%S') setup finished." From b012e4cb48676a77e512658145829878630e9bf3 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 23 Oct 2017 18:04:03 +0200 Subject: [PATCH 140/181] refactor deploy script to use deployer container --- .../files/default/user-data/deploy-escmserver | 186 +++++++++--------- 1 file changed, 97 insertions(+), 89 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 08dbed2b87..cdb03fe475 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -37,80 +37,45 @@ if [ -f ${DOCKER_PATH}/docker-compose-escm.yml ]; then fi fi -# Create Docker directories if they do not exist yet -for docker_directory in \ - ${DOCKER_PATH}/data/oscm-db/data \ - ${DOCKER_PATH}/config/certs \ - ${DOCKER_PATH}/config/oscm-branding/brandings \ - ${DOCKER_PATH}/config/oscm-core/ssl/privkey \ - ${DOCKER_PATH}/config/oscm-core/ssl/cert \ - ${DOCKER_PATH}/config/oscm-core/ssl/chain \ - ${DOCKER_PATH}/config/oscm-app/ssl/privkey \ - ${DOCKER_PATH}/config/oscm-app/ssl/cert \ - ${DOCKER_PATH}/config/oscm-app/ssl/chain \ - ${DOCKER_PATH}/config/oscm-birt/ssl/privkey \ - ${DOCKER_PATH}/config/oscm-birt/ssl/cert \ - ${DOCKER_PATH}/config/oscm-birt/ssl/chain \ - ${DOCKER_PATH}/config/oscm-branding/ssl/privkey \ - ${DOCKER_PATH}/config/oscm-branding/ssl/cert \ - ${DOCKER_PATH}/config/oscm-branding/ssl/chain \ - ${DOCKER_PATH}/logs/oscm-app \ - ${DOCKER_PATH}/logs/oscm-birt \ - ${DOCKER_PATH}/logs/oscm-branding \ - ${DOCKER_PATH}/logs/oscm-core \ - ${DOCKER_PATH}/logs/oscm-db; do - if [ ! -d ${docker_directory} ]; then - mkdir -p ${docker_directory} - fi -done - -# Create Docker log files if they do not exist yet -for docker_log_file in \ - ${DOCKER_PATH}/logs/oscm-app/oscm-app.out.log \ - ${DOCKER_PATH}/logs/oscm-birt/oscm-birt.out.log \ - ${DOCKER_PATH}/logs/oscm-branding/oscm-branding.out.log \ - ${DOCKER_PATH}/logs/oscm-core/oscm-core.out.log \ - ${DOCKER_PATH}/logs/oscm-db/oscm-db.out.log; do - if [ ! -f {docker_log_file} ]; then - touch ${docker_log_file} - chmod 640 ${docker_log_file} - fi -done - -# Create rsyslog configuration for Docker log files -if [ ! -f /etc/rsyslog.d/oscm.conf ]; then - cat < /etc/rsyslog.d/oscm.conf -local0.* /docker/logs/oscm-db/oscm-db.out.log -local1.* /docker/logs/oscm-core/oscm-core.out.log -local2.* /docker/logs/oscm-app/oscm-app.out.log -local3.* /docker/logs/oscm-birt/oscm-birt.out.log -local4.* /docker/logs/oscm-branding/oscm-branding.out.log -EOF - sed -i '/^local/d' /etc/rsyslog.conf - systemctl restart rsyslog -fi - -# Copy Docker Compose files -if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then - cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml -fi -if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ]; then - cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ${DOCKER_PATH}/docker-compose-escm.yml -fi -if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then - cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env -fi - -# Copy certificate files -if [ -f ${SSL_CONFIG_PATH}/escm.key ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/privkey ${DOCKER_PATH}/config/oscm-app/ssl/privkey ${DOCKER_PATH}/config/oscm-birt/ssl/privkey ${DOCKER_PATH}/config/oscm-branding/ssl/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.key -fi -if [ -f ${SSL_CONFIG_PATH}/escm.crt ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/cert ${DOCKER_PATH}/config/oscm-app/ssl/cert ${DOCKER_PATH}/config/oscm-birt/ssl/cert ${DOCKER_PATH}/config/oscm-branding/ssl/cert ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.crt -fi -if [ -f ${SSL_CONFIG_PATH}/escm.chain ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/chain ${DOCKER_PATH}/config/oscm-app/ssl/chain ${DOCKER_PATH}/config/oscm-birt/ssl/chain ${DOCKER_PATH}/config/oscm-branding/ssl/chain | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.chain -fi +# # Create Docker directories if they do not exist yet +# for docker_directory in \ +# ${DOCKER_PATH}/data/oscm-db/data \ +# ${DOCKER_PATH}/config/certs \ +# ${DOCKER_PATH}/config/oscm-branding/brandings \ +# ${DOCKER_PATH}/config/oscm-core/ssl/privkey \ +# ${DOCKER_PATH}/config/oscm-core/ssl/cert \ +# ${DOCKER_PATH}/config/oscm-core/ssl/chain \ +# ${DOCKER_PATH}/config/oscm-app/ssl/privkey \ +# ${DOCKER_PATH}/config/oscm-app/ssl/cert \ +# ${DOCKER_PATH}/config/oscm-app/ssl/chain \ +# ${DOCKER_PATH}/config/oscm-birt/ssl/privkey \ +# ${DOCKER_PATH}/config/oscm-birt/ssl/cert \ +# ${DOCKER_PATH}/config/oscm-birt/ssl/chain \ +# ${DOCKER_PATH}/config/oscm-branding/ssl/privkey \ +# ${DOCKER_PATH}/config/oscm-branding/ssl/cert \ +# ${DOCKER_PATH}/config/oscm-branding/ssl/chain \ +# ${DOCKER_PATH}/logs/oscm-app \ +# ${DOCKER_PATH}/logs/oscm-birt \ +# ${DOCKER_PATH}/logs/oscm-branding \ +# ${DOCKER_PATH}/logs/oscm-core \ +# ${DOCKER_PATH}/logs/oscm-db; do +# if [ ! -d ${docker_directory} ]; then +# mkdir -p ${docker_directory} +# fi +# done + +# # Create Docker log files if they do not exist yet +# for docker_log_file in \ +# ${DOCKER_PATH}/logs/oscm-app/oscm-app.out.log \ +# ${DOCKER_PATH}/logs/oscm-birt/oscm-birt.out.log \ +# ${DOCKER_PATH}/logs/oscm-branding/oscm-branding.out.log \ +# ${DOCKER_PATH}/logs/oscm-core/oscm-core.out.log \ +# ${DOCKER_PATH}/logs/oscm-db/oscm-db.out.log; do +# if [ ! -f {docker_log_file} ]; then +# touch ${docker_log_file} +# chmod 640 ${docker_log_file} +# fi +# done # If http proxy is enabled, create Docker config file if [ ${PROXY_ENABLED} == "true" ]; then @@ -177,21 +142,64 @@ EOF systemctl restart docker fi -# Pull images -egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull -egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-escm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull - -# Initialize databases -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up -d oscm-db -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-core -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-jms -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-app -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-controller-openstack -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml stop -docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f - -# Start application containers -docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml up -d +# Create rsyslog configuration for Docker log files +if [ ! -f /etc/rsyslog.d/oscm.conf ]; then + cat < /etc/rsyslog.d/oscm.conf +local0.* /docker/logs/oscm-db/oscm-db.out.log +local1.* /docker/logs/oscm-core/oscm-core.out.log +local2.* /docker/logs/oscm-app/oscm-app.out.log +local3.* /docker/logs/oscm-birt/oscm-birt.out.log +local4.* /docker/logs/oscm-branding/oscm-branding.out.log +EOF + sed -i '/^local/d' /etc/rsyslog.conf + systemctl restart rsyslog +fi + +# # Copy Docker Compose files +# if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then +# cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml +# fi +# if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ]; then +# cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ${DOCKER_PATH}/docker-compose-escm.yml +# fi +# if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then +# cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env +# fi + +#TODO: Where does the deployer image come from? +# Create Docker directory structure and Docker Compose files +docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target oscm-deployer + +# Copy certificate files +if [ -f ${SSL_CONFIG_PATH}/escm.key ]; then + echo ${DOCKER_PATH}/config/oscm-core/ssl/privkey ${DOCKER_PATH}/config/oscm-app/ssl/privkey ${DOCKER_PATH}/config/oscm-birt/ssl/privkey ${DOCKER_PATH}/config/oscm-branding/ssl/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.key +fi +if [ -f ${SSL_CONFIG_PATH}/escm.crt ]; then + echo ${DOCKER_PATH}/config/oscm-core/ssl/cert ${DOCKER_PATH}/config/oscm-app/ssl/cert ${DOCKER_PATH}/config/oscm-birt/ssl/cert ${DOCKER_PATH}/config/oscm-branding/ssl/cert ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.crt +fi +if [ -f ${SSL_CONFIG_PATH}/escm.chain ]; then + echo ${DOCKER_PATH}/config/oscm-core/ssl/chain ${DOCKER_PATH}/config/oscm-app/ssl/chain ${DOCKER_PATH}/config/oscm-birt/ssl/chain ${DOCKER_PATH}/config/oscm-branding/ssl/chain | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.chain +fi + +#TODO: Where does the deployer image come from? +# Initialize the databases and start the application containers +docker run --name deployer-init-start --rm -v ${DOCKER_PATH}:/target -v /var/run/docker.sock:/var/run/docker.sock -e INITDB=true -e STARTUP=true oscm-deployer + +# # Pull images +# egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull +# egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-escm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull + +# # Initialize databases +# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up -d oscm-db +# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-core +# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-jms +# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-app +# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-controller-openstack +# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml stop +# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f + +# # Start application containers +# docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml up -d # Backup the current configuration for update check by the next run cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.current From 4548f8096b77f3c361a861c760314294218f8d37 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 25 Oct 2017 07:58:20 +0200 Subject: [PATCH 141/181] changed template files to work with deployer --- .../files/default/user-data/deploy-escmserver | 8 +++ chef/cookbooks/escm/recipes/server.rb | 49 ++++++++++++------- .../cookbooks/escm/templates/default/.env.erb | 43 ++++++++++++++++ .../escm/templates/default/var.env.erb | 11 +++-- 4 files changed, 89 insertions(+), 22 deletions(-) create mode 100644 chef/cookbooks/escm/templates/default/.env.erb diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index cdb03fe475..662bbf8fa1 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -166,6 +166,14 @@ fi # cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env # fi +# Copy environment and configuration files to Docker directory +if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then + cp ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH} +fi +if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then + cp ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH} +fi + #TODO: Where does the deployer image come from? # Create Docker directory structure and Docker Compose files docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target oscm-deployer diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index ab9d968164..81a10f081a 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -297,28 +297,42 @@ action :create end -template "#{escm_install_path}/docker-compose-initdb.yml" do - source "docker-compose-initdb.yml.erb" - owner escm_group - group escm_group - mode 0640 - variables( - docker: node[:escm][:docker] - ) -end +# template "#{escm_install_path}/docker-compose-initdb.yml" do +# source "docker-compose-initdb.yml.erb" +# owner escm_group +# group escm_group +# mode 0640 +# variables( +# docker: node[:escm][:docker] +# ) +# end + +# template "#{escm_install_path}/docker-compose-escm.yml" do +# source "docker-compose-escm.yml.erb" +# owner escm_group +# group escm_group +# mode 0640 +# variables( +# docker: node[:escm][:docker] +# ) +# end -template "#{escm_install_path}/docker-compose-escm.yml" do - source "docker-compose-escm.yml.erb" +template "#{escm_install_path}/var.env" do + source "var.env.erb" owner escm_group group escm_group mode 0640 variables( - docker: node[:escm][:docker] + mail: node[:escm][:mail], + docker: node[:escm][:docker], + proxy: node[:escm][:proxy], + host_fqdn: node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn], + instance: node[:escm][:openstack][:instance_stack] ) end -template "#{escm_install_path}/var.env" do - source "var.env.erb" +template "#{escm_install_path}/.env" do + source ".env.erb" owner escm_group group escm_group mode 0640 @@ -338,10 +352,12 @@ Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/docker-compose'" command_out = shell_out(command) - command = "scp #{args} #{escm_install_path}/docker-compose-*.yml #{ip_appserver}:#{escm_path}/docker-compose" - command_out = shell_out(command) +# command = "scp #{args} #{escm_install_path}/docker-compose-*.yml #{ip_appserver}:#{escm_path}/docker-compose" +# command_out = shell_out(command) command = "scp #{args} #{escm_install_path}/var.env #{ip_appserver}:#{escm_path}/docker-compose" command_out = shell_out(command) + command = "scp #{args} #{escm_install_path}/.env #{ip_appserver}:#{escm_path}/docker-compose" + command_out = shell_out(command) if node[:escm][:api][:protocol] == "https" command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/ssl'" command_out = shell_out(command) @@ -360,4 +376,3 @@ end action :create end - diff --git a/chef/cookbooks/escm/templates/default/.env.erb b/chef/cookbooks/escm/templates/default/.env.erb new file mode 100644 index 0000000000..32b6edced6 --- /dev/null +++ b/chef/cookbooks/escm/templates/default/.env.erb @@ -0,0 +1,43 @@ +<% if @docker["dockerhub"] == true %> +IMAGE_DB=<%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> +<% else %> +IMAGE_DB=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> +<% end %> +IMAGE_CORE=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-core:soc +<% if @docker["dockerhub"] == true %> +IMAGE_CORE=<%= @docker["organization"]+"/oscm-core:"+@docker["tag"] %> +<% else %> +IMAGE_CORE=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> +<% end %> +IMAGE_APP=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-app:soc +<% if @docker["dockerhub"] == true %> +IMAGE_APP=<%= @docker["organization"]+"/oscm-app:"+@docker["tag"] %> +<% else %> +IMAGE_APP=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> +<% end %> +IMAGE_BIRT=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-birt:soc +<% if @docker["dockerhub"] == true %> +IMAGE_BIRT=<%= @docker["organization"]+"/oscm-birt:"+@docker["tag"] %> +<% else %> +IMAGE_BIRT=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> +<% end %> +IMAGE_BRANDING=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-branding:soc +<% if @docker["dockerhub"] == true %> +IMAGE_BRANDING=<%= @docker["organization"]+"/oscm-branding:"+@docker["tag"] %> +<% else %> +IMAGE_BRANDING=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> +<% end %> +IMAGE_INITDB=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-initdb:soc +<% if @docker["dockerhub"] == true %> +IMAGE_INITDB=<%= @docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> +<% else %> +IMAGE_INITDB=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> +<% end %> +IMAGE_PROXY=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-proxy:soc +<% if @docker["dockerhub"] == true %> +IMAGE_PROXY=<%= @docker["organization"]+"/oscm-proxy:"+@docker["tag"] %> +<% else %> +IMAGE_PROXY=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-proxy:"+@docker["tag"] %> +<% end %> + +DOCKER_PATH=/docker diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index f1416001ac..a130777b21 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -27,24 +27,25 @@ DOCKER_TAG=<%= @docker["tag"] %> #mail SMTP_HOST=<%= @mail["host"] %> SMTP_PORT=<%= @mail["port"] %> -SMTP_TLS=<%= @mail["tls"] %> +SMTP_FROM=<%= @mail["from"] %> SMTP_USER=<%= @mail["user"] %> SMTP_PWD=<%= @mail["password"] %> -SMTP_FROM=<%= @mail["from"] %> SMTP_AUTH=<%= @mail["auth"] %> +SMTP_TLS=<%= @mail["tls"] %> +KEY_SECRET=<%= @instance["key_secret"] %> HOST_FQDN=<%= @host_fqdn %> +REPORT_ENGINEURL=https://<%= @host_fqdn %>:8681/birt/frameset?__report=${reportname}.rptdesign&SessionId=${sessionid}&__locale=${locale}&WSDLURL=${wsdlurl}&SOAPEndPoint=${soapendpoint}&wsname=Report&wsport=ReportPort + DB_PORT_CORE=5432 DB_PORT_JMS=5432 DB_PORT_APP=5432 -REPORT_ENGINEURL=https://<%= @host_fqdn %>:8681/birt/frameset?__report=${reportname}.rptdesign&SessionId=${sessionid}&__locale=${locale}&WSDLURL=${wsdlurl}&SOAPEndPoint=${soapendpoint}&wsname=Report&wsport=ReportPort - DB_PWD_CORE=<%= @instance["db_core_password"] %> DB_PWD_APP=<%= @instance["db_app_password"] %> + DB_SUPERPWD=<%= @instance["db_password"] %> -KEY_SECRET=<%= @instance["key_secret"] %> APP_ADMIN_MAIL_ADDRESS=<%= @mail["from"] %> From f7a1c32b6534f41b9531f8a6931f52b8bb9f9343 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 25 Oct 2017 08:59:01 +0200 Subject: [PATCH 142/181] change ssl cert generation to ruby block --- chef/cookbooks/escm/recipes/server.rb | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 81a10f081a..be56982483 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -267,15 +267,20 @@ action :create end -if node[:escm][:api][:protocol] == "https" - ssl_setup "setting up ssl for escm" do - generate_certs node[:escm][:ssl][:generate_certs] - certfile node[:escm][:ssl][:certfile] - keyfile node[:escm][:ssl][:keyfile] - group escm_group - fqdn node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] - ca_certs node[:escm][:ssl][:ca_certs] - end +ruby_block "set_up_ssl_for_escm" do + block do + if node[:escm][:api][:protocol] == "https" + ssl_setup "setting up ssl for escm" do + generate_certs node[:escm][:ssl][:generate_certs] + certfile node[:escm][:ssl][:certfile] + keyfile node[:escm][:ssl][:keyfile] + group escm_group + fqdn node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] + ca_certs node[:escm][:ssl][:ca_certs] + end + end + end + action :create end ruby_block "get_escm_secrets" do From a10b83402e84394a731a9c2414f98a068feeacad Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 25 Oct 2017 09:21:12 +0200 Subject: [PATCH 143/181] Revert "change ssl cert generation to ruby block" This reverts commit f7a1c32b6534f41b9531f8a6931f52b8bb9f9343. --- chef/cookbooks/escm/recipes/server.rb | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index be56982483..81a10f081a 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -267,20 +267,15 @@ action :create end -ruby_block "set_up_ssl_for_escm" do - block do - if node[:escm][:api][:protocol] == "https" - ssl_setup "setting up ssl for escm" do - generate_certs node[:escm][:ssl][:generate_certs] - certfile node[:escm][:ssl][:certfile] - keyfile node[:escm][:ssl][:keyfile] - group escm_group - fqdn node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] - ca_certs node[:escm][:ssl][:ca_certs] - end - end - end - action :create +if node[:escm][:api][:protocol] == "https" + ssl_setup "setting up ssl for escm" do + generate_certs node[:escm][:ssl][:generate_certs] + certfile node[:escm][:ssl][:certfile] + keyfile node[:escm][:ssl][:keyfile] + group escm_group + fqdn node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] + ca_certs node[:escm][:ssl][:ca_certs] + end end ruby_block "get_escm_secrets" do From 915faeb44c562beb540d79180ce1b9d041c945eb Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 25 Oct 2017 09:23:51 +0200 Subject: [PATCH 144/181] ssl generation with lazy evaluation --- chef/cookbooks/escm/recipes/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 81a10f081a..7141caa1c2 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -273,7 +273,7 @@ certfile node[:escm][:ssl][:certfile] keyfile node[:escm][:ssl][:keyfile] group escm_group - fqdn node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] + fqdn lazy { "node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn]" } ca_certs node[:escm][:ssl][:ca_certs] end end From 0f16ef73aeeb2065e321e4b6d72d2227a19a736c Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Wed, 25 Oct 2017 16:40:56 +0200 Subject: [PATCH 145/181] fix variables for .env template --- chef/cookbooks/escm/recipes/server.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 7141caa1c2..ab1ce20599 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -336,13 +336,9 @@ owner escm_group group escm_group mode 0640 - variables( - mail: node[:escm][:mail], - docker: node[:escm][:docker], - proxy: node[:escm][:proxy], - host_fqdn: node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn], - instance: node[:escm][:openstack][:instance_stack] - ) + variables( + docker: node[:escm][:docker] + ) end ruby_block "inject_escm_scripts" do From 249a1ceabff302496b016ae1b9cf64dd551694b3 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 30 Oct 2017 15:58:43 +0100 Subject: [PATCH 146/181] fix syntax --- chef/cookbooks/escm/recipes/server.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index ab1ce20599..8546fd0804 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -273,7 +273,7 @@ certfile node[:escm][:ssl][:certfile] keyfile node[:escm][:ssl][:keyfile] group escm_group - fqdn lazy { "node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn]" } + fqdn lazy { node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] } ca_certs node[:escm][:ssl][:ca_certs] end end @@ -326,7 +326,7 @@ mail: node[:escm][:mail], docker: node[:escm][:docker], proxy: node[:escm][:proxy], - host_fqdn: node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn], + host_fqdn: lazy { node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] }, instance: node[:escm][:openstack][:instance_stack] ) end From 69c28dc45decdf065578e24812369aa87e94153d Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 7 Nov 2017 13:33:57 +0100 Subject: [PATCH 147/181] fix nil variable --- chef/cookbooks/escm/recipes/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 8546fd0804..aca70f7952 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -327,7 +327,7 @@ docker: node[:escm][:docker], proxy: node[:escm][:proxy], host_fqdn: lazy { node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] }, - instance: node[:escm][:openstack][:instance_stack] + instance: lazy { node[:escm][:openstack][:instance_stack] } ) end From bcf7e7a3d550aa92cadc9272fe7e21279759d7a1 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 7 Nov 2017 13:34:56 +0100 Subject: [PATCH 148/181] improve setting no_proxy --- chef/cookbooks/escm/templates/default/var.env.erb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index a130777b21..1c24be15d2 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -4,12 +4,11 @@ PROXY_HTTP_HOST=<%= @proxy["http_host"] %> PROXY_HTTP_PORT=<%= @proxy["http_port"] %> PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> -#<% if @proxy["no_proxy"].empty? %> -#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> -#<% else %> -#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> -#<% end %> -PROXY_NOPROXY=<%= @proxy["no_proxy"] %> +<% if @proxy["no_proxy"].empty? %> +PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> +<% else %> +PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> +<% end %> PROXY_AUTH=<%= @proxy["auth"] %> PROXY_USER=<%= @proxy["user"] %> PROXY_PWD=<%= @proxy["password"] %> From 470016ec73869c1fca92d573ccd012f9c48c68a7 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 9 Nov 2017 14:53:02 +0100 Subject: [PATCH 149/181] fix setting no_proxy --- chef/cookbooks/escm/recipes/server.rb | 3 ++- chef/cookbooks/escm/templates/default/var.env.erb | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index aca70f7952..9356796776 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -327,7 +327,8 @@ docker: node[:escm][:docker], proxy: node[:escm][:proxy], host_fqdn: lazy { node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] }, - instance: lazy { node[:escm][:openstack][:instance_stack] } + instance: lazy { node[:escm][:openstack][:instance_stack] }, + no_proxy: lazy { node[:escm][:proxy][:no_proxy].empty? ? "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]}" : "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]},#{node[:escm][:proxy][:no_proxy]}" } ) end diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 1c24be15d2..2038359e4e 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -4,11 +4,7 @@ PROXY_HTTP_HOST=<%= @proxy["http_host"] %> PROXY_HTTP_PORT=<%= @proxy["http_port"] %> PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> -<% if @proxy["no_proxy"].empty? %> -PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> -<% else %> -PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> -<% end %> +PROXY_NOPROXY=<%= @no_proxy %> PROXY_AUTH=<%= @proxy["auth"] %> PROXY_USER=<%= @proxy["user"] %> PROXY_PWD=<%= @proxy["password"] %> From 9b8ffca2867ebd4e421e2b4a08c2d67d6f18d8b7 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 9 Nov 2017 17:33:48 +0100 Subject: [PATCH 150/181] use variables for mutable attributes --- chef/cookbooks/escm/recipes/server.rb | 16 +++++++++++++--- .../cookbooks/escm/templates/default/var.env.erb | 8 ++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 9356796776..0e0161591f 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -317,6 +317,13 @@ # ) # end +var_no_proxy = node[:escm][:proxy][:no_proxy].empty? ? "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]}" : "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]},#{node[:escm][:proxy][:no_proxy]}" +var_key_secret = "#{node[:escm][:openstack][:instance_stack][:key_secret]}" +var_host_fqdn = node[:escm][:ssl][:fqdn].empty? ? "#{node[:escm][:openstack][:instance_stack][:ip_appserver]}" : "#{node[:escm][:ssl][:fqdn]}" +var_db_pwd_core = "#{node[:escm][:openstack][:instance_stack][:db_core_password]}" +var_db_pwd_app = "#{node[:escm][:openstack][:instance_stack][:db_app_password]}" +var_db_superpwd = "#{node[:escm][:openstack][:instance_stack][:db_password]}" + template "#{escm_install_path}/var.env" do source "var.env.erb" owner escm_group @@ -326,9 +333,12 @@ mail: node[:escm][:mail], docker: node[:escm][:docker], proxy: node[:escm][:proxy], - host_fqdn: lazy { node[:escm][:ssl][:fqdn].empty? ? node[:escm][:openstack][:instance_stack][:ip_appserver] : node[:escm][:ssl][:fqdn] }, - instance: lazy { node[:escm][:openstack][:instance_stack] }, - no_proxy: lazy { node[:escm][:proxy][:no_proxy].empty? ? "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]}" : "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]},#{node[:escm][:proxy][:no_proxy]}" } + no_proxy: var_no_proxy, + key_secret: var_key_secret, + host_fqdn: var_host_fqdn, + db_pwd_core: var_db_pwd_core, + db_pwd_app: var_db_pwd_app, + db_superpwd: var_db_superpwd ) end diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 2038359e4e..850909299e 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -28,7 +28,7 @@ SMTP_PWD=<%= @mail["password"] %> SMTP_AUTH=<%= @mail["auth"] %> SMTP_TLS=<%= @mail["tls"] %> -KEY_SECRET=<%= @instance["key_secret"] %> +KEY_SECRET=<%= @key_secret %> HOST_FQDN=<%= @host_fqdn %> REPORT_ENGINEURL=https://<%= @host_fqdn %>:8681/birt/frameset?__report=${reportname}.rptdesign&SessionId=${sessionid}&__locale=${locale}&WSDLURL=${wsdlurl}&SOAPEndPoint=${soapendpoint}&wsname=Report&wsport=ReportPort @@ -37,10 +37,10 @@ DB_PORT_CORE=5432 DB_PORT_JMS=5432 DB_PORT_APP=5432 -DB_PWD_CORE=<%= @instance["db_core_password"] %> -DB_PWD_APP=<%= @instance["db_app_password"] %> +DB_PWD_CORE=<%= @db_pwd_core %> +DB_PWD_APP=<%= @db_pwd_app %> -DB_SUPERPWD=<%= @instance["db_password"] %> +DB_SUPERPWD=<%= @var_db_superpwd %> APP_ADMIN_MAIL_ADDRESS=<%= @mail["from"] %> From 70f57434f02e48fbf3708624170d016c112e70e9 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 9 Nov 2017 17:51:55 +0100 Subject: [PATCH 151/181] change to new var.env file structure --- .../escm/templates/default/var.env.erb | 115 ++++++++++++------ 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 850909299e..820d57d291 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -1,53 +1,98 @@ -#proxy -PROXY_ENABLED=<%= @proxy["use_proxy"] %> -PROXY_HTTP_HOST=<%= @proxy["http_host"] %> -PROXY_HTTP_PORT=<%= @proxy["http_port"] %> -PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> -PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> -PROXY_NOPROXY=<%= @no_proxy %> -PROXY_AUTH=<%= @proxy["auth"] %> -PROXY_USER=<%= @proxy["user"] %> -PROXY_PWD=<%= @proxy["password"] %> +# Version: v17.5.0 +# Please adjust _all_ of the ${PLACEHOLDERS} in order for the application +# to work correctly. -#docker -DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> -DOCKER_REGISTRY_HOST=<%= @docker["host"] %> -DOCKER_REGISTRY_PORT=<%= @docker["port"] %> -DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> -DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> -DOCKER_REGISTRY_USER=<%= @docker["user"] %> -DOCKER_REGISTRY_PWD=<%= @docker["password"] %> -DOCKER_TAG=<%= @docker["tag"] %> +##################### +# Database settings # +##################### +# The ports of the PostgreSQL databases +# Please set to 5432 +DB_PORT_CORE=5432 +DB_PORT_JMS=5432 +DB_PORT_APP=5432 +# Database password for the application core database +DB_PWD_CORE=<%= @db_pwd_core %> +# Database password for the Asynchronous Provisioning Platform (APP) database +DB_PWD_APP=<%= @db_pwd_app %> +# Database superuser (postgres) password +DB_SUPERPWD=<%= @var_db_superpwd %> -#mail +################# +# Mail settings # +################# +# The host name or IP address of your mail server SMTP_HOST=<%= @mail["host"] %> +# The port of your mail server SMTP_PORT=<%= @mail["port"] %> +# The sender email address that OSCM should use SMTP_FROM=<%= @mail["from"] %> +# The user name for your mail server if it requires authentication; if no +# authentication is required, please set none SMTP_USER=<%= @mail["user"] %> +# The password for your mail server if it requires authentication; if no +# authentication is required, please set none SMTP_PWD=<%= @mail["password"] %> +# Whether your mail server requires authentication; can be true or false SMTP_AUTH=<%= @mail["auth"] %> +# Whether to use TLS for mail server communication; can be true or false SMTP_TLS=<%= @mail["tls"] %> - -KEY_SECRET=<%= @key_secret %> -HOST_FQDN=<%= @host_fqdn %> - -REPORT_ENGINEURL=https://<%= @host_fqdn %>:8681/birt/frameset?__report=${reportname}.rptdesign&SessionId=${sessionid}&__locale=${locale}&WSDLURL=${wsdlurl}&SOAPEndPoint=${soapendpoint}&wsname=Report&wsport=ReportPort - -DB_PORT_CORE=5432 -DB_PORT_JMS=5432 -DB_PORT_APP=5432 - -DB_PWD_CORE=<%= @db_pwd_core %> -DB_PWD_APP=<%= @db_pwd_app %> - -DB_SUPERPWD=<%= @var_db_superpwd %> - +# The sender email address that the Asynchronous Provisioning Platform (APP) +# should use APP_ADMIN_MAIL_ADDRESS=<%= @mail["from"] %> +####################### +# Controller settings # +####################### +# The ID of the controller to be configured CONTROLLER_ID=ess.openstack +# Organization ID for controller authentication +# Please set to PLATFORM_OPERATOR CONTROLLER_ORG_ID=PLATFORM_OPERATOR +# Numerical user key of the controller administrator +# Please set to 1000 CONTROLLER_USER_KEY=1000 +# User name of the controller administrator +# Please set to administrator CONTROLLER_USER_NAME=administrator +# Password of the controller administrator +# Please set to admin123 CONTROLLER_USER_PASS=admin123 +################# +# Misc settings # +################# +# A secret string which will be used as a seed for encryption in the database +# Please do not lose this if you plan to keep your database +KEY_SECRET=<%= @key_secret %> +# The host name or IP address which you will use to access the application +HOST_FQDN=<%= @host_fqdn %> +# URL for the Report Engine +# Replace HOST_FQDN with the same value as above; please leave the other +# placeholders intact +REPORT_ENGINEURL=https://<%= @host_fqdn %>:8681/birt/frameset?__report=${reportname}.rptdesign&SessionId=${sessionid}&__locale=${locale}&WSDLURL=${wsdlurl}&SOAPEndPoint=${soapendpoint}&wsname=Report&wsport=ReportPort +# Debug logs; can be true or false TOMEE_DEBUG=false + +################ +# SOC settings # +################ +# Proxy +PROXY_ENABLED=<%= @proxy["use_proxy"] %> +PROXY_HTTP_HOST=<%= @proxy["http_host"] %> +PROXY_HTTP_PORT=<%= @proxy["http_port"] %> +PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> +PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> +PROXY_NOPROXY=<%= @no_proxy %> +PROXY_AUTH=<%= @proxy["auth"] %> +PROXY_USER=<%= @proxy["user"] %> +PROXY_PWD=<%= @proxy["password"] %> + +# Docker +DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> +DOCKER_REGISTRY_HOST=<%= @docker["host"] %> +DOCKER_REGISTRY_PORT=<%= @docker["port"] %> +DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> +DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> +DOCKER_REGISTRY_USER=<%= @docker["user"] %> +DOCKER_REGISTRY_PWD=<%= @docker["password"] %> +DOCKER_TAG=<%= @docker["tag"] %> From cfee582c919df89caa18b96b19ab18e4bf768c5a Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 9 Nov 2017 18:11:00 +0100 Subject: [PATCH 152/181] change setup log location to cinder volume --- chef/cookbooks/escm/files/default/user-data/deploy-escmserver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 662bbf8fa1..ec304de7e5 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -11,7 +11,7 @@ COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose DOCKER_PATH=/docker # Redirect all output to log file -exec &>> ${DOCKER_PATH}/setup-machine-$(date '+%Y-%m-%d').log +exec &>> ${DOCKER_PATH}/logs/setup-machine-$(date '+%Y-%m-%d').log echo "$(date '+%Y-%m-%d %H:%M:%S') starting setup..." From 48304dfcb1e5135df3f0455de8efe2548e104510 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Thu, 9 Nov 2017 18:13:18 +0100 Subject: [PATCH 153/181] change default version tag attribute format --- chef/data_bags/crowbar/template-escm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-escm.json b/chef/data_bags/crowbar/template-escm.json index d5731906a7..be8984ce0f 100644 --- a/chef/data_bags/crowbar/template-escm.json +++ b/chef/data_bags/crowbar/template-escm.json @@ -52,7 +52,7 @@ "user": "", "password": "", "organization": "servicecatalog", - "tag": "17.5" + "tag": "v17.5.0" }, "proxy": { "use_proxy": false, From 50702d3df6228b04aad99da32847ebadd649c4cc Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 10 Nov 2017 08:48:36 +0100 Subject: [PATCH 154/181] fix missing copy of .env file --- .../files/default/user-data/deploy-escmserver | 71 +------------------ 1 file changed, 2 insertions(+), 69 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index ec304de7e5..a90c0261da 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -37,46 +37,6 @@ if [ -f ${DOCKER_PATH}/docker-compose-escm.yml ]; then fi fi -# # Create Docker directories if they do not exist yet -# for docker_directory in \ -# ${DOCKER_PATH}/data/oscm-db/data \ -# ${DOCKER_PATH}/config/certs \ -# ${DOCKER_PATH}/config/oscm-branding/brandings \ -# ${DOCKER_PATH}/config/oscm-core/ssl/privkey \ -# ${DOCKER_PATH}/config/oscm-core/ssl/cert \ -# ${DOCKER_PATH}/config/oscm-core/ssl/chain \ -# ${DOCKER_PATH}/config/oscm-app/ssl/privkey \ -# ${DOCKER_PATH}/config/oscm-app/ssl/cert \ -# ${DOCKER_PATH}/config/oscm-app/ssl/chain \ -# ${DOCKER_PATH}/config/oscm-birt/ssl/privkey \ -# ${DOCKER_PATH}/config/oscm-birt/ssl/cert \ -# ${DOCKER_PATH}/config/oscm-birt/ssl/chain \ -# ${DOCKER_PATH}/config/oscm-branding/ssl/privkey \ -# ${DOCKER_PATH}/config/oscm-branding/ssl/cert \ -# ${DOCKER_PATH}/config/oscm-branding/ssl/chain \ -# ${DOCKER_PATH}/logs/oscm-app \ -# ${DOCKER_PATH}/logs/oscm-birt \ -# ${DOCKER_PATH}/logs/oscm-branding \ -# ${DOCKER_PATH}/logs/oscm-core \ -# ${DOCKER_PATH}/logs/oscm-db; do -# if [ ! -d ${docker_directory} ]; then -# mkdir -p ${docker_directory} -# fi -# done - -# # Create Docker log files if they do not exist yet -# for docker_log_file in \ -# ${DOCKER_PATH}/logs/oscm-app/oscm-app.out.log \ -# ${DOCKER_PATH}/logs/oscm-birt/oscm-birt.out.log \ -# ${DOCKER_PATH}/logs/oscm-branding/oscm-branding.out.log \ -# ${DOCKER_PATH}/logs/oscm-core/oscm-core.out.log \ -# ${DOCKER_PATH}/logs/oscm-db/oscm-db.out.log; do -# if [ ! -f {docker_log_file} ]; then -# touch ${docker_log_file} -# chmod 640 ${docker_log_file} -# fi -# done - # If http proxy is enabled, create Docker config file if [ ${PROXY_ENABLED} == "true" ]; then if [ ! -d /etc/systemd/system/docker.service.d ]; then @@ -155,20 +115,9 @@ EOF systemctl restart rsyslog fi -# # Copy Docker Compose files -# if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ]; then -# cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-initdb.yml ${DOCKER_PATH}/docker-compose-initdb.yml -# fi -# if [ -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ]; then -# cp -f ${COMPOSE_CONFIG_PATH}/docker-compose-escm.yml ${DOCKER_PATH}/docker-compose-escm.yml -# fi -# if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then -# cp -f ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH}/var.env -# fi - # Copy environment and configuration files to Docker directory -if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then - cp ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH} +if [ -f ${COMPOSE_CONFIG_PATH}/.env ]; then + cp ${COMPOSE_CONFIG_PATH}/.env ${DOCKER_PATH} fi if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then cp ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH} @@ -193,22 +142,6 @@ fi # Initialize the databases and start the application containers docker run --name deployer-init-start --rm -v ${DOCKER_PATH}:/target -v /var/run/docker.sock:/var/run/docker.sock -e INITDB=true -e STARTUP=true oscm-deployer -# # Pull images -# egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-initdb.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull -# egrep '^\s*image:' ${DOCKER_PATH}/docker-compose-escm.yml | sed 's|\s*image: \(.*\)|\1|g' | xargs -L 1 docker pull - -# # Initialize databases -# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up -d oscm-db -# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-core -# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-jms -# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-app -# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml up oscm-initdb-controller-openstack -# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml stop -# docker-compose -f ${DOCKER_PATH}/docker-compose-initdb.yml rm -f - -# # Start application containers -# docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml up -d - # Backup the current configuration for update check by the next run cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.current From e12a621981b85f5c5d26e380a72a170269f2a793 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Fri, 10 Nov 2017 09:24:11 +0100 Subject: [PATCH 155/181] fix pwd for docker-compose --- chef/cookbooks/escm/files/default/user-data/deploy-escmserver | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index a90c0261da..3743e73107 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -125,6 +125,7 @@ fi #TODO: Where does the deployer image come from? # Create Docker directory structure and Docker Compose files +pushd ${DOCKER_PATH} docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target oscm-deployer # Copy certificate files From 20d8d496cb8e4c6e7d9c7ec7700d255ef89298d1 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 10:01:17 +0100 Subject: [PATCH 156/181] add check for mandatory variables --- .../cookbooks/escm/files/default/user-data/deploy-escmserver | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 3743e73107..de10d82a43 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -29,6 +29,11 @@ source ${COMPOSE_CONFIG_PATH}/var.env # Disable automatic exporting of variables set +a +# If mandatory variables are unset, abort +if [ -z ${DB_PWD_CORE} ] || [ -z ${DB_PWD_APP} ] || [ -z ${KEY_SECRET} ] || [ -z ${HOST_FQDN} ]; then + exit 0 +fi + # If containers are running, stop them if [ -f ${DOCKER_PATH}/docker-compose-escm.yml ]; then if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml ps -q | wc -l) != "0" ]; then From 113ab78a940be9683483017b5a87975270ba0be3 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 10:45:43 +0100 Subject: [PATCH 157/181] fix db super password --- chef/cookbooks/escm/files/default/user-data/deploy-escmserver | 3 ++- chef/cookbooks/escm/templates/default/var.env.erb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index de10d82a43..5174ed0537 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -30,7 +30,8 @@ source ${COMPOSE_CONFIG_PATH}/var.env set +a # If mandatory variables are unset, abort -if [ -z ${DB_PWD_CORE} ] || [ -z ${DB_PWD_APP} ] || [ -z ${KEY_SECRET} ] || [ -z ${HOST_FQDN} ]; then +if [ -z ${DB_PWD_CORE} ] || [ -z ${DB_PWD_APP} ] || [ -z ${KEY_SECRET} ] || [ -z ${HOST_FQDN} ] || [ -z ${DB_SUPERPWD} ]; then + echo "Mandatory variables unset - aborting." exit 0 fi diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 820d57d291..5fb77a087d 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -15,7 +15,7 @@ DB_PWD_CORE=<%= @db_pwd_core %> # Database password for the Asynchronous Provisioning Platform (APP) database DB_PWD_APP=<%= @db_pwd_app %> # Database superuser (postgres) password -DB_SUPERPWD=<%= @var_db_superpwd %> +DB_SUPERPWD=<%= @db_password %> ################# # Mail settings # From 5c8905f6c06989832d2c789eeb633ecee27c8e29 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 11:04:53 +0100 Subject: [PATCH 158/181] fix db super password --- chef/cookbooks/escm/templates/default/var.env.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/templates/default/var.env.erb b/chef/cookbooks/escm/templates/default/var.env.erb index 5fb77a087d..9c6129444f 100644 --- a/chef/cookbooks/escm/templates/default/var.env.erb +++ b/chef/cookbooks/escm/templates/default/var.env.erb @@ -15,7 +15,7 @@ DB_PWD_CORE=<%= @db_pwd_core %> # Database password for the Asynchronous Provisioning Platform (APP) database DB_PWD_APP=<%= @db_pwd_app %> # Database superuser (postgres) password -DB_SUPERPWD=<%= @db_password %> +DB_SUPERPWD=<%= @db_superpwd %> ################# # Mail settings # From ee3e37b52d3e9af44bee5fca3fc53a613930cb5e Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 13:03:56 +0100 Subject: [PATCH 159/181] bump number of cpu for instance to 4 --- chef/data_bags/crowbar/template-escm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-escm.json b/chef/data_bags/crowbar/template-escm.json index be8984ce0f..49fc304508 100644 --- a/chef/data_bags/crowbar/template-escm.json +++ b/chef/data_bags/crowbar/template-escm.json @@ -18,7 +18,7 @@ "flavor": { "name": "escm.small", "ram": 8192, - "vcpus": 1, + "vcpus": 4, "disk": 20 }, "keypair": { From db5feb40da19b6c40e7c1ca57c22f30286b1e36c Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 14:06:27 +0100 Subject: [PATCH 160/181] enable syslog --- .../files/default/user-data/deploy-escmserver | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 5174ed0537..b840a6fc88 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -116,6 +116,7 @@ local1.* /docker/logs/oscm-core/oscm-core.out.log local2.* /docker/logs/oscm-app/oscm-app.out.log local3.* /docker/logs/oscm-birt/oscm-birt.out.log local4.* /docker/logs/oscm-branding/oscm-branding.out.log +local5.* /docker/logs/oscm-help/oscm-help.out.log EOF sed -i '/^local/d' /etc/rsyslog.conf systemctl restart rsyslog @@ -136,18 +137,25 @@ docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target os # Copy certificate files if [ -f ${SSL_CONFIG_PATH}/escm.key ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/privkey ${DOCKER_PATH}/config/oscm-app/ssl/privkey ${DOCKER_PATH}/config/oscm-birt/ssl/privkey ${DOCKER_PATH}/config/oscm-branding/ssl/privkey | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.key + for dir_privkey in ${DOCKER_PATH}/config/*; do + cp -f ${SSL_CONFIG_PATH}/escm.key ${dir_privkey}/ssl/privkey + done fi if [ -f ${SSL_CONFIG_PATH}/escm.crt ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/cert ${DOCKER_PATH}/config/oscm-app/ssl/cert ${DOCKER_PATH}/config/oscm-birt/ssl/cert ${DOCKER_PATH}/config/oscm-branding/ssl/cert ${DOCKER_PATH}/config/certs | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.crt + for dir_privkey in ${DOCKER_PATH}/config/*; do + cp -f ${SSL_CONFIG_PATH}/escm.crt ${dir_privkey}/ssl/cert + done + cp -f ${SSL_CONFIG_PATH}/escm.crt ${DOCKER_PATH}/config/certs fi if [ -f ${SSL_CONFIG_PATH}/escm.chain ]; then - echo ${DOCKER_PATH}/config/oscm-core/ssl/chain ${DOCKER_PATH}/config/oscm-app/ssl/chain ${DOCKER_PATH}/config/oscm-birt/ssl/chain ${DOCKER_PATH}/config/oscm-branding/ssl/chain | xargs -n 1 cp -f ${SSL_CONFIG_PATH}/escm.chain + for dir_privkey in ${DOCKER_PATH}/config/*; do + cp -f ${SSL_CONFIG_PATH}/escm.chain ${dir_privkey}/ssl/chain + done fi #TODO: Where does the deployer image come from? # Initialize the databases and start the application containers -docker run --name deployer-init-start --rm -v ${DOCKER_PATH}:/target -v /var/run/docker.sock:/var/run/docker.sock -e INITDB=true -e STARTUP=true oscm-deployer +docker run --name deployer-init-start --rm -v ${DOCKER_PATH}:/target -v /var/run/docker.sock:/var/run/docker.sock -e SYSLOG=true -e INITDB=true -e STARTUP=true oscm-deployer # Backup the current configuration for update check by the next run cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.current From bfe54cf7a32a3d33db824e09cd0ab2b75a0b78b9 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 14:56:09 +0100 Subject: [PATCH 161/181] fix and update docker image locations --- .../files/default/user-data/deploy-escmserver | 7 +++---- chef/cookbooks/escm/templates/default/.env.erb | 16 ++++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index b840a6fc88..263386f987 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -25,6 +25,7 @@ fi # Enable automatic exporting of variables set -a # Read configuration files +source ${COMPOSE_CONFIG_PATH}/.env source ${COMPOSE_CONFIG_PATH}/var.env # Disable automatic exporting of variables set +a @@ -130,10 +131,9 @@ if [ -f ${COMPOSE_CONFIG_PATH}/var.env ]; then cp ${COMPOSE_CONFIG_PATH}/var.env ${DOCKER_PATH} fi -#TODO: Where does the deployer image come from? # Create Docker directory structure and Docker Compose files pushd ${DOCKER_PATH} -docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target oscm-deployer +docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target ${IMAGE_DEPLOYER} # Copy certificate files if [ -f ${SSL_CONFIG_PATH}/escm.key ]; then @@ -153,9 +153,8 @@ if [ -f ${SSL_CONFIG_PATH}/escm.chain ]; then done fi -#TODO: Where does the deployer image come from? # Initialize the databases and start the application containers -docker run --name deployer-init-start --rm -v ${DOCKER_PATH}:/target -v /var/run/docker.sock:/var/run/docker.sock -e SYSLOG=true -e INITDB=true -e STARTUP=true oscm-deployer +docker run --name deployer-init-start --rm -v ${DOCKER_PATH}:/target -v /var/run/docker.sock:/var/run/docker.sock -e SYSLOG=true -e INITDB=true -e STARTUP=true ${IMAGE_DEPLOYER} # Backup the current configuration for update check by the next run cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.current diff --git a/chef/cookbooks/escm/templates/default/.env.erb b/chef/cookbooks/escm/templates/default/.env.erb index 32b6edced6..ade4cb718c 100644 --- a/chef/cookbooks/escm/templates/default/.env.erb +++ b/chef/cookbooks/escm/templates/default/.env.erb @@ -3,41 +3,45 @@ IMAGE_DB=<%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% else %> IMAGE_DB=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> <% end %> -IMAGE_CORE=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-core:soc <% if @docker["dockerhub"] == true %> IMAGE_CORE=<%= @docker["organization"]+"/oscm-core:"+@docker["tag"] %> <% else %> IMAGE_CORE=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> <% end %> -IMAGE_APP=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-app:soc <% if @docker["dockerhub"] == true %> IMAGE_APP=<%= @docker["organization"]+"/oscm-app:"+@docker["tag"] %> <% else %> IMAGE_APP=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> <% end %> -IMAGE_BIRT=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-birt:soc <% if @docker["dockerhub"] == true %> IMAGE_BIRT=<%= @docker["organization"]+"/oscm-birt:"+@docker["tag"] %> <% else %> IMAGE_BIRT=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> <% end %> -IMAGE_BRANDING=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-branding:soc <% if @docker["dockerhub"] == true %> IMAGE_BRANDING=<%= @docker["organization"]+"/oscm-branding:"+@docker["tag"] %> <% else %> IMAGE_BRANDING=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> <% end %> -IMAGE_INITDB=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-initdb:soc <% if @docker["dockerhub"] == true %> IMAGE_INITDB=<%= @docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> <% else %> IMAGE_INITDB=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> <% end %> -IMAGE_PROXY=artifactory.intern.est.fujitsu.com:5002/oscmdocker/oscm-proxy:soc <% if @docker["dockerhub"] == true %> IMAGE_PROXY=<%= @docker["organization"]+"/oscm-proxy:"+@docker["tag"] %> <% else %> IMAGE_PROXY=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-proxy:"+@docker["tag"] %> <% end %> +<% if @docker["dockerhub"] == true %> +IMAGE_HELP=<%= @docker["organization"]+"/oscm-help:"+@docker["tag"] %> +<% else %> +IMAGE_HELP=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-help:"+@docker["tag"] %> +<% end %> +<% if @docker["dockerhub"] == true %> +IMAGE_DEPLOYER=<%= @docker["organization"]+"/oscm-deployer:"+@docker["tag"] %> +<% else %> +IMAGE_DEPLOYER=<%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-deployer:"+@docker["tag"] %> +<% end %> DOCKER_PATH=/docker From 7975a25fc18a959ec5915db0fa1f99f6163d2f05 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 14:58:12 +0100 Subject: [PATCH 162/181] fix bug in ssl keypair deployment --- .../escm/files/default/user-data/deploy-escmserver | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 263386f987..07763c5746 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -137,18 +137,18 @@ docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target ${ # Copy certificate files if [ -f ${SSL_CONFIG_PATH}/escm.key ]; then - for dir_privkey in ${DOCKER_PATH}/config/*; do + for dir_privkey in ${DOCKER_PATH}/config/oscm-*; do cp -f ${SSL_CONFIG_PATH}/escm.key ${dir_privkey}/ssl/privkey done fi if [ -f ${SSL_CONFIG_PATH}/escm.crt ]; then - for dir_privkey in ${DOCKER_PATH}/config/*; do + for dir_privkey in ${DOCKER_PATH}/config/oscm-*; do cp -f ${SSL_CONFIG_PATH}/escm.crt ${dir_privkey}/ssl/cert done cp -f ${SSL_CONFIG_PATH}/escm.crt ${DOCKER_PATH}/config/certs fi if [ -f ${SSL_CONFIG_PATH}/escm.chain ]; then - for dir_privkey in ${DOCKER_PATH}/config/*; do + for dir_privkey in ${DOCKER_PATH}/config/oscm-*; do cp -f ${SSL_CONFIG_PATH}/escm.chain ${dir_privkey}/ssl/chain done fi From 4cddb183fb2d34a41b30910de6f773fea0f2d56a Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 17:05:46 +0100 Subject: [PATCH 163/181] fix stopping containers on update --- .../escm/files/default/user-data/deploy-escmserver | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 07763c5746..0283bb73dc 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -18,7 +18,8 @@ echo "$(date '+%Y-%m-%d %H:%M:%S') starting setup..." # Check if update is necessary if [ -f ${DOCKER_PATH}/var.env.current ]; then if [ $(diff ${DOCKER_PATH}/var.env.current ${COMPOSE_CONFIG_PATH}/var.env | wc -l) == "0" ]; then - exit + echo "$(date '+%Y-%m-%d %H:%M:%S') configuration not changed - aborting..." + exit 0 fi fi @@ -32,15 +33,15 @@ set +a # If mandatory variables are unset, abort if [ -z ${DB_PWD_CORE} ] || [ -z ${DB_PWD_APP} ] || [ -z ${KEY_SECRET} ] || [ -z ${HOST_FQDN} ] || [ -z ${DB_SUPERPWD} ]; then - echo "Mandatory variables unset - aborting." + echo "$(date '+%Y-%m-%d %H:%M:%S') Mandatory variables unset - aborting." exit 0 fi # If containers are running, stop them -if [ -f ${DOCKER_PATH}/docker-compose-escm.yml ]; then - if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml ps -q | wc -l) != "0" ]; then - docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml stop - docker-compose -f ${DOCKER_PATH}/docker-compose-escm.yml rm -f +if [ -f ${DOCKER_PATH}/docker-compose-oscm.yml ]; then + if [ $(docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml ps -q | wc -l) != "0" ]; then + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml stop + docker-compose -f ${DOCKER_PATH}/docker-compose-oscm.yml rm -f fi fi From 5f36ca0e754a2ced8a5d4c38e08702bd3f797f0d Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 17:09:35 +0100 Subject: [PATCH 164/181] updated ports in security policy --- .../escm/files/default/application.yaml | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/chef/cookbooks/escm/files/default/application.yaml b/chef/cookbooks/escm/files/default/application.yaml index 1803f90ffa..f3d9e54ad7 100644 --- a/chef/cookbooks/escm/files/default/application.yaml +++ b/chef/cookbooks/escm/files/default/application.yaml @@ -115,44 +115,32 @@ resources: remote_ip_prefix: 0.0.0.0/0 protocol: tcp ethertype: IPv4 - port_range_min: 80 - port_range_max: 80 + port_range_min: 8081 + port_range_max: 8081 - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp ethertype: IPv4 - port_range_min: 8080 - port_range_max: 8080 + port_range_min: 8443 + port_range_max: 8443 - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp ethertype: IPv4 - port_range_min: 8880 - port_range_max: 8880 + port_range_min: 8543 + port_range_max: 8543 - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp ethertype: IPv4 - port_range_min: 8081 - port_range_max: 8081 + port_range_min: 8681 + port_range_max: 8681 - direction: ingress remote_ip_prefix: 0.0.0.0/0 protocol: tcp ethertype: IPv4 port_range_min: 8881 port_range_max: 8881 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - ethertype: IPv4 - port_range_min: 8680 - port_range_max: 8680 - - direction: ingress - remote_ip_prefix: 0.0.0.0/0 - protocol: tcp - ethertype: IPv4 - port_range_min: 8681 - port_range_max: 8681 - direction: egress protocol: tcp ethertype: IPv4 From 0b5d06c39861cdd99eb519fd85d4d86ba5176fa2 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 17:17:20 +0100 Subject: [PATCH 165/181] remove protocol choice from ui, always ssl --- .../views/barclamp/escm/_edit_attributes.html.haml | 5 +---- crowbar_framework/config/locales/escm/en.yml | 13 ++++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml index 4fd259c284..df4ba1f181 100644 --- a/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml @@ -80,8 +80,7 @@ %legend = t(".ssl_header") - = select_field %w(api protocol), - :collection => :api_protocols_for_escm, + = :collection => :api_protocols_for_escm, "data-sslprefix" => "ssl", "data-sslcert" => "/etc/escm/ssl/certs/signing_cert.pem", "data-sslkey" => "/etc/escm/ssl/private/signing_key.pem" @@ -93,6 +92,4 @@ = string_field %w(ssl fqdn) = string_field %w(ssl certfile) = string_field %w(ssl keyfile) - = boolean_field %w(ssl insecure) = string_field %w(ssl ca_certs) - diff --git a/crowbar_framework/config/locales/escm/en.yml b/crowbar_framework/config/locales/escm/en.yml index afb259997c..7a6b1c8248 100644 --- a/crowbar_framework/config/locales/escm/en.yml +++ b/crowbar_framework/config/locales/escm/en.yml @@ -57,13 +57,12 @@ en: keypair_hint: 'SSH public key of the keypair for openstack instance which hosts ESCM. It cannot be updated.' api: protocol: 'Protocol' - ssl_header: 'SSL Support' + ssl_header: 'SSL Settings' ssl: - generate_certs: 'Generate (self-signed) certificates (implies insecure)' - fqdn: 'Host FQDN (if left empty, the floating IP will be used)' + generate_certs: 'Generate self-signed certificates' + fqdn: 'Host FQDN/IP address (if left empty, the floating IP will be used)' certfile: 'SSL Certificate File' - keyfile: 'SSL (Private) Key File' - insecure: 'SSL Certificate is insecure (for instance, self-signed)' - ca_certs: 'SSL CA Certificates File' + keyfile: 'SSL Private Key File' + ca_certs: 'SSL CA Certificates File (optional)' ssl_hint: 'Delete the SSL key pair on the given locations, in order to generate new one.' - \ No newline at end of file + From 306cc32955dbd2d8bbd45475bd7ac34e5dabc02d Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 17:28:18 +0100 Subject: [PATCH 166/181] Revert "remove protocol choice from ui, always ssl" This reverts commit 0b5d06c39861cdd99eb519fd85d4d86ba5176fa2. --- .../views/barclamp/escm/_edit_attributes.html.haml | 5 ++++- crowbar_framework/config/locales/escm/en.yml | 13 +++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml index df4ba1f181..4fd259c284 100644 --- a/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml @@ -80,7 +80,8 @@ %legend = t(".ssl_header") - = :collection => :api_protocols_for_escm, + = select_field %w(api protocol), + :collection => :api_protocols_for_escm, "data-sslprefix" => "ssl", "data-sslcert" => "/etc/escm/ssl/certs/signing_cert.pem", "data-sslkey" => "/etc/escm/ssl/private/signing_key.pem" @@ -92,4 +93,6 @@ = string_field %w(ssl fqdn) = string_field %w(ssl certfile) = string_field %w(ssl keyfile) + = boolean_field %w(ssl insecure) = string_field %w(ssl ca_certs) + diff --git a/crowbar_framework/config/locales/escm/en.yml b/crowbar_framework/config/locales/escm/en.yml index 7a6b1c8248..afb259997c 100644 --- a/crowbar_framework/config/locales/escm/en.yml +++ b/crowbar_framework/config/locales/escm/en.yml @@ -57,12 +57,13 @@ en: keypair_hint: 'SSH public key of the keypair for openstack instance which hosts ESCM. It cannot be updated.' api: protocol: 'Protocol' - ssl_header: 'SSL Settings' + ssl_header: 'SSL Support' ssl: - generate_certs: 'Generate self-signed certificates' - fqdn: 'Host FQDN/IP address (if left empty, the floating IP will be used)' + generate_certs: 'Generate (self-signed) certificates (implies insecure)' + fqdn: 'Host FQDN (if left empty, the floating IP will be used)' certfile: 'SSL Certificate File' - keyfile: 'SSL Private Key File' - ca_certs: 'SSL CA Certificates File (optional)' + keyfile: 'SSL (Private) Key File' + insecure: 'SSL Certificate is insecure (for instance, self-signed)' + ca_certs: 'SSL CA Certificates File' ssl_hint: 'Delete the SSL key pair on the given locations, in order to generate new one.' - + \ No newline at end of file From 4b614c2318de712184b070040927054263df540d Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 18:11:53 +0100 Subject: [PATCH 167/181] disable container logging to messages --- .../escm/files/default/user-data/deploy-escmserver | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 0283bb73dc..8a8655bfde 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -111,14 +111,16 @@ EOF fi # Create rsyslog configuration for Docker log files -if [ ! -f /etc/rsyslog.d/oscm.conf ]; then - cat < /etc/rsyslog.d/oscm.conf +if [ ! -f /etc/rsyslog.d/escm.conf ]; then + sed -i '/\/var\/log\/messages$/d' /etc/rsyslog.conf + cat < /etc/rsyslog.d/escm.conf local0.* /docker/logs/oscm-db/oscm-db.out.log local1.* /docker/logs/oscm-core/oscm-core.out.log local2.* /docker/logs/oscm-app/oscm-app.out.log local3.* /docker/logs/oscm-birt/oscm-birt.out.log local4.* /docker/logs/oscm-branding/oscm-branding.out.log local5.* /docker/logs/oscm-help/oscm-help.out.log +*.*;mail.none;news.none;local0.none;local1.none;local2.none;local3.none;local4.none;local5.none;local6.none;local7.none -/var/log/messages EOF sed -i '/^local/d' /etc/rsyslog.conf systemctl restart rsyslog From 0c5e1409dd243e796e9de199b5f3f1556a9143b7 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 18:41:38 +0100 Subject: [PATCH 168/181] removed superfluous comments and files --- chef/cookbooks/escm/recipes/server.rb | 22 -------------- .../escm/templates/default/escm.conf.erb | 29 ------------------- 2 files changed, 51 deletions(-) delete mode 100644 chef/cookbooks/escm/templates/default/escm.conf.erb diff --git a/chef/cookbooks/escm/recipes/server.rb b/chef/cookbooks/escm/recipes/server.rb index 0e0161591f..cfbd55fd71 100644 --- a/chef/cookbooks/escm/recipes/server.rb +++ b/chef/cookbooks/escm/recipes/server.rb @@ -297,26 +297,6 @@ action :create end -# template "#{escm_install_path}/docker-compose-initdb.yml" do -# source "docker-compose-initdb.yml.erb" -# owner escm_group -# group escm_group -# mode 0640 -# variables( -# docker: node[:escm][:docker] -# ) -# end - -# template "#{escm_install_path}/docker-compose-escm.yml" do -# source "docker-compose-escm.yml.erb" -# owner escm_group -# group escm_group -# mode 0640 -# variables( -# docker: node[:escm][:docker] -# ) -# end - var_no_proxy = node[:escm][:proxy][:no_proxy].empty? ? "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]}" : "#{node[:escm][:proxy][:no_proxy_default]},#{node[:escm][:openstack][:instance_stack][:ip_appserver]},#{node[:escm][:proxy][:no_proxy]}" var_key_secret = "#{node[:escm][:openstack][:instance_stack][:key_secret]}" var_host_fqdn = node[:escm][:ssl][:fqdn].empty? ? "#{node[:escm][:openstack][:instance_stack][:ip_appserver]}" : "#{node[:escm][:ssl][:fqdn]}" @@ -359,8 +339,6 @@ Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) command = "ssh #{args} #{ip_appserver} 'mkdir -p #{escm_path}/docker-compose'" command_out = shell_out(command) -# command = "scp #{args} #{escm_install_path}/docker-compose-*.yml #{ip_appserver}:#{escm_path}/docker-compose" -# command_out = shell_out(command) command = "scp #{args} #{escm_install_path}/var.env #{ip_appserver}:#{escm_path}/docker-compose" command_out = shell_out(command) command = "scp #{args} #{escm_install_path}/.env #{ip_appserver}:#{escm_path}/docker-compose" diff --git a/chef/cookbooks/escm/templates/default/escm.conf.erb b/chef/cookbooks/escm/templates/default/escm.conf.erb deleted file mode 100644 index 9363ae77e2..0000000000 --- a/chef/cookbooks/escm/templates/default/escm.conf.erb +++ /dev/null @@ -1,29 +0,0 @@ -#proxy -PROXY_ENABLED=<%= @proxy["use_proxy"] %> -PROXY_HTTP_HOST=<%= @proxy["http_host"] %> -PROXY_HTTP_PORT=<%= @proxy["http_port"] %> -PROXY_HTTPS_HOST=<%= @proxy["https_host"] %> -PROXY_HTTPS_PORT=<%= @proxy["https_port"] %> -#<% if @proxy["no_proxy"].empty? %> -#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"] %> -#<% else %> -#PROXY_NOPROXY=<%= @proxy["no_proxy_default"]+","+@instance["ip_appserver"]+","+@proxy["no_proxy"] %> -#<% end %> -PROXY_NOPROXY=<%= @proxy["no_proxy"] %> -PROXY_AUTH=<%= @proxy["auth"] %> -PROXY_USER=<%= @proxy["user"] %> -PROXY_PWD=<%= @proxy["password"] %> - -#docker -DOCKER_REGISTRY_DOCKERHUB=<%= @docker["dockerhub"] %> -DOCKER_REGISTRY_HOST=<%= @docker["host"] %> -DOCKER_REGISTRY_PORT=<%= @docker["port"] %> -DOCKER_REGISTRY_ORGANIZATION=<%= @docker["organization"] %> -DOCKER_REGISTRY_AUTH=<%= @docker["auth"] %> -DOCKER_REGISTRY_USER=<%= @docker["user"] %> -DOCKER_REGISTRY_PWD=<%= @docker["password"] %> - -#host -HOST_FQDN=<%= @host_fqdn %> - - \ No newline at end of file From dd7ac844442e5facfc293ecbcc62d33ef1426072 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Mon, 13 Nov 2017 18:57:58 +0100 Subject: [PATCH 169/181] removed unused insecure flag --- chef/cookbooks/escm/attributes/default.rb | 2 -- chef/data_bags/crowbar/template-escm.json | 1 - chef/data_bags/crowbar/template-escm.schema | 1 - .../views/barclamp/escm/_edit_attributes.html.haml | 2 -- crowbar_framework/config/locales/escm/en.yml | 13 ++++++------- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/chef/cookbooks/escm/attributes/default.rb b/chef/cookbooks/escm/attributes/default.rb index 819bd01324..031d4e257f 100644 --- a/chef/cookbooks/escm/attributes/default.rb +++ b/chef/cookbooks/escm/attributes/default.rb @@ -19,6 +19,4 @@ default[:escm][:ssl][:certfile] = "/etc/escm/ssl/certs/signing_cert.pem" default[:escm][:ssl][:keyfile] = "/etc/escm/ssl/private/signing_key.pem" default[:escm][:ssl][:generate_certs] = false -default[:escm][:ssl][:insecure] = false default[:escm][:ssl][:ca_certs] = "/etc/escm/ssl/certs/ca.pem" - diff --git a/chef/data_bags/crowbar/template-escm.json b/chef/data_bags/crowbar/template-escm.json index 49fc304508..505ebc0661 100644 --- a/chef/data_bags/crowbar/template-escm.json +++ b/chef/data_bags/crowbar/template-escm.json @@ -71,7 +71,6 @@ "keyfile": "/etc/escm/ssl/private/signing_key.pem", "generate_certs": false, "fqdn": "", - "insecure": false, "ca_certs": "/etc/escm/ssl/certs/ca.pem" }, "api": { diff --git a/chef/data_bags/crowbar/template-escm.schema b/chef/data_bags/crowbar/template-escm.schema index d09d45b353..9323e52d5b 100644 --- a/chef/data_bags/crowbar/template-escm.schema +++ b/chef/data_bags/crowbar/template-escm.schema @@ -119,7 +119,6 @@ "keyfile": { "type" : "str", "required" : true }, "generate_certs": { "type" : "bool", "required" : true }, "fqdn": { "type" : "str", "required" : true }, - "insecure": { "type" : "bool", "required" : true }, "ca_certs": { "type" : "str", "required" : true } } }, diff --git a/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml index 4fd259c284..8669c5988c 100644 --- a/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/escm/_edit_attributes.html.haml @@ -93,6 +93,4 @@ = string_field %w(ssl fqdn) = string_field %w(ssl certfile) = string_field %w(ssl keyfile) - = boolean_field %w(ssl insecure) = string_field %w(ssl ca_certs) - diff --git a/crowbar_framework/config/locales/escm/en.yml b/crowbar_framework/config/locales/escm/en.yml index afb259997c..7a6b1c8248 100644 --- a/crowbar_framework/config/locales/escm/en.yml +++ b/crowbar_framework/config/locales/escm/en.yml @@ -57,13 +57,12 @@ en: keypair_hint: 'SSH public key of the keypair for openstack instance which hosts ESCM. It cannot be updated.' api: protocol: 'Protocol' - ssl_header: 'SSL Support' + ssl_header: 'SSL Settings' ssl: - generate_certs: 'Generate (self-signed) certificates (implies insecure)' - fqdn: 'Host FQDN (if left empty, the floating IP will be used)' + generate_certs: 'Generate self-signed certificates' + fqdn: 'Host FQDN/IP address (if left empty, the floating IP will be used)' certfile: 'SSL Certificate File' - keyfile: 'SSL (Private) Key File' - insecure: 'SSL Certificate is insecure (for instance, self-signed)' - ca_certs: 'SSL CA Certificates File' + keyfile: 'SSL Private Key File' + ca_certs: 'SSL CA Certificates File (optional)' ssl_hint: 'Delete the SSL key pair on the given locations, in order to generate new one.' - \ No newline at end of file + From 9efb6d5d0cff1f693a9e72c7814a564fd3e2ed1f Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 10:09:15 +0100 Subject: [PATCH 170/181] reduce protocol choice to https --- crowbar_framework/app/helpers/barclamp/escm_helper.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/crowbar_framework/app/helpers/barclamp/escm_helper.rb b/crowbar_framework/app/helpers/barclamp/escm_helper.rb index c6b253331e..2ae26a9015 100644 --- a/crowbar_framework/app/helpers/barclamp/escm_helper.rb +++ b/crowbar_framework/app/helpers/barclamp/escm_helper.rb @@ -19,7 +19,6 @@ module EscmHelper def api_protocols_for_escm(selected) options_for_select( [ - ["HTTP", "http"], ["HTTPS", "https"] ], selected.to_s From 3a6e309627a74137bf8fc84100ed9f922e14e2bc Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 10:48:28 +0100 Subject: [PATCH 171/181] remove superfluous templates --- .../default/docker-compose-escm.yml.erb | 111 ------------------ .../default/docker-compose-initdb.yml.erb | 72 ------------ 2 files changed, 183 deletions(-) delete mode 100644 chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb delete mode 100644 chef/cookbooks/escm/templates/default/docker-compose-initdb.yml.erb diff --git a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb deleted file mode 100644 index 4395102015..0000000000 --- a/chef/cookbooks/escm/templates/default/docker-compose-escm.yml.erb +++ /dev/null @@ -1,111 +0,0 @@ -version: "3" -services: - oscm-db: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> -<% end %> - container_name: oscm-db - logging: - driver: syslog - options: - syslog-facility: "local0" - volumes: - - /docker/data/oscm-db/data:/var/lib/postgresql/data - ports: - - 5432:5432 - - oscm-core: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-core:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-core:"+@docker["tag"] %> -<% end %> - container_name: oscm-core - logging: - driver: syslog - options: - syslog-facility: "local1" - env_file: var.env - environment: - - JPDA_ADDRESS=8000 - - JPDA_TRANSPORT=dt_socket - links: - - oscm-db:oscm-db - volumes: - - /docker/config/oscm-core/ssl/privkey:/import/ssl/privkey - - /docker/config/oscm-core/ssl/cert:/import/ssl/cert - - /docker/config/oscm-core/ssl/chain:/import/ssl/chain - - /docker/config/certs:/import/certs - ports: - - 8080:8080 - - 8081:8081 - - oscm-app: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-app:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-app:"+@docker["tag"] %> -<% end %> - container_name: oscm-app - logging: - driver: syslog - options: - syslog-facility: "local2" - env_file: var.env - environment: - - JPDA_ADDRESS=8000 - - JPDA_TRANSPORT=dt_socket - links: - - oscm-db:oscm-db - volumes: - - /docker/config/oscm-app/ssl/privkey:/import/ssl/privkey - - /docker/config/oscm-app/ssl/cert:/import/ssl/cert - - /docker/config/oscm-app/ssl/chain:/import/ssl/chain - - /docker/config/certs:/import/certs - ports: - - 8880:8880 - - 8881:8881 - - oscm-birt: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-birt:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-birt:"+@docker["tag"] %> -<% end %> - container_name: oscm-birt - logging: - driver: syslog - options: - syslog-facility: "local3" - env_file: var.env - volumes: - - /docker/config/oscm-birt/ssl/privkey:/import/ssl/privkey - - /docker/config/oscm-birt/ssl/cert:/import/ssl/cert - - /docker/config/oscm-birt/ssl/chain:/import/ssl/chain - - /docker/config/certs:/import/certs - ports: - - 8680:8680 - - 8681:8681 - - oscm-branding: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-branding:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-branding:"+@docker["tag"] %> -<% end %> - container_name: oscm-branding - logging: - driver: syslog - options: - syslog-facility: "local4" - env_file: var.env - volumes: - - /docker/config/brandings:/import/brandings:ro - - /docker/config/oscm-branding/ssl/privkey:/import/ssl/privkey - - /docker/config/oscm-branding/ssl/cert:/import/ssl/cert - - /docker/config/oscm-branding/ssl/chain:/import/ssl/chain - - /docker/config/certs:/import/certs - ports: - - 4443:443 diff --git a/chef/cookbooks/escm/templates/default/docker-compose-initdb.yml.erb b/chef/cookbooks/escm/templates/default/docker-compose-initdb.yml.erb deleted file mode 100644 index e172da8088..0000000000 --- a/chef/cookbooks/escm/templates/default/docker-compose-initdb.yml.erb +++ /dev/null @@ -1,72 +0,0 @@ -version: "3" -services: - oscm-db: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-db:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-db:"+@docker["tag"] %> -<% end %> - container_name: oscm-db - env_file: var.env - volumes: - - /docker/data/oscm-db/data:/var/lib/postgresql/data - - oscm-initdb-core: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-initdb-core:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> -<% end %> - container_name: oscm-initdb-core - env_file: var.env - environment: - - TARGET=CORE - - SOURCE=INIT - - OVERWRITE=true - links: - - oscm-db:oscm-db - - oscm-initdb-jms: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-initdb-jms:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> -<% end %> - container_name: oscm-initdb-jms - env_file: var.env - environment: - - TARGET=JMS - - SOURCE=INIT - - OVERWRITE=true - links: - - oscm-db:oscm-db - - oscm-initdb-app: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-initdb-app:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> -<% end %> - container_name: oscm-initdb-app - env_file: var.env - environment: - - TARGET=APP - - SOURCE=INIT - - OVERWRITE=true - links: - - oscm-db:oscm-db - - oscm-initdb-controller-openstack: -<% if @docker["dockerhub"] == true %> - image: <%= @docker["organization"]+"/oscm-initdb-controller-openstack:"+@docker["tag"] %> -<% else %> - image: <%= @docker["host"]+":"+@docker["port"].to_s+"/"+@docker["organization"]+"/oscm-initdb:"+@docker["tag"] %> -<% end %> - container_name: oscm-initdb-controller-openstack - env_file: var.env - environment: - - TARGET=CONTROLLER - - SOURCE=NONE - - OVERWRITE=true - links: - - oscm-db:oscm-db From e2bef30f55b8299c43fbbf524db308ff73e2cc7a Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 11:23:23 +0100 Subject: [PATCH 172/181] add swap file for Docker instance --- chef/cookbooks/escm/files/default/user-data/heat-config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index 2955c833db..b1a58e98d1 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -58,6 +58,12 @@ if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then mount ${volume_dev_data} fi +if [ $(grep SwapTotal /proc/meminfo | awk '{ print $2 }') -gt 0 ]; then + dd if=/dev/zero of=${DOCKER_PATH}/data/swapfile bs=1g count=1 + echo "${DOCKER_PATH}/data/swapfile swap swap defaults 0 0" + swapon ${DOCKER_PATH}/data/swapfile +fi + # Signal to OpenStack that we're finished and ready WAIT_CURL_FULL="$WAIT_CURL --data-binary '{\"status\": \"SUCCESS\"}' --noproxy '*'" eval ${WAIT_CURL_FULL} From 565271c2b74452f241c2e31b6a9811593cd054f8 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 12:47:21 +0100 Subject: [PATCH 173/181] fix error with swap creation --- chef/cookbooks/escm/files/default/user-data/heat-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index b1a58e98d1..cb04b68b2b 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -58,7 +58,7 @@ if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then mount ${volume_dev_data} fi -if [ $(grep SwapTotal /proc/meminfo | awk '{ print $2 }') -gt 0 ]; then +if [ $(grep SwapTotal /proc/meminfo | awk '{ print $2 }') -eq 0 ]; then dd if=/dev/zero of=${DOCKER_PATH}/data/swapfile bs=1g count=1 echo "${DOCKER_PATH}/data/swapfile swap swap defaults 0 0" swapon ${DOCKER_PATH}/data/swapfile From 67c569da5df1d33719065a346823c6ed357e8ea6 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 13:05:53 +0100 Subject: [PATCH 174/181] fix another error in swap creation --- chef/cookbooks/escm/files/default/user-data/heat-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index cb04b68b2b..e073307cb3 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -59,7 +59,7 @@ if ! grep -qs "${DOCKER_PATH}/data" /proc/mounts; then fi if [ $(grep SwapTotal /proc/meminfo | awk '{ print $2 }') -eq 0 ]; then - dd if=/dev/zero of=${DOCKER_PATH}/data/swapfile bs=1g count=1 + dd if=/dev/zero of=${DOCKER_PATH}/data/swapfile bs=1G count=1 echo "${DOCKER_PATH}/data/swapfile swap swap defaults 0 0" swapon ${DOCKER_PATH}/data/swapfile fi From 662d80f5dd175fc0ac8f2ebc872925f15e4fa9ad Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 13:19:37 +0100 Subject: [PATCH 175/181] increase size of cinder volumes --- chef/cookbooks/escm/files/default/volumes.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/volumes.yaml b/chef/cookbooks/escm/files/default/volumes.yaml index a8cb50fc95..8ddd8abd92 100644 --- a/chef/cookbooks/escm/files/default/volumes.yaml +++ b/chef/cookbooks/escm/files/default/volumes.yaml @@ -5,11 +5,11 @@ heat_template_version: 2015-10-15 parameters: logs_size: type: number - default: 1 + default: 2 description: Size of the logs volume in gigabytes data_size: type: number - default: 1 + default: 2 description: Size of the data volume in gigabytes From 57d877bc6a8cfa56d0cf15bceb20aa4b7b2be2ae Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 14:23:23 +0100 Subject: [PATCH 176/181] fix size increase for cinder volumes --- chef/data_bags/crowbar/template-escm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/data_bags/crowbar/template-escm.json b/chef/data_bags/crowbar/template-escm.json index 505ebc0661..df34035aec 100644 --- a/chef/data_bags/crowbar/template-escm.json +++ b/chef/data_bags/crowbar/template-escm.json @@ -27,8 +27,8 @@ }, "volume_stack": { "stack_name": "escm-volumes", - "data_volume_size": 1, - "logs_volume_size": 1 + "data_volume_size": 2, + "logs_volume_size": 2 }, "instance_stack": { "stack_name": "escm-instances" From 0bc853ff9f4be27dbbbffefe782e463e952d4724 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 14:33:23 +0100 Subject: [PATCH 177/181] create swap --- chef/cookbooks/escm/files/default/user-data/heat-config | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index e073307cb3..c5af61fd7c 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -60,6 +60,7 @@ fi if [ $(grep SwapTotal /proc/meminfo | awk '{ print $2 }') -eq 0 ]; then dd if=/dev/zero of=${DOCKER_PATH}/data/swapfile bs=1G count=1 + mkswap -f ${DOCKER_PATH}/data/swapfile echo "${DOCKER_PATH}/data/swapfile swap swap defaults 0 0" swapon ${DOCKER_PATH}/data/swapfile fi From aaf05a6b5bd8647164fff549a85da54af04e9706 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 14:44:40 +0100 Subject: [PATCH 178/181] fix permissions on swap file --- chef/cookbooks/escm/files/default/user-data/heat-config | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/escm/files/default/user-data/heat-config b/chef/cookbooks/escm/files/default/user-data/heat-config index c5af61fd7c..5b68a800fe 100644 --- a/chef/cookbooks/escm/files/default/user-data/heat-config +++ b/chef/cookbooks/escm/files/default/user-data/heat-config @@ -60,6 +60,7 @@ fi if [ $(grep SwapTotal /proc/meminfo | awk '{ print $2 }') -eq 0 ]; then dd if=/dev/zero of=${DOCKER_PATH}/data/swapfile bs=1G count=1 + chmod 600 ${DOCKER_PATH}/data/swapfile mkswap -f ${DOCKER_PATH}/data/swapfile echo "${DOCKER_PATH}/data/swapfile swap swap defaults 0 0" swapon ${DOCKER_PATH}/data/swapfile From ec35c080e3a59b45d8d43d7ff8dcf9c54b63c68a Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 16:08:10 +0100 Subject: [PATCH 179/181] rename old config file for clarity --- .../escm/files/default/user-data/deploy-escmserver | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 8a8655bfde..ee212d6c0f 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -16,8 +16,8 @@ exec &>> ${DOCKER_PATH}/logs/setup-machine-$(date '+%Y-%m-%d').log echo "$(date '+%Y-%m-%d %H:%M:%S') starting setup..." # Check if update is necessary -if [ -f ${DOCKER_PATH}/var.env.current ]; then - if [ $(diff ${DOCKER_PATH}/var.env.current ${COMPOSE_CONFIG_PATH}/var.env | wc -l) == "0" ]; then +if [ -f ${DOCKER_PATH}/var.env.old ]; then + if [ $(diff ${DOCKER_PATH}/var.env.old ${COMPOSE_CONFIG_PATH}/var.env | wc -l) == "0" ]; then echo "$(date '+%Y-%m-%d %H:%M:%S') configuration not changed - aborting..." exit 0 fi @@ -160,6 +160,6 @@ fi docker run --name deployer-init-start --rm -v ${DOCKER_PATH}:/target -v /var/run/docker.sock:/var/run/docker.sock -e SYSLOG=true -e INITDB=true -e STARTUP=true ${IMAGE_DEPLOYER} # Backup the current configuration for update check by the next run -cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.current +cp ${DOCKER_PATH}/var.env ${DOCKER_PATH}/var.env.old echo "$(date '+%Y-%m-%d %H:%M:%S') setup finished." From 1e3b8edf3d27d0a9877debf78a885324da0b9191 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 16:45:41 +0100 Subject: [PATCH 180/181] add check to update on certificate change --- .../files/default/user-data/deploy-escmserver | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index ee212d6c0f..89e23d40e3 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -9,6 +9,8 @@ CONFIG_BASE=/etc/escm SSL_CONFIG_PATH=${CONFIG_BASE}/ssl COMPOSE_CONFIG_PATH=${CONFIG_BASE}/docker-compose DOCKER_PATH=/docker +UPDATE_NECESSARY_CONFIG=true +UPDATE_NECESSARY_CERTS=true # Redirect all output to log file exec &>> ${DOCKER_PATH}/logs/setup-machine-$(date '+%Y-%m-%d').log @@ -18,9 +20,17 @@ echo "$(date '+%Y-%m-%d %H:%M:%S') starting setup..." # Check if update is necessary if [ -f ${DOCKER_PATH}/var.env.old ]; then if [ $(diff ${DOCKER_PATH}/var.env.old ${COMPOSE_CONFIG_PATH}/var.env | wc -l) == "0" ]; then + UPDATE_NECESSARY_CONFIG=false + fi +fi + +if ! (sha512sum --status --ignore-missing -c ${DOCKER_PATH}/escm.key.old.sha512sum && sha512sum --status --ignore-missing -c ${DOCKER_PATH}/escm.crt.old.sha512sum); then + UPDATE_NECESSARY_CERTS=false +fi + +if [ ${UPDATE_NECESSARY_CONFIG} == "false" ] && [ ${UPDATE_NECESSARY_CERTS} == "false" ]; then echo "$(date '+%Y-%m-%d %H:%M:%S') configuration not changed - aborting..." exit 0 - fi fi # Enable automatic exporting of variables @@ -139,20 +149,24 @@ pushd ${DOCKER_PATH} docker run --name deployer-directories-compose --rm -v ${DOCKER_PATH}:/target ${IMAGE_DEPLOYER} # Copy certificate files +# and save the hashes of the current certificate files for update check by the next run if [ -f ${SSL_CONFIG_PATH}/escm.key ]; then for dir_privkey in ${DOCKER_PATH}/config/oscm-*; do cp -f ${SSL_CONFIG_PATH}/escm.key ${dir_privkey}/ssl/privkey done + sha512sum ${SSL_CONFIG_PATH}/escm.key > ${DOCKER_PATH}/escm.key.old.sha512sum fi if [ -f ${SSL_CONFIG_PATH}/escm.crt ]; then for dir_privkey in ${DOCKER_PATH}/config/oscm-*; do cp -f ${SSL_CONFIG_PATH}/escm.crt ${dir_privkey}/ssl/cert done cp -f ${SSL_CONFIG_PATH}/escm.crt ${DOCKER_PATH}/config/certs + sha512sum ${SSL_CONFIG_PATH}/escm.crt > ${DOCKER_PATH}/escm.crt.old.sha512sum fi if [ -f ${SSL_CONFIG_PATH}/escm.chain ]; then for dir_privkey in ${DOCKER_PATH}/config/oscm-*; do cp -f ${SSL_CONFIG_PATH}/escm.chain ${dir_privkey}/ssl/chain + sha512sum ${SSL_CONFIG_PATH}/escm.chain > ${DOCKER_PATH}/escm.chain.old.sha512sum done fi From c3e1b3bb8f2a1d003740c51ff55273beb78cb8a2 Mon Sep 17 00:00:00 2001 From: Frank Shimizu Date: Tue, 14 Nov 2017 17:27:16 +0100 Subject: [PATCH 181/181] additional check for certificate changed check --- .../escm/files/default/user-data/deploy-escmserver | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver index 89e23d40e3..d8ea750bc3 100644 --- a/chef/cookbooks/escm/files/default/user-data/deploy-escmserver +++ b/chef/cookbooks/escm/files/default/user-data/deploy-escmserver @@ -24,8 +24,10 @@ if [ -f ${DOCKER_PATH}/var.env.old ]; then fi fi -if ! (sha512sum --status --ignore-missing -c ${DOCKER_PATH}/escm.key.old.sha512sum && sha512sum --status --ignore-missing -c ${DOCKER_PATH}/escm.crt.old.sha512sum); then - UPDATE_NECESSARY_CERTS=false +if [ -f ${DOCKER_PATH}/escm.key.old.sha512sum ] && [ -f ${DOCKER_PATH}/escm.crt.old.sha512sum ]; then + if sha512sum --status --ignore-missing -c ${DOCKER_PATH}/escm.key.old.sha512sum && sha512sum --status --ignore-missing -c ${DOCKER_PATH}/escm.crt.old.sha512sum; then + UPDATE_NECESSARY_CERTS=false + fi fi if [ ${UPDATE_NECESSARY_CONFIG} == "false" ] && [ ${UPDATE_NECESSARY_CERTS} == "false" ]; then