Fix glue update script

.docker/base/Dockerfile.base

@@ -0,0 +1,106 @@
+FROM debian:stretch
+
+RUN echo "deb http://archive.debian.org/debian stretch main" > /etc/apt/sources.list 
+RUN echo "deb http://archive.debian.org/debian-security stretch/updates main" >> /etc/apt/sources.list 
+
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends \
+		bzip2 \
+		ca-certificates \
+		libffi-dev \
+		libgdbm3 \
+		libssl1.0-dev \
+		libyaml-dev \
+		procps \
+		zlib1g-dev \
+	&& rm -rf /var/lib/apt/lists/*
+
+# skip installing gem documentation
+RUN mkdir -p /usr/local/etc \
+	&& { \
+		echo 'install: --no-document'; \
+		echo 'update: --no-document'; \
+	} >> /usr/local/etc/gemrc
+
+ENV RUBY_MAJOR 2.1
+ENV RUBY_VERSION 2.1.10
+ENV RUBY_DOWNLOAD_SHA256 5be9f8d5d29d252cd7f969ab7550e31bbb001feb4a83532301c0dd3b5006e148
+ENV RUBYGEMS_VERSION 2.6.12
+ENV BUNDLER_VERSION 1.15.1
+
+# some of ruby's build scripts are written in ruby
+#   we purge system ruby later to make sure our final image uses what we just built
+RUN set -ex \
+	\
+	&& buildDeps=' \
+		autoconf \
+		bison \
+		dpkg-dev \
+		gcc \
+		libbz2-dev \
+		libgdbm-dev \
+		libglib2.0-dev \
+		libncurses-dev \
+		libreadline-dev \
+		libxml2-dev \
+		libxslt-dev \
+		make \
+		ruby \
+		wget \
+		xz-utils \
+	' \
+	&& apt-get update \
+	&& apt-get install -y --no-install-recommends $buildDeps \
+	&& rm -rf /var/lib/apt/lists/* \
+	\
+	&& wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz" \
+	&& echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum -c - \
+	\
+	&& mkdir -p /usr/src/ruby \
+	&& tar -xJf ruby.tar.xz -C /usr/src/ruby --strip-components=1 \
+	&& rm ruby.tar.xz \
+	\
+	&& cd /usr/src/ruby \
+	\
+# hack in "ENABLE_PATH_CHECK" disabling to suppress:
+#   warning: Insecure world writable dir
+	&& { \
+		echo '#define ENABLE_PATH_CHECK 0'; \
+		echo; \
+		cat file.c; \
+	} > file.c.new \
+	&& mv file.c.new file.c \
+	\
+	&& autoconf \
+	&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
+	&& ./configure \
+		--build="$gnuArch" \
+		--disable-install-doc \
+		--enable-shared \
+	&& make -j "$(nproc)" \
+	&& make install \
+	\
+	&& dpkg-query --show --showformat '${package}\n' \
+		| grep -P '^libreadline\d+$' \
+		| xargs apt-mark manual \
+	&& apt-get purge -y --auto-remove $buildDeps \
+	&& cd / \
+	&& rm -r /usr/src/ruby \
+	\
+	&& gem update --system "$RUBYGEMS_VERSION" \
+	&& gem install bundler --version "$BUNDLER_VERSION" --force \
+	&& rm -r /root/.gem/
+
+# install things globally, for great justice
+# and don't create ".bundle" in all our apps
+ENV GEM_HOME /usr/local/bundle
+ENV BUNDLE_PATH="$GEM_HOME" \
+	BUNDLE_SILENCE_ROOT_WARNING=1 \
+	BUNDLE_APP_CONFIG="$GEM_HOME"
+# path recommendation: https://github.com/bundler/bundler/pull/6469#issuecomment-383235438
+ENV PATH $GEM_HOME/bin:$BUNDLE_PATH/gems/bin:$PATH
+# adjust permissions of a few directories for running "gem install" as an arbitrary user
+RUN mkdir -p "$GEM_HOME" && chmod 777 "$GEM_HOME"
+# (BUNDLE_PATH = GEM_HOME, no need to mkdir/chown both)
+
+CMD [ "irb" ]

.docker/config/database.rb

@@ -0,0 +1,9 @@
+require File.join(File.dirname(__FILE__), "..", "lib/ojdbc7-12.1.0.2.0.jar")
+Sequel::Model.plugin(:schema)
+Sequel::Model.raise_on_save_failure = false # Do not throw exceptions on failure
+Sequel::Model.db = case Padrino.env
+  when :development then Sequel.connect("B2B_URI")
+  when :production  then Sequel.connect("B2B_URI")
+  when :test        then Sequel.connect("B2B_URI")
+  when :cte         then Sequel.connect("B2B_URI")
+end

.docker/config/glue_update.sh

@@ -0,0 +1,235 @@
+#!/bin/bash -xe
+#set -e
+## Global Vars
+
+cd /edidb
+
+if [ -z "$1" ]
+then
+  DAYS=2
+else
+  DAYS=$1
+fi
+
+export START_DATE=`date --date="$DAYS days ago" +%Y%m%d000000`
+export ENV_NAME
+export HBX_ID
+export EDIDB_DB_HOST
+export EDIDB_DB_NAME
+export EDIDB_DB_PASSWORD
+export B2B_HOST
+export B2B_SERVICE_PASSWORD
+export SLACK_TOKEN
+export SLACK_CHANNEL
+export TO_ADDRESSES
+export EMAIL_FROM_ADDRESS
+export EDIDB_CURL_URL
+export RABBITMQ_CURL_URL
+export RABBITMQ_USER
+export RABBITMQ_PASSWORD
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+export REPORT_ZIP_PASSWORD
+
+## notification function
+function send_sms_notification
+{
+cat << EOH > sms_notification.rb
+#!/usr/bin/env ruby
+
+require 'active_resource'
+require 'json'
+require 'aws-sdk'
+
+ses = Aws::SES::Client.new(
+  region: 'us-east-1',
+  access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+  secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
+)
+
+date = Time.now.inspect
+email_subject = "GlueDB Update $1 \n\n"
+email_body = "\n$1 at: \n#{date}\n\n$2"
+
+
+resp = ses.send_email({
+  source: ENV['EMAIL_FROM_ADDRESS'],
+  destination: {
+    to_addresses: ENV.fetch('TO_ADDRESSES').split(',')
+  },
+  message: {
+    subject: {
+      data: email_subject
+    },
+    body: {
+      text: {
+        data: email_body
+      }
+    },
+  },
+  reply_to_addresses: ENV.fetch('EMAIL_FROM_ADDRESS').split(','),
+})
+
+EOH
+
+ruby ./sms_notification.rb
+
+}
+
+## slack message to note the beginning of the glue update
+curl -X POST -H "Content-type: application/json; charset=utf-8" -H "Authorization: Bearer ${SLACK_TOKEN}" --data '{"channel": "#SLACK_CHANNEL", "username": "EDI Database Bot", "text": "'\`' ### GlueDB Update Started ### '\`'", "icon_emoji": ":gear:"}' https://slack.com/api/chat.postMessage
+
+cat > script.sh <<'EOL'
+#!/bin/bash -xe
+
+##App Server Vars
+export PARSER_DIRECTORY='/edidb/ediparser'
+export GLUEDB_DIRECTORY='/edidb'
+export UPDATER_DIRECTORY='/edidb/hbx_oracle'
+export START_DATE
+
+##cleanup files
+rm -f ${GLUEDB_DIRECTORY}/todays_data.zip
+rm -f ${GLUEDB_DIRECTORY}/db/data/all_json.csv
+rm -f ${PARSER_DIRECTORY}/*.csv
+rm -f ${UPDATER_DIRECTORY}/*.csv
+
+cat << EOH >  /edidb/gateway_transmissions.sh
+unset BUNDLE_APP_CONFIG
+unset BUNDLE_BIN
+unset BUNDLE_PATH
+unset BUNDLER_VERSION
+unset GEM_HOME
+unset RUBYGEMS_VERSION
+PATH=/edidb/jruby-1.7.27/bin:$PATH
+GEM_PATH=/edidb/jruby-1.7.27/lib/ruby/gems/shared
+cd ${UPDATER_DIRECTORY}
+padrino r scripts/gateway_transmissions.rb --start $START_DATE
+EOH
+
+chmod 744 /edidb/gateway_transmissions.sh
+
+set +e
+batch_handler=$( kubectl get pods | grep edidb-glue-batch | grep Running )
+set -e
+if [ -z "$batch_handler" ]; then
+  kubectl patch cronjobs edidb-glue-batch -p "{\"spec\" : {\"suspend\" : true }}"
+  curl -X POST -H "Content-type: application/json; charset=utf-8" -H "Authorization: Bearer ${SLACK_TOKEN}" --data '{"channel": "#'$SLACK_CHANNEL'", "username": "EDI Database Bot", "text": "'\`' ### GlueDB Update Started ### '\`'", "icon_emoji": ":gear:"}' https://slack.com/api/chat.postMessage
+else
+  exit 5
+fi
+
+
+## bring down the listeners
+echo "bringing down listeners: "$(date)
+kubectl scale --replicas=0  deployment/edidb-enrollment-validator deployment/edidb-broker-updated-listener \
+                            deployment/edidb-policy-id-list-listener deployment/edidb-enrollment-event-listener \
+                            deployment/edidb-enrollment-event-handler 
+                            #deployment/edidb-enrollment-event-batch-processor
+sleep 60
+kubectl scale --replicas=0 deployment/edidb-enroll-query-result-handler
+sleep 120
+kubectl scale --replicas=0 deployment/edidb-employer-workers
+sleep 120
+kubectl scale --replicas=0 deployment/edidb-legacy-listeners
+sleep 180
+
+echo "copying prod databaase: "$(date)
+mongo --host $EDIDB_DB_HOST --authenticationDatabase 'admin' -u 'admin' -p $EDIDB_DB_PASSWORD < ~/scripts/prepare_dev.js
+
+sleep 10
+
+/edidb/gateway_transmissions.sh
+cp ${UPDATER_DIRECTORY}/b2b_edi.csv ${PARSER_DIRECTORY}
+cat ${PARSER_DIRECTORY}/b2b_edi.csv | ${PARSER_DIRECTORY}/dist/build/InterchangeTest/InterchangeTest > ${PARSER_DIRECTORY}/all_json.csv
+mkdir -p ${GLUEDB_DIRECTORY}/db/data
+cp ${PARSER_DIRECTORY}/all_json.csv ${GLUEDB_DIRECTORY}/db/data/
+
+cd ${GLUEDB_DIRECTORY}
+#echo -e '\ngem "rubycritic"' >> Gemfile
+#bundle install
+RAILS_ENV=development bundle exec rake edi:import:all
+RAILS_ENV=development rails r script/queries/set_authority_members.rb
+#head -n -1 Gemfile > Gemfile.tmp
+#mv Gemfile.tmp Gemfile
+
+echo "updating prod database: "$(date)
+update=`mongo --host $EDIDB_DB_HOST --authenticationDatabase 'admin' -u 'admin' -p $EDIDB_DB_PASSWORD < ~/scripts/prepare_prod.js`
+echo $update
+update=$(echo -n ${update#*"db ${EDIDB_DB_NAME}_dev"})
+update=$(echo -n ${update%bye*})
+update=$(echo -n ${update#*clone})
+update_status=`echo $update | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["'ok'"]'`
+
+sleep 60
+
+if [ "$update_status" -eq 1 ]; then
+
+  #curl -X POST -H "Content-type: application/json; charset=utf-8" -H "Authorization: Bearer ${SLACK_TOKEN}" --data '{"channel": "#'$SLACK_CHANNEL'", "username": "EDI Database Bot", "text": "'\`' ### GlueDB Update Completed :: Running Reports Before Starting Listeners ### '\`'", "icon_emoji": ":gear:"}' https://slack.com/api/chat.postMessage
+
+  #cp /etc/reports/glue_enrollment_report.sh /edidb/glue_enrollment_report.sh && chmod 744 /edidb/glue_enrollment_report.sh
+  #cp /etc/reports/glue_enrollment_report.json.template /edidb/glue_enrollment_report.json.template
+  #/edidb/glue_enrollment_report.sh > glue_enrollment_report.log
+  #tail -30 glue_enrollment_report.log
+
+  #cp /etc/reports/policies_missing_transmissions.sh /edidb/policies_missing_transmissions.sh && chmod 744 /edidb/policies_missing_transmissions.sh
+  #cp /etc/reports/policies_missing_transmissions.json.template /edidb/policies_missing_transmissions.json.template
+  #/edidb/policies_missing_transmissions.sh > policies_missing_transmissions.log
+  #tail -10 policies_missing_transmissions.log
+
+  kubectl scale --replicas=1 deployment/edidb-legacy-listeners
+  messages=1
+  while [ $messages -gt 0 ]
+  do
+    sleep 120
+    messages=$( curl --user $RABBITMQ_USER:$RABBITMQ_PASSWORD $RABBITMQ_CURL_URL/api/queues/%2F/$HBX_ID.$ENV_NAME.q.glue.individual_updated_listener | jq .messages | tail -1 )
+  done
+  kubectl scale --replicas=1 deployment/edidb-employer-workers
+  sleep 120
+  kubectl scale --replicas=2 deployment/edidb-enroll-query-result-handler
+  sleep 120
+  kubectl scale --replicas=2 deployment/edidb-enrollment-validator deployment/edidb-broker-updated-listener \
+                             deployment/edidb-policy-id-list-listener deployment/edidb-enrollment-event-listener \
+                             deployment/edidb-enrollment-event-handler \
+                             deployment/edidb-enrollment-event-batch-processor
+  sleep 120
+  kubectl patch cronjobs edidb-glue-batch -p "{\"spec\" : {\"suspend\" : false }}"
+  kubectl rollout restart deployment edidb-$ENV_NAME
+else
+  exit 1
+fi
+
+EOL
+
+chmod +x script.sh
+set +e
+./script.sh
+update_status=$?
+set -e
+sleep 120
+
+curlTestCmd="curl -sLk -w "%{http_code}" -o /dev/null ${EDIDB_CURL_URL}/accounts/sign_in"
+curlTest=`eval $curlTestCmd`
+
+if [ "$update_status" -eq 0 ]
+then
+  if [ "$curlTest" == "200" ]
+  then
+    curl -X POST -H "Content-type: application/json; charset=utf-8" -H "Authorization: Bearer ${SLACK_TOKEN}" --data '{"channel": "#'$SLACK_CHANNEL'", "username": "EDI Database Bot", "text": "'\`' ### GlueDB Update Completed :: Listeners Are Up ### '\`'", "icon_emoji": ":gear:"}' https://slack.com/api/chat.postMessage
+    send_sms_notification Success
+    exit 0
+  else
+    curl -X POST -H "Content-type: application/json; charset=utf-8" -H "Authorization: Bearer ${SLACK_TOKEN}" --data '{"channel": "#'$SLACK_CHANNEL'", "username": "EDI Database Bot", "text": "'\`' ### GlueDB Update Completed :: But Restart Failed ### '\`'", "icon_emoji": ":gear:"}' https://slack.com/api/chat.postMessage
+    send_sms_notification "Restart Failed"
+    exit 1
+  fi
+elif [ "$update_status" -eq 5 ]
+then
+  curl -X POST -H "Content-type: application/json; charset=utf-8" -H "Authorization: Bearer ${SLACK_TOKEN}" --data '{"channel": "#'$SLACK_CHANNEL'", "username": "EDI Database Bot", "text": "<!channel> '\`' ### GlueDB Update Did Not Start -- Batch Handler Is Running ### '\`'", "icon_emoji": ":gear:"}' https://slack.com/api/chat.postMessage
+  send_sms_notification "Did Not Start" "The batch handler is running!"
+  exit 1
+else
+  curl -X POST -H "Content-type: application/json; charset=utf-8" -H "Authorization: Bearer ${SLACK_TOKEN}" --data '{"channel": "#'$SLACK_CHANNEL'", "username": "EDI Database Bot", "text": "<!channel> '\`' ### GlueDB Update Failed ### '\`'", "icon_emoji": ":gear:"}' https://slack.com/api/chat.postMessage
+  send_sms_notification Failed "Please check GlueDB Update job in ${ENV_NAME}"
+  exit 1
+fi

.docker/config/prepare_dev.js

@@ -0,0 +1,8 @@
+use DB_NAME_dev;
+db.dropDatabase();
+sleep(10000);
+use DB_NAME_dev;
+db.dropDatabase();
+sleep(10000);
+use DB_NAME;
+db.copyDatabase('DB_NAME', 'DB_NAME_dev');

.docker/config/prepare_prod.js

@@ -0,0 +1,12 @@
+use  DB_NAME;
+db.dropDatabase();
+sleep(10000);
+use  DB_NAME;
+db.dropDatabase();
+sleep(20000);
+use  DB_NAME;
+db.dropDatabase();
+sleep(20000);
+use  DB_NAME_dev;
+sleep(10000);
+db.copyDatabase('DB_NAME_dev', 'DB_NAME');

.docker/config/update_variables.sh

@@ -0,0 +1,2 @@
+sed -i "s|B2B_URI|$B2B_URI|g"  /edidb/hbx_oracle/config/database.rb
+sed -i "s|DB_NAME|$EDIDB_DB_NAME|g" /edidb/scripts/prepare*