From 492fafc1497310c5067939a4cbf7ceab975523d9 Mon Sep 17 00:00:00 2001
From: maftie <scottmichcohen@yahoo.com>
Date: Fri, 3 Jan 2020 10:54:23 -0800
Subject: [PATCH 1/2] modify setup script

---
 AWS/AWS_Setup.sh | 44 +++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/AWS/AWS_Setup.sh b/AWS/AWS_Setup.sh
index 1c6ccb5..136470e 100644
--- a/AWS/AWS_Setup.sh
+++ b/AWS/AWS_Setup.sh
@@ -11,6 +11,8 @@ set -e
 curl -fsSL https://get.docker.com -o get-docker.sh
 sh get-docker.sh
 docker pull "$IMAGE"
+usermod -aG docker ubuntu
+newgrp docker
 set +e
 )
 
@@ -31,7 +33,6 @@ AWS_SECRET_ACCESS_KEY=<secret key>
 EOF
 
 #allow for timestamps on STDERR
-
 (
 set -e
 mkdir /home/ubuntu/utils
@@ -42,57 +43,61 @@ while read -r line; do
    echo "[\$(date +%Y-%m-%d\ %H:%M:%S)] \$line" >> /home/ubuntu/logs/errors.log
 done
 EOF
-chmod 755 /home/ubuntu/utils/timestamp_errors.sh
-chown -R ubuntu /home/ubuntu/utils
+chmod -R 777 ubuntu /home/ubuntu/utils
 set +e
 )
 
 #Create Shell Scripts
+(
+set -e
 mkdir /home/ubuntu/logs
 touch /home/ubuntu/logs/errors.log
 touch /home/ubuntu/logs/output.log
-chown -R ubuntu /home/ubuntu/logs 
+chmod -R 777 ubuntu /home/ubuntu/logs 
+set +e
+)
+
+mkdir /home/ubuntu/scripts
 
 (
 set -e
-cat > /home/ubuntu/daily_at_2.sh <<EOF
+cat > /home/ubuntu/scripts/daily_at_2.sh <<EOF
 #!/bin/bash
-sudo docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/DAILY_AT_2AM.sh
+docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/DAILY_AT_2AM.sh
 EOF
-chmod 755 daily_at_2.sh
 set +e
 )
 
 (
 set -e
-cat > /home/ubuntu/every_minute.sh <<EOF
+cat > /home/ubuntu/scripts/every_minute.sh <<EOF
 #!/bin/bash
-sudo docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/EVERY_MINUTE.sh
+docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/EVERY_MINUTE.sh
 EOF
-chmod 755 every_minute.sh
 set +e
 )
 
 #run the EVERY_15_MIN tasks every hour in dev to reduce the amount of writes to S3 (free tier caps at 2k per month) 
 (
 set -e
-cat > /home/ubuntu/every_hour.sh <<EOF
+cat > /home/ubuntu/scripts/every_hour.sh <<EOF
 #!/bin/bash
-sudo docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/EVERY_15_MINS.sh
+docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/EVERY_15_MINS.sh
 EOF
-chmod 755 every_hour.sh
 set +e
 )
 
+chmod -R 777 /home/ubuntu/scripts
+
 #Configure cron tasks
 
 (
 set -e
 cat > every_minute <<EOF
 CRON_TZ=UTC
-* * * * * root /home/ubuntu/every_minute.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
+SHELL=/bin/bash
+* * * * * root /home/ubuntu/scripts/every_minute.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
 EOF
-chmod 644 every_minute
 mv every_minute /etc/cron.d
 set +e
 )
@@ -101,9 +106,9 @@ set +e
 set -e
 cat > every_hour <<EOF
 CRON_TZ=UTC
+SHELL=/bin/bash
 0 * * * * root /home/ubuntu/every_hour.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log 
 EOF
-chmod 644 every_hour
 mv every_hour /etc/cron.d
 set +e
 )
@@ -112,12 +117,13 @@ set +e
 set -e
 cat > every_day <<EOF
 CRON_TZ=UTC
-* 11 * * * root rm -rf ~/data/tmp && /home/ubuntu/daily_at_2.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
+SHELL=/bin/bash
+* 10 * * * root rm -rf ~/data/tmp && /home/ubuntu/daily_at_2.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
 EOF
-chmod 644 every_day
 mv every_day /etc/cron.d
 set +e
 )
+chmod -R 755 /etc/cron.d
 
 #run the daily script once on launch to obtain schedule for current day
-bash /home/ubuntu/daily_at_2.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
+bash /home/ubuntu/scripts/daily_at_2.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
\ No newline at end of file

From 379042773da67f3f1434b82a4f101050cb466498 Mon Sep 17 00:00:00 2001
From: root <root@Scott-PC.localdomain>
Date: Fri, 10 Jan 2020 21:25:44 -0800
Subject: [PATCH 2/2] Updated AWS launch script

---
 AWS/AWS_Setup.sh | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/AWS/AWS_Setup.sh b/AWS/AWS_Setup.sh
index 136470e..e3443d8 100644
--- a/AWS/AWS_Setup.sh
+++ b/AWS/AWS_Setup.sh
@@ -10,9 +10,10 @@ IMAGE="<image name>"
 set -e
 curl -fsSL https://get.docker.com -o get-docker.sh
 sh get-docker.sh
-docker pull "$IMAGE"
-usermod -aG docker ubuntu
-newgrp docker
+docker pull $IMAGE
+#make docker invokable without requiring sudo
+sudo usermod -aG docker ubuntu
+sudo newgrp docker
 set +e
 )
 
@@ -20,7 +21,7 @@ set +e
 mkdir /home/ubuntu/docker_config && cat > /home/ubuntu/docker_config/.env <<EOF
 LOCAL_DATA=/app/data
 DATASTORE_NAME=S3
-DATASTORE_PATH=performance-tracker-test
+DATASTORE_PATH=<your bucket name here>
 TMP_DIR=/app/data/tmp
 METRO_LINES=801,802,803,804,805,806
 METRO_AGENCY=lametro-rail
@@ -61,9 +62,9 @@ mkdir /home/ubuntu/scripts
 
 (
 set -e
-cat > /home/ubuntu/scripts/daily_at_2.sh <<EOF
+cat > /home/ubuntu/scripts/every_six_hours.sh <<EOF
 #!/bin/bash
-docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/DAILY_AT_2AM.sh
+docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp $IMAGE bash ./tasks/EVERY_6_HOURS.sh
 EOF
 set +e
 )
@@ -72,7 +73,7 @@ set +e
 set -e
 cat > /home/ubuntu/scripts/every_minute.sh <<EOF
 #!/bin/bash
-docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/EVERY_MINUTE.sh
+docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp $IMAGE bash ./tasks/EVERY_MINUTE.sh
 EOF
 set +e
 )
@@ -82,7 +83,7 @@ set +e
 set -e
 cat > /home/ubuntu/scripts/every_hour.sh <<EOF
 #!/bin/bash
-docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp "$IMAGE" bash ./tasks/EVERY_15_MINS.sh
+docker run --rm --env-file /home/ubuntu/docker_config/.env -v /home/ubuntu/data/tmp:/app/data/tmp $IMAGE bash ./tasks/EVERY_15_MINS.sh
 EOF
 set +e
 )
@@ -104,10 +105,11 @@ set +e
 
 (
 set -e
+#change to every 15 min for prod
 cat > every_hour <<EOF
 CRON_TZ=UTC
 SHELL=/bin/bash
-0 * * * * root /home/ubuntu/every_hour.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log 
+0 * * * * root /home/ubuntu/scripts/every_hour.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log 
 EOF
 mv every_hour /etc/cron.d
 set +e
@@ -115,15 +117,15 @@ set +e
 
 (
 set -e
-cat > every_day <<EOF
+cat > every_six_hours <<EOF
 CRON_TZ=UTC
 SHELL=/bin/bash
-* 10 * * * root rm -rf ~/data/tmp && /home/ubuntu/daily_at_2.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
+0 5,11,17,23 * * * root rm -rf ~/data/tmp && /home/ubuntu/scripts/every_six_hours.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
 EOF
-mv every_day /etc/cron.d
+mv every_six_hours /etc/cron.d
 set +e
 )
 chmod -R 755 /etc/cron.d
 
-#run the daily script once on launch to obtain schedule for current day
-bash /home/ubuntu/scripts/daily_at_2.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
\ No newline at end of file
+#obtain the current schedule before cron actions fire
+bash /home/ubuntu/scripts/every_six_hours.sh 2> >(/home/ubuntu/utils/timestamp_errors.sh) >> /home/ubuntu/logs/output.log
\ No newline at end of file