From e182fb06a612073785d5ede895e695e6447321f2 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Fri, 20 Sep 2019 13:02:37 +0300 Subject: [PATCH 01/35] Added s3 endpoint to postgres-restore-s3 --- postgres-restore-s3/Dockerfile | 1 + postgres-restore-s3/restore.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/postgres-restore-s3/Dockerfile b/postgres-restore-s3/Dockerfile index 501b66d..a0f8ce3 100644 --- a/postgres-restore-s3/Dockerfile +++ b/postgres-restore-s3/Dockerfile @@ -14,6 +14,7 @@ ENV S3_SECRET_ACCESS_KEY **None** ENV S3_BUCKET **None** ENV S3_REGION us-west-1 ENV S3_PATH 'backup' +ENV S3_ENDPOINT **None** ENV DROP_PUBLIC 'no' ADD restore.sh restore.sh diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 495c237..0a8adef 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -43,6 +43,12 @@ if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then exit 1 fi +if [ "${S3_ENDPOINT}" == "**None**" ]; then + AWS_ARGS="" +else + AWS_ARGS="--endpoint-url ${S3_ENDPOINT}" +fi + # env vars needed for aws tools export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY @@ -53,11 +59,11 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER" echo "Finding latest backup" -LATEST_BACKUP=$(aws s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }') +LATEST_BACKUP=$(aws $AWS_ARGS s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }') echo "Fetching ${LATEST_BACKUP} from S3" -aws s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz +aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz gzip -d dump.sql.gz if [ "${DROP_PUBLIC}" == "yes" ]; then From b43a49877229b3b2874525fb66356eb3fa34088d Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Fri, 20 Sep 2019 13:40:43 +0300 Subject: [PATCH 02/35] Added recreating existing tables --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 0a8adef..d7bc863 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE < dump.sql +psql $POSTGRES_HOST_OPTS -C -c --if-exists -d $POSTGRES_DATABASE < dump.sql echo "Restore complete" From 2b3a4a0da3d4cc46b7833774e74d86b58f3d5511 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Fri, 20 Sep 2019 13:47:59 +0300 Subject: [PATCH 03/35] Added recreating existing tables --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index d7bc863..ba483e3 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS -C -c --if-exists -d $POSTGRES_DATABASE < dump.sql +psql $POSTGRES_HOST_OPTS --create -c --if-exists -d $POSTGRES_DATABASE < dump.sql echo "Restore complete" From 4b2d13bc6ddaef07459a73a5e06a38cc5ae00ea3 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Fri, 20 Sep 2019 14:39:15 +0300 Subject: [PATCH 04/35] Return back to first variant --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index ba483e3..2fc138e 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS --create -c --if-exists -d $POSTGRES_DATABASE < dump.sql +psql $POSTGRES_HOST_OPTS -c -d $POSTGRES_DATABASE < dump.sql echo "Restore complete" From 2f86476c603bcb64da926159fc792f172114f2fc Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Fri, 20 Sep 2019 15:19:05 +0300 Subject: [PATCH 05/35] Changed psql to pg_restore --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 2fc138e..a66d14f 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS -c -d $POSTGRES_DATABASE < dump.sql +pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists -no-owner -v -d $POSTGRES_DATABASE < dump.sql echo "Restore complete" From e04083c6eb68dc62b6354831d9d7313e6d86856f Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Fri, 20 Sep 2019 15:27:28 +0300 Subject: [PATCH 06/35] Changed psql to pg_restore --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index a66d14f..57a793e 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists -no-owner -v -d $POSTGRES_DATABASE < dump.sql +pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists -no-owner -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From ff32d6d016454910ba4f6ebb1fba1fa4c11cd4be Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Fri, 20 Sep 2019 15:44:11 +0300 Subject: [PATCH 07/35] Changed pg_dump format for pg_restore --- postgres-backup-s3/backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-backup-s3/backup.sh b/postgres-backup-s3/backup.sh index 6e5a7f0..dca744f 100644 --- a/postgres-backup-s3/backup.sh +++ b/postgres-backup-s3/backup.sh @@ -59,7 +59,7 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTG echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..." -pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | gzip > dump.sql.gz +pg_dump $POSTGRES_HOST_OPTS -Ft $POSTGRES_DATABASE | gzip > dump.sql.gz echo "Uploading dump to $S3_BUCKET" From b7118f48cb1bbf74134289dfb2354a4fce0473d5 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Mon, 23 Sep 2019 12:42:55 +0300 Subject: [PATCH 08/35] Fixed pg_restore flag --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 57a793e..f6df25d 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists -no-owner -v -d $POSTGRES_DATABASE dump.sql +pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists --no-owner -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From 3348023d8fb6b6c6e5b07174ab105ab87a630ac5 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Wed, 25 Sep 2019 17:37:34 +0300 Subject: [PATCH 09/35] Added multithreading --- postgres-backup-s3/backup.sh | 2 +- postgres-restore-s3/restore.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres-backup-s3/backup.sh b/postgres-backup-s3/backup.sh index dca744f..db84fe1 100644 --- a/postgres-backup-s3/backup.sh +++ b/postgres-backup-s3/backup.sh @@ -59,7 +59,7 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTG echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..." -pg_dump $POSTGRES_HOST_OPTS -Ft $POSTGRES_DATABASE | gzip > dump.sql.gz +pg_dump $POSTGRES_HOST_OPTS -Fc $POSTGRES_DATABASE | gzip > dump.sql.gz echo "Uploading dump to $S3_BUCKET" diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index f6df25d..58b04c2 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists --no-owner -v -d $POSTGRES_DATABASE dump.sql +pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists --no-owner -j 4 -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From 0ca035d7ef00fb997a5b96cf17556a2e50d45950 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Thu, 26 Sep 2019 11:03:17 +0300 Subject: [PATCH 10/35] Added jobs count --- postgres-restore-s3/Dockerfile | 1 + postgres-restore-s3/restore.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/postgres-restore-s3/Dockerfile b/postgres-restore-s3/Dockerfile index a0f8ce3..8bac82f 100644 --- a/postgres-restore-s3/Dockerfile +++ b/postgres-restore-s3/Dockerfile @@ -9,6 +9,7 @@ ENV POSTGRES_HOST **None** ENV POSTGRES_PORT 5432 ENV POSTGRES_USER **None** ENV POSTGRES_PASSWORD **None** +ENV RESTORE_JOBS 4 ENV S3_ACCESS_KEY_ID **None** ENV S3_SECRET_ACCESS_KEY **None** ENV S3_BUCKET **None** diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 58b04c2..a569d2c 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists --no-owner -j 4 -v -d $POSTGRES_DATABASE dump.sql +pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From 696096917abb130e2b311307ff1171e1c35bcba3 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 22 Oct 2019 10:30:08 +0300 Subject: [PATCH 11/35] Added DB recreation --- postgres-restore-s3/restore.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index a569d2c..9770912 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,8 @@ fi echo "Restoring ${LATEST_BACKUP}" -pg_restore $POSTGRES_HOST_OPTS -C -c --if-exists --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql +psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILIGIES ON '$POSTGRES_DATABASE' TO '$POSTGRES_USER';" +pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From f4bc87a72c4698acdb46410eb2401676c7f1c9dd Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 22 Oct 2019 10:45:33 +0300 Subject: [PATCH 12/35] Added DB recreation --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 9770912..1423f2e 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILIGIES ON '$POSTGRES_DATABASE' TO '$POSTGRES_USER';" +psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGIES ON '$POSTGRES_DATABASE' TO '$POSTGRES_USER';" pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From 983b75b576f816d1ebccad7b08b896c0132c4354 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 22 Oct 2019 11:05:19 +0300 Subject: [PATCH 13/35] Fixed DB recreation --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 1423f2e..bb8e0c2 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGIES ON '$POSTGRES_DATABASE' TO '$POSTGRES_USER';" +psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGES ON '$POSTGRES_DATABASE' TO '$POSTGRES_USER';" pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From f4358e59901ec0ecfce213b00a2c363cd9b4f7d7 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 22 Oct 2019 11:14:31 +0300 Subject: [PATCH 14/35] Fixed DB recreation --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index bb8e0c2..e698fff 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,7 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGES ON '$POSTGRES_DATABASE' TO '$POSTGRES_USER';" +psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGES ON $POSTGRES_DATABASE TO $POSTGRES_USER;" pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From e933a38cf358a0b30e4a88547552351fb1b1427c Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 22 Oct 2019 11:22:37 +0300 Subject: [PATCH 15/35] Fixed drop database --- postgres-restore-s3/restore.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index e698fff..b15b87e 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -73,7 +73,9 @@ fi echo "Restoring ${LATEST_BACKUP}" -psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid(); DROP DATABASE IF EXISTS $POSTGRES_DATABASE; CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGES ON $POSTGRES_DATABASE TO $POSTGRES_USER;" +psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid();" +psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS $POSTGRES_DATABASE;" +psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGES ON $POSTGRES_DATABASE TO $POSTGRES_USER;" pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From 8dfc17ffd295f7bd1e5e14d8d71fddc0946bba39 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 22 Oct 2019 11:36:43 +0300 Subject: [PATCH 16/35] Fixed drop database --- postgres-restore-s3/restore.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index b15b87e..68f17ff 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -75,7 +75,8 @@ echo "Restoring ${LATEST_BACKUP}" psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid();" psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS $POSTGRES_DATABASE;" -psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE $POSTGRES_DATABASE; GRANT ALL PRIVILEGES ON $POSTGRES_DATABASE TO $POSTGRES_USER;" +psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE $POSTGRES_DATABASE;" +psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON $POSTGRES_DATABASE TO $POSTGRES_USER;" pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From cc2d05d33157d66b9dde3719c55da472e088c4b9 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 22 Oct 2019 11:56:53 +0300 Subject: [PATCH 17/35] Fixed drop database --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 68f17ff..0cc7ed7 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -76,7 +76,7 @@ echo "Restoring ${LATEST_BACKUP}" psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid();" psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS $POSTGRES_DATABASE;" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE $POSTGRES_DATABASE;" -psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON $POSTGRES_DATABASE TO $POSTGRES_USER;" +psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE $POSTGRES_DATABASE TO $POSTGRES_USER;" pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql echo "Restore complete" From 34326102a95fcdf9cc463df3ce1d67c5ee1b02d7 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Tue, 14 Jan 2020 14:25:24 +0300 Subject: [PATCH 18/35] Added alerting to telegram --- postgres-backup-s3/Dockerfile | 4 ++++ postgres-backup-s3/backup.sh | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/postgres-backup-s3/Dockerfile b/postgres-backup-s3/Dockerfile index af5740d..9ce8181 100644 --- a/postgres-backup-s3/Dockerfile +++ b/postgres-backup-s3/Dockerfile @@ -18,6 +18,10 @@ ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** ENV S3_S3V4 no ENV SCHEDULE **None** +# Telegram alerting +ENV BOT_ID **None** +ENV CHAT_ID **None** +ENV MSG '' ADD run.sh run.sh ADD backup.sh backup.sh diff --git a/postgres-backup-s3/backup.sh b/postgres-backup-s3/backup.sh index db84fe1..84a6bd4 100644 --- a/postgres-backup-s3/backup.sh +++ b/postgres-backup-s3/backup.sh @@ -63,6 +63,10 @@ pg_dump $POSTGRES_HOST_OPTS -Fc $POSTGRES_DATABASE | gzip > dump.sql.gz echo "Uploading dump to $S3_BUCKET" -cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2 +if [ "${BOT_ID}" == "**None**" ] || [ "$CHAT_ID" == "**None**" ] || [ -z "$MSG" ] ; then + cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2 +else + cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || (curl "https://api.telegram.org/$BOT_ID/sendMessage?chat_id=$CHAT_ID&parse_mode=html&text=$MSG" && exit 2) || exit 2 +fi echo "SQL backup uploaded successfully" From 745d70fcd46a58b320d7bfd31a8709c5facac6aa Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Thu, 16 Jan 2020 10:29:17 +0300 Subject: [PATCH 19/35] Added patched nginx-proxy --- nginx-proxy/Dockerfile | 11 +++++++++++ nginx-proxy/README.md | 3 +++ nginx-proxy/docker-compose.yml | 14 ++++++++++++++ nginx-proxy/vhost/whoami.local | 0 nginx-proxy/vhost/whoami.local_location | 4 ++++ 5 files changed, 32 insertions(+) create mode 100644 nginx-proxy/Dockerfile create mode 100644 nginx-proxy/README.md create mode 100644 nginx-proxy/docker-compose.yml create mode 100644 nginx-proxy/vhost/whoami.local create mode 100644 nginx-proxy/vhost/whoami.local_location diff --git a/nginx-proxy/Dockerfile b/nginx-proxy/Dockerfile new file mode 100644 index 0000000..dee7411 --- /dev/null +++ b/nginx-proxy/Dockerfile @@ -0,0 +1,11 @@ +FROM gcc as builder +RUN apt update && apt install make libpcre3-dev zlib1g-dev unzip git +WORKDIR /build +RUN wget -qO - http://nginx.org/download/nginx-1.17.5.tar.gz | tar zxfv - +RUN git clone https://github.com/openresty/headers-more-nginx-module +RUN cd nginx-1.17.5 && ./configure --prefix=/opt/nginx --with-compat --add-dynamic-module=/build/headers-more-nginx-module && make modules +RUN ls /build/nginx-1.17.5/objs/ + +FROM jwilder/nginx-proxy:alpine +COPY --from=builder /build/nginx-1.17.5/objs/ngx_http_headers_more_filter_module.so /usr/lib/nginx/modules/ +RUN sed -i "/user nginx;/aload_module modules/ngx_http_headers_more_filter_module.so;" /etc/nginx/nginx.conf diff --git a/nginx-proxy/README.md b/nginx-proxy/README.md new file mode 100644 index 0000000..4b003f9 --- /dev/null +++ b/nginx-proxy/README.md @@ -0,0 +1,3 @@ +This is a version of jwilder/nginx-proxy with 'headers more'-module installed. + +Example of usage is in `docker-compose.yml` and `/vhost/` diff --git a/nginx-proxy/docker-compose.yml b/nginx-proxy/docker-compose.yml new file mode 100644 index 0000000..1779315 --- /dev/null +++ b/nginx-proxy/docker-compose.yml @@ -0,0 +1,14 @@ +version: "2.0" +services: + nginx-proxy: + image: nginx-proxy:1.17-headers-more + ports: + - "80:80" + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + - ./vhost:/etc/nginx/vhost.d:ro + + whoami: + image: jwilder/whoami + environment: + - VIRTUAL_HOST=whoami.local diff --git a/nginx-proxy/vhost/whoami.local b/nginx-proxy/vhost/whoami.local new file mode 100644 index 0000000..e69de29 diff --git a/nginx-proxy/vhost/whoami.local_location b/nginx-proxy/vhost/whoami.local_location new file mode 100644 index 0000000..2ad7111 --- /dev/null +++ b/nginx-proxy/vhost/whoami.local_location @@ -0,0 +1,4 @@ + more_set_headers 'Access-Control-Allow-Origin: http://whoami.local'; + more_set_headers 'Access-Control-Allow-Methods: GET'; + more_set_headers 'Access-Control-Allow-Headers: User-Agent,Keep-Alive,Content-Type'; + more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range'; From 6afe55134a9db5f9abaf5fc64dca72308a71e72b Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Mon, 13 Apr 2020 18:25:58 +0300 Subject: [PATCH 20/35] Fixed quotation --- postgres-restore-s3/restore.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 0cc7ed7..2a5e473 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -68,16 +68,16 @@ gzip -d dump.sql.gz if [ "${DROP_PUBLIC}" == "yes" ]; then echo "Recreating the public schema" - psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE -c "drop schema public cascade; create schema public;" + psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "drop schema public cascade; create schema public;" fi echo "Restoring ${LATEST_BACKUP}" psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid();" -psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS $POSTGRES_DATABASE;" -psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE $POSTGRES_DATABASE;" -psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE $POSTGRES_DATABASE TO $POSTGRES_USER;" -pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d $POSTGRES_DATABASE dump.sql +psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";" +psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" +psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" +pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From 8f8c52908fced9fdc11b05dd353548967cc41c0e Mon Sep 17 00:00:00 2001 From: aido93 Date: Thu, 11 Jun 2020 15:46:19 +0300 Subject: [PATCH 21/35] Update restore.sh --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 2a5e473..501fdfa 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -77,7 +77,7 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_act psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" -pg_restore $POSTGRES_HOST_OPTS --no-owner -j $RESTORE_JOBS -v -d "$POSTGRES_DATABASE" dump.sql +pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From 935fd938856f4d0ca4be4fe7c3bdbf59435d4a18 Mon Sep 17 00:00:00 2001 From: aido93 Date: Thu, 11 Jun 2020 17:55:32 +0300 Subject: [PATCH 22/35] Update restore.sh --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 501fdfa..f09369f 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -77,7 +77,7 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_act psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" -pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql +pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From bbce20596ac0e2b2d08d7c255b5b0dc72c4ba59e Mon Sep 17 00:00:00 2001 From: aido93 Date: Thu, 11 Jun 2020 18:08:28 +0300 Subject: [PATCH 23/35] Update restore.sh --- postgres-restore-s3/restore.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index f09369f..f6d329a 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -1,6 +1,7 @@ #! /bin/sh set -e +set -x set -o pipefail if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then @@ -71,7 +72,7 @@ if [ "${DROP_PUBLIC}" == "yes" ]; then psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "drop schema public cascade; create schema public;" fi -echo "Restoring ${LATEST_BACKUP}" +echo "Restoring ${LATEST_BACKUP} to ${POSTGRES_DATABASE} on ${POSTGRES_HOST}:${POSTGRES_PORT} with user ${POSTGRES_USER}" psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid();" psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";" From 9ade58b56ad2784a6ad87b3e84a96927ebe90f45 Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 15 Jun 2020 11:53:34 +0300 Subject: [PATCH 24/35] Update restore.sh --- postgres-restore-s3/restore.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index f6d329a..5b3af10 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -74,10 +74,13 @@ fi echo "Restoring ${LATEST_BACKUP} to ${POSTGRES_DATABASE} on ${POSTGRES_HOST}:${POSTGRES_PORT} with user ${POSTGRES_USER}" +set +e psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid();" psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" + +set -e pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From 0431db8e1d109f22c301b82871bbe7d07b183426 Mon Sep 17 00:00:00 2001 From: Igor Diakonov Date: Mon, 29 Jun 2020 12:55:56 +0300 Subject: [PATCH 25/35] Added citext --- postgres-restore-s3/restore.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 5b3af10..93d5389 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -79,6 +79,7 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_act psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" +psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS citext;" set -e pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql From e619558218471f3ea038afc9bcb9ebdda3cf873b Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 15:20:17 +0300 Subject: [PATCH 26/35] Update restore.sh added plpgsql --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 93d5389..b4570f3 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -80,7 +80,7 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DAT psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS citext;" - +psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" set -e pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql From f1ed478b13016433c3cffb26353fac48ec9e1d1f Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 15:37:11 +0300 Subject: [PATCH 27/35] Update restore.sh --- postgres-restore-s3/restore.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index b4570f3..d44c159 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -80,7 +80,10 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DAT psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS citext;" +psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA public;" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" +psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION plpgsql SET SCHEMA public;" + set -e pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql From 3baac7d5f13f71beca4c8fb994d0bcd1866f78cf Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 15:59:31 +0300 Subject: [PATCH 28/35] Update restore.sh --- postgres-restore-s3/restore.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index d44c159..f135f66 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -82,7 +82,6 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POS psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS citext;" psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA public;" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" -psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION plpgsql SET SCHEMA public;" set -e pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql From d18b234dbe21aeda70197b2d225512fc3d3fad17 Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 17:06:38 +0300 Subject: [PATCH 29/35] Update restore.sh --- postgres-restore-s3/restore.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index f135f66..6ba06d1 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -84,6 +84,7 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA publi psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" set -e +sed -i -e 's/public.citext/citext/g' dump.sql pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From dbb3714373e1d0e70191d65dafaf2479bd9bfc90 Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 17:14:53 +0300 Subject: [PATCH 30/35] Update restore.sh --- postgres-restore-s3/restore.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 6ba06d1..4c956f2 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -84,8 +84,8 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA publi psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" set -e -sed -i -e 's/public.citext/citext/g' dump.sql -pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql +sed -i 's/public.citext/citext/g' dump.sql +pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From 0db48a2776071bc7f2b8cf647c1f3fc0708e1f9c Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 17:25:10 +0300 Subject: [PATCH 31/35] Update restore.sh --- postgres-restore-s3/restore.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 4c956f2..e3767a4 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -84,7 +84,6 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA publi psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" set -e -sed -i 's/public.citext/citext/g' dump.sql pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From cc63dec4c78238f51f30f43d024eab7d412d4b83 Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 17:47:25 +0300 Subject: [PATCH 32/35] Update restore.sh --- postgres-restore-s3/restore.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index e3767a4..0fb852c 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -84,6 +84,7 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA publi psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" set -e +pg_restore -v pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From c1280384b288b0341f6f664439458dc5129c441e Mon Sep 17 00:00:00 2001 From: aido93 Date: Mon, 29 Jun 2020 18:02:04 +0300 Subject: [PATCH 33/35] Update restore.sh --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 0fb852c..8fe0e1a 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -84,7 +84,7 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA publi psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" set -e -pg_restore -v +pg_restore -V pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete" From 09a43792a1012747af0f912cc247c6dd332a4a94 Mon Sep 17 00:00:00 2001 From: aido93 Date: Tue, 30 Jun 2020 09:27:03 +0300 Subject: [PATCH 34/35] Update restore.sh --- postgres-restore-s3/restore.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index 8fe0e1a..fde2031 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -79,11 +79,11 @@ psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_act psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";" psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;" -psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS citext;" -psql $POSTGRES_HOST_OPTS -d postgres -c "ALTER EXTENSION citext SET SCHEMA public;" -psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" set -e +psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "CREATE EXTENSION IF NOT EXISTS citext;" +psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "ALTER EXTENSION citext SET SCHEMA public;" +psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" pg_restore -V pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public -v -d "$POSTGRES_DATABASE" dump.sql From 61afbd51fa94841b7c28ae35db3de18f041bf6d7 Mon Sep 17 00:00:00 2001 From: aido93 Date: Tue, 30 Jun 2020 09:41:45 +0300 Subject: [PATCH 35/35] Update restore.sh --- postgres-restore-s3/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-restore-s3/restore.sh b/postgres-restore-s3/restore.sh index fde2031..583d1af 100644 --- a/postgres-restore-s3/restore.sh +++ b/postgres-restore-s3/restore.sh @@ -85,7 +85,7 @@ psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "CREATE EXTENSION IF NOT EXI psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "ALTER EXTENSION citext SET SCHEMA public;" psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "CREATE EXTENSION IF NOT EXISTS plpgsql;" pg_restore -V -pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public -v -d "$POSTGRES_DATABASE" dump.sql +pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql echo "Restore complete"