From 592e0cd8938f8cb8b74714e267647ca0cb79dfd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Faurel?= <12185901+mfaurel@users.noreply.github.com> Date: Tue, 6 Dec 2022 14:40:43 +0100 Subject: [PATCH 1/9] Add files via upload --- rhodecode-backup.sh | 72 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 rhodecode-backup.sh diff --git a/rhodecode-backup.sh b/rhodecode-backup.sh new file mode 100644 index 0000000..6b44382 --- /dev/null +++ b/rhodecode-backup.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +############################################################################# +# Nom du script : rhodecode-backup.sh +# Auteur : E.RIEGEL (QM HENIX) +# Date de Création : 15/02/2022 +# Version : 0.0.1 +# Descritpion : Script permettant la sauvegarde de la BDD de RhodeCode et de sauvegarder les repos de l'ANS +# +# Historique des mises à jour : +#-----------+--------+-------------+------------------------------------------------------ +# Version | Date | Auteur | Description +#-----------+--------+-------------+------------------------------------------------------ +# 0.0.1 | 15/02/22 | E.RIEGEL | Initialisation du script +#-----------+--------+-------------+------------------------------------------------------ +############################################################################################### + + +# Configuration de base: datestamp e.g. YYYYMMDD +DATE=$(date +"%Y%m%d") + +# Dossier où sauvegarder les backups +BACKUP_DIR="/var/BACKUP/RHODECODE" + +# Commande NOMAD +NOMAD=/usr/local/bin/nomad + +#Repo PATH To BACKUP in the container +REPO_PATH=/root/ +#Archive Name of the backup repo directory +BACKUP_REPO_FILENAME="BACKUP_REPOS_RHODECODE_${DATE}.tar.gz" +#Name of the dump file (Bdd Rhodecode) +DUMP_FILENAME="BACKUP_RHODECODE_BDD_${DATE}.dump" + +# Nombre de jours à garder les dossiers (seront effacés après X jours) +RETENTION=5 + +# ---- NE RIEN MODIFIER SOUS CETTE LIGNE ------------------------------------------ +# +# Create a new directory into backup directory location for this date +mkdir -p $BACKUP_DIR/$DATE + +# Backup repos +echo "Starting backup repos..." + +$NOMAD exec -job rhodecode tar -cOzv -C $REPO_PATH my_dev_repos > $BACKUP_DIR/$DATE/$BACKUP_REPO_FILENAME +BACKUP_RESULT=$? +if [ $BACKUP_RESULT -gt 1 ] +then + echo "Repo backup failed with error code : ${BACKUP_RESULT}" + exit 1 +else + echo "Repo backup done" +fi + +# Dump rhodecode bdd +echo "starting rhodecode dump..." +$NOMAD exec -job rhodecode pg_dump -F c --dbname=postgresql://postgres:postgres@localhost/rhodecode > $BACKUP_DIR/$DATE/$DUMP_FILENAME + +DUMP_RESULT=$? +if [ $DUMP_RESULT -gt 0 ] +then + echo "Rhodecode dump failed with error code : ${DUMP_RESULT}" + exit 1 +else + echo "Rhodecode dump done" +fi + +# Remove files older than X days +find $BACKUP_DIR/* -mtime +$RETENTION -delete + +echo "Backup Rhodecode finished" From 82fa5e9bd0768d0d40392b66a6ba9898a3966f65 Mon Sep 17 00:00:00 2001 From: Yann Etrillard Date: Tue, 22 Aug 2023 17:44:27 +0200 Subject: [PATCH 2/9] Add Log-shipper --- rhodecode-app/rhodecode-community.nomad.tpl | 37 +++++++++++++++++- rhodecode-db/rhodecode-postgres.nomad.tpl | 33 ++++++++++++++++ .../rhodecode-syncldap.nomad.tpl | 35 +++++++++++++++++ rhodecode-tools/rhodecode-tools.nomad.tpl | 39 +++++++++++++++++-- 4 files changed, 139 insertions(+), 5 deletions(-) diff --git a/rhodecode-app/rhodecode-community.nomad.tpl b/rhodecode-app/rhodecode-community.nomad.tpl index 52ac6f6..0c603ce 100644 --- a/rhodecode-app/rhodecode-community.nomad.tpl +++ b/rhodecode-app/rhodecode-community.nomad.tpl @@ -696,6 +696,10 @@ EOT } task "rhodecode-celery" { driver = "docker" + + # log-shipper + leader = true + template { data = < Date: Thu, 31 Aug 2023 12:07:26 +0200 Subject: [PATCH 3/9] Modification de la task leader --- rhodecode-app/rhodecode-community.nomad.tpl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rhodecode-app/rhodecode-community.nomad.tpl b/rhodecode-app/rhodecode-community.nomad.tpl index 0c603ce..0db4e54 100644 --- a/rhodecode-app/rhodecode-community.nomad.tpl +++ b/rhodecode-app/rhodecode-community.nomad.tpl @@ -67,6 +67,10 @@ job "rhodecode-community" { } } task "rhodecode" { + + # log-shipper + leader = true + artifact { source = "${extensions_url}" destination = "local" @@ -696,10 +700,6 @@ EOT } task "rhodecode-celery" { driver = "docker" - - # log-shipper - leader = true - template { data = < Date: Mon, 25 Sep 2023 16:26:52 +0200 Subject: [PATCH 4/9] Modification du chemin de sauvegarde dans le script de backup --- rhodecode-backup.sh | 72 ---------------------------------- rhodecode_backup.sh | 96 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 72 deletions(-) delete mode 100644 rhodecode-backup.sh create mode 100644 rhodecode_backup.sh diff --git a/rhodecode-backup.sh b/rhodecode-backup.sh deleted file mode 100644 index 6b44382..0000000 --- a/rhodecode-backup.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -############################################################################# -# Nom du script : rhodecode-backup.sh -# Auteur : E.RIEGEL (QM HENIX) -# Date de Création : 15/02/2022 -# Version : 0.0.1 -# Descritpion : Script permettant la sauvegarde de la BDD de RhodeCode et de sauvegarder les repos de l'ANS -# -# Historique des mises à jour : -#-----------+--------+-------------+------------------------------------------------------ -# Version | Date | Auteur | Description -#-----------+--------+-------------+------------------------------------------------------ -# 0.0.1 | 15/02/22 | E.RIEGEL | Initialisation du script -#-----------+--------+-------------+------------------------------------------------------ -############################################################################################### - - -# Configuration de base: datestamp e.g. YYYYMMDD -DATE=$(date +"%Y%m%d") - -# Dossier où sauvegarder les backups -BACKUP_DIR="/var/BACKUP/RHODECODE" - -# Commande NOMAD -NOMAD=/usr/local/bin/nomad - -#Repo PATH To BACKUP in the container -REPO_PATH=/root/ -#Archive Name of the backup repo directory -BACKUP_REPO_FILENAME="BACKUP_REPOS_RHODECODE_${DATE}.tar.gz" -#Name of the dump file (Bdd Rhodecode) -DUMP_FILENAME="BACKUP_RHODECODE_BDD_${DATE}.dump" - -# Nombre de jours à garder les dossiers (seront effacés après X jours) -RETENTION=5 - -# ---- NE RIEN MODIFIER SOUS CETTE LIGNE ------------------------------------------ -# -# Create a new directory into backup directory location for this date -mkdir -p $BACKUP_DIR/$DATE - -# Backup repos -echo "Starting backup repos..." - -$NOMAD exec -job rhodecode tar -cOzv -C $REPO_PATH my_dev_repos > $BACKUP_DIR/$DATE/$BACKUP_REPO_FILENAME -BACKUP_RESULT=$? -if [ $BACKUP_RESULT -gt 1 ] -then - echo "Repo backup failed with error code : ${BACKUP_RESULT}" - exit 1 -else - echo "Repo backup done" -fi - -# Dump rhodecode bdd -echo "starting rhodecode dump..." -$NOMAD exec -job rhodecode pg_dump -F c --dbname=postgresql://postgres:postgres@localhost/rhodecode > $BACKUP_DIR/$DATE/$DUMP_FILENAME - -DUMP_RESULT=$? -if [ $DUMP_RESULT -gt 0 ] -then - echo "Rhodecode dump failed with error code : ${DUMP_RESULT}" - exit 1 -else - echo "Rhodecode dump done" -fi - -# Remove files older than X days -find $BACKUP_DIR/* -mtime +$RETENTION -delete - -echo "Backup Rhodecode finished" diff --git a/rhodecode_backup.sh b/rhodecode_backup.sh new file mode 100644 index 0000000..9d97747 --- /dev/null +++ b/rhodecode_backup.sh @@ -0,0 +1,96 @@ +#!/bin/bash +echo "Démarrage du script de sauvegarde de Rhodecode" +############################################################################# +# Nom du script : rhodecode-backup.sh +# Auteur : E.RIEGEL (QM HENIX) +# Date de Création : 15/02/2022 +# Version : 0.0.4 +# Descritpion : Script permettant la sauvegarde de la BDD de RhodeCode et de sauvegarder les repos de l'ANS +# +# Historique des mises à jour : +#-----------+--------+-------------+------------------------------------------------------ +# Version | Date | Auteur | Description +#-----------+--------+-------------+------------------------------------------------------ +# 0.0.1 | 15/02/22 | E.RIEGEL | Initialisation du script +#-----------+--------+-------------+------------------------------------------------------ +# 0.0.2 | 07/03/22 | M.JURAVLIOV | Ajout de la purge de la table user_logs (entrées de plus de 2 mois) +#-----------+--------+-------------+------------------------------------------------------ +# 0.0.3 | 11/05/22 | E.RIEGEL | Adaptation du script pour la version multi-conteneurs de Rhodecode +#-----------+--------+-------------+------------------------------------------------------ +# 0.0.4 | 21/09/23 | Y.ETRILLARD | Ajout -task dans la commande nomad exec de rhodecode-postgres +#-----------+--------+-------------+------------------------------------------------------ +############################################################################################### + +. /root/.bash_profile + +# Configuration de base: datestamp e.g. YYYYMMDD +DATE=$(date +"%Y%m%d") + +# Dossier où sauvegarder les backups +BACKUP_DIR="/var/BACKUP/RHODECODE" + +# Dossier de sauvegarde de la table user_logs +BACKUP_USER_LOGS_DIR="/var/BACKUP/RHODECODE_USER_LOGS" + +# Commande NOMAD +#NOMAD=/usr/local/bin/nomad +NOMAD=$(which nomad) + +#Repo PATH To BACKUP in the container +REPO_PATH=/var/opt +#Archive Name of the backup repo directory +BACKUP_REPO_FILENAME="BACKUP_REPOS_RHODECODE_${DATE}.tar.gz" +#Name of the dump file (Bdd Rhodecode) +DUMP_FILENAME="BACKUP_RHODECODE_BDD_${DATE}.dump" +#Archive Name of the backup user_logs table +BACKUP_USER_LOGS_FILENAME="BACKUP_USER_LOGS_${DATE}.tar.gz" + + +# Nombre de jours à garder les dossiers (seront effacés après X jours) +RETENTION=1 + +# ---- NE RIEN MODIFIER SOUS CETTE LIGNE ------------------------------------------ +# +# Create a new directory into backup directory location for this date +mkdir -p $BACKUP_DIR/$DATE +mkdir -p $BACKUP_USER_LOGS_DIR + +# Backup repos +echo "Starting backup repos..." + +$NOMAD exec -job -task rhodecode rhodecode-community tar -cOzv -C $REPO_PATH rhodecode_repo_store > $BACKUP_DIR/$DATE/$BACKUP_REPO_FILENAME +BACKUP_RESULT=$? +if [ $BACKUP_RESULT -gt 1 ] +then + echo "Repo backup failed with error code : ${BACKUP_RESULT}" + exit 1 +else + echo "Repo backup done" +fi + +# Dump rhodecode bdd +echo "starting rhodecode dump..." +$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres pg_dump -F c --dbname=postgresql://postgres@localhost/rhodecode > $BACKUP_DIR/$DATE/$DUMP_FILENAME + +DUMP_RESULT=$? +if [ $DUMP_RESULT -gt 0 ] +then + echo "Rhodecode dump failed with error code : ${DUMP_RESULT}" + exit 1 +else + echo "Rhodecode dump done" +fi + +# Remove files older than X days +find $BACKUP_DIR/* -mtime +$RETENTION -exec rm -rf {} \; + +echo "Backup Rhodecode finished" + +echo "Purging data from user_logs" +$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"copy (select * from user_logs where DATE(action_date) < NOW() - INTERVAL '15 DAY') to '/tmp/rhodecode_user_logs.csv' delimiter ',' CSV HEADER;\"" - postgres +$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres tar -cOzv /tmp/rhodecode_user_logs.csv > $BACKUP_USER_LOGS_DIR/$BACKUP_USER_LOGS_FILENAME +$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres rm -f /tmp/rhodecode_user_logs.csv +$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"delete from user_logs where DATE(action_date) < NOW() - INTERVAL '15 DAY';\"" - postgres +$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"VACUUM (VERBOSE, ANALYZE) user_logs;\"" - postgres + +echo "Data purging finished" From 03d3f261acf0a3213690982782248bea4e7a9320 Mon Sep 17 00:00:00 2001 From: Yann Etrillard Date: Mon, 25 Sep 2023 16:29:10 +0200 Subject: [PATCH 5/9] Modification du chemin de sauvegarde dans le script de backup --- rhodecode_backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rhodecode_backup.sh b/rhodecode_backup.sh index 9d97747..a972c40 100644 --- a/rhodecode_backup.sh +++ b/rhodecode_backup.sh @@ -93,4 +93,4 @@ $NOMAD exec -task rhodecode-postgres -job rhodecode-postgres rm -f /tmp/rhodecod $NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"delete from user_logs where DATE(action_date) < NOW() - INTERVAL '15 DAY';\"" - postgres $NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"VACUUM (VERBOSE, ANALYZE) user_logs;\"" - postgres -echo "Data purging finished" +echo "Data purging finished" \ No newline at end of file From cbf9d5448abce1651eca66d3c3210c199a9d32b7 Mon Sep 17 00:00:00 2001 From: Yann Etrillard Date: Mon, 25 Sep 2023 16:29:56 +0200 Subject: [PATCH 6/9] Modification du chemin de sauvegarde dans le script de backup --- rhodecode_backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rhodecode_backup.sh b/rhodecode_backup.sh index a972c40..9d97747 100644 --- a/rhodecode_backup.sh +++ b/rhodecode_backup.sh @@ -93,4 +93,4 @@ $NOMAD exec -task rhodecode-postgres -job rhodecode-postgres rm -f /tmp/rhodecod $NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"delete from user_logs where DATE(action_date) < NOW() - INTERVAL '15 DAY';\"" - postgres $NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"VACUUM (VERBOSE, ANALYZE) user_logs;\"" - postgres -echo "Data purging finished" \ No newline at end of file +echo "Data purging finished" From a4c338a6f2f25ff5240a3cd792bd11f4ea2d85ea Mon Sep 17 00:00:00 2001 From: Yann Etrillard Date: Tue, 26 Sep 2023 17:55:57 +0200 Subject: [PATCH 7/9] Modification de la casse du chemin de sauvegarde --- rhodecode_backup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rhodecode_backup.sh b/rhodecode_backup.sh index 9d97747..c6ba7cd 100644 --- a/rhodecode_backup.sh +++ b/rhodecode_backup.sh @@ -27,10 +27,10 @@ echo "Démarrage du script de sauvegarde de Rhodecode" DATE=$(date +"%Y%m%d") # Dossier où sauvegarder les backups -BACKUP_DIR="/var/BACKUP/RHODECODE" +BACKUP_DIR="/var/backup/RHODECODE" # Dossier de sauvegarde de la table user_logs -BACKUP_USER_LOGS_DIR="/var/BACKUP/RHODECODE_USER_LOGS" +BACKUP_USER_LOGS_DIR="/var/backup/RHODECODE_USER_LOGS" # Commande NOMAD #NOMAD=/usr/local/bin/nomad From 68e9c395eabdf9458419ac4eb95d793e5256912c Mon Sep 17 00:00:00 2001 From: Yann Etrillard Date: Fri, 29 Sep 2023 12:25:54 +0200 Subject: [PATCH 8/9] =?UTF-8?q?Dur=C3=A9e=20de=20r=C3=A9tention=203=20jour?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rhodecode_backup.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rhodecode_backup.sh b/rhodecode_backup.sh index c6ba7cd..e62f414 100644 --- a/rhodecode_backup.sh +++ b/rhodecode_backup.sh @@ -4,7 +4,7 @@ echo "Démarrage du script de sauvegarde de Rhodecode" # Nom du script : rhodecode-backup.sh # Auteur : E.RIEGEL (QM HENIX) # Date de Création : 15/02/2022 -# Version : 0.0.4 +# Version : 0.0.5 # Descritpion : Script permettant la sauvegarde de la BDD de RhodeCode et de sauvegarder les repos de l'ANS # # Historique des mises à jour : @@ -17,7 +17,9 @@ echo "Démarrage du script de sauvegarde de Rhodecode" #-----------+--------+-------------+------------------------------------------------------ # 0.0.3 | 11/05/22 | E.RIEGEL | Adaptation du script pour la version multi-conteneurs de Rhodecode #-----------+--------+-------------+------------------------------------------------------ -# 0.0.4 | 21/09/23 | Y.ETRILLARD | Ajout -task dans la commande nomad exec de rhodecode-postgres +# 0.0.4 | 21/09/23 | Y.ETRILLARD | Ajout -task dans la commande nomad exec de rhodecode-postgres +#-----------+--------+-------------+------------------------------------------------------ +# 0.0.5 | 29/09/23 | Y.ETRILLARD | Rétention à 3 jours #-----------+--------+-------------+------------------------------------------------------ ############################################################################################### @@ -47,7 +49,7 @@ BACKUP_USER_LOGS_FILENAME="BACKUP_USER_LOGS_${DATE}.tar.gz" # Nombre de jours à garder les dossiers (seront effacés après X jours) -RETENTION=1 +RETENTION=3 # ---- NE RIEN MODIFIER SOUS CETTE LIGNE ------------------------------------------ # From 50d4aec3020b0db65ccea8e871495e32fe7f4191 Mon Sep 17 00:00:00 2001 From: Yann Etrillard Date: Fri, 17 Nov 2023 11:41:34 +0100 Subject: [PATCH 9/9] Ajout d'un nouveau fqdn --- rhodecode-app/rhodecode-community.nomad.tpl | 6 +++++- rhodecode-syncldap/rhodecode-syncldap.nomad.tpl | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rhodecode-app/rhodecode-community.nomad.tpl b/rhodecode-app/rhodecode-community.nomad.tpl index 0db4e54..a8d8897 100644 --- a/rhodecode-app/rhodecode-community.nomad.tpl +++ b/rhodecode-app/rhodecode-community.nomad.tpl @@ -107,6 +107,7 @@ DB_UPGRADE=1 SETUP_APP=0 MAIN_INI_PATH="/secrets/rhodecode.optimized.ini" EXTERNAL_HOSTNAME={{ with secret "forge/rhodecode/app" }}{{ .Data.data.rc_external_hostname }}{{end}} +EXTERNAL_VIP_HOSTNAME={{ with secret "forge/rhodecode/app" }}{{ .Data.data.rc_external_vip_hostname }}{{end}} EOT destination="local/rc.env" env = true @@ -663,7 +664,10 @@ EOT } service { name = "$\u007BNOMAD_TASK_NAME\u007D-http" - tags = ["urlprefix-$\u007BEXTERNAL_HOSTNAME\u007D"] + tags = [ + "urlprefix-$${EXTERNAL_HOSTNAME}", + "urlprefix-$${EXTERNAL_VIP_HOSTNAME}" + ] port = "rhodecode" check { name = "rhodecode-alive" diff --git a/rhodecode-syncldap/rhodecode-syncldap.nomad.tpl b/rhodecode-syncldap/rhodecode-syncldap.nomad.tpl index 1c0eade..d34ba7c 100644 --- a/rhodecode-syncldap/rhodecode-syncldap.nomad.tpl +++ b/rhodecode-syncldap/rhodecode-syncldap.nomad.tpl @@ -30,7 +30,7 @@ RHODECODE_AUTH_TOKEN="{{with secret "forge/rhodecode/api"}}{{.Data.data.auth_tok destination = "secrets/file.env" change_mode = "restart" env = true - } + } resources { cpu = 100 memory = 64