ansforge · mfaurel · Oct 17, 2022 · Dec 6, 2022 · Mar 28, 2023 · Aug 22, 2023
diff --git a/rhodecode-app/rhodecode-community.nomad.tpl b/rhodecode-app/rhodecode-community.nomad.tpl
@@ -67,6 +67,10 @@ job "rhodecode-community" {
 			}
 		}
 		task "rhodecode" {
+
+			# log-shipper
+			leader = true 
+
 			artifact {
 				source = "${extensions_url}"
 				destination = "local"
@@ -103,6 +107,7 @@ DB_UPGRADE=1
 SETUP_APP=0
 MAIN_INI_PATH="/secrets/rhodecode.optimized.ini"
 EXTERNAL_HOSTNAME={{ with secret "forge/rhodecode/app" }}{{ .Data.data.rc_external_hostname }}{{end}}
+EXTERNAL_VIP_HOSTNAME={{ with secret "forge/rhodecode/app" }}{{ .Data.data.rc_external_vip_hostname }}{{end}}
 EOT
 						destination="local/rc.env"
 						env = true
@@ -659,7 +664,10 @@ EOT
 			}
 			service {
 				name = "$\u007BNOMAD_TASK_NAME\u007D-http"
-				tags = ["urlprefix-$\u007BEXTERNAL_HOSTNAME\u007D"]
+				tags = [
+					"urlprefix-$${EXTERNAL_HOSTNAME}",
+					"urlprefix-$${EXTERNAL_VIP_HOSTNAME}"
+					]
 				port = "rhodecode"
 				check {
 					name         = "rhodecode-alive"
@@ -1081,6 +1089,35 @@ EOT
 				}
 		}
 
-	}
-}
+        # log-shipper
+        task "log-shipper" {
+            driver = "docker"
+            restart {
+				interval = "3m"
+				attempts = 5
+				delay    = "15s"
+				mode     = "delay"
+            }
+            meta {
+                INSTANCE = "$\u007BNOMAD_ALLOC_NAME\u007D"
+            }
+            template {
+                data = <<EOH
+REDIS_HOSTS = {{ range service "PileELK-redis" }}{{ .Address }}:{{ .Port }}{{ end }}
+PILE_ELK_APPLICATION = RHODECODE 
+EOH
+                destination = "local/file.env"
+                change_mode = "restart"
+                env = true
+            }
+            config {
+                image = "ans/nomad-filebeat:8.2.3-2.0"
+            }
+            resources {
+                cpu    = 100
+                memory = 150
+            }
+        } #end log-shipper 
 
+	}
+}
diff --git a/rhodecode-db/rhodecode-postgres.nomad.tpl b/rhodecode-db/rhodecode-postgres.nomad.tpl
@@ -33,6 +33,10 @@ job "rhodecode-postgres" {
 
     task "rhodecode-postgres" {
       driver = "docker"
+
+      # log-shipper
+      leader = true 
+
                 template {
                         data = <<EOH
 POSTGRES_DB = "rhodecode"
@@ -91,5 +95,34 @@ EOH
         }
       }
     }
+    # log-shipper
+    task "log-shipper" {
+        driver = "docker"
+        restart {
+                interval = "3m"
+                attempts = 5
+                delay    = "15s"
+                mode     = "delay"
+        }
+        meta {
+            INSTANCE = "$\u007BNOMAD_ALLOC_NAME\u007D"
+        }
+        template {
+            data = <<EOH
+REDIS_HOSTS = {{ range service "PileELK-redis" }}{{ .Address }}:{{ .Port }}{{ end }}
+PILE_ELK_APPLICATION = RHODECODE 
+EOH
+            destination = "local/file.env"
+            change_mode = "restart"
+            env = true
+        }
+        config {
+            image = "ans/nomad-filebeat:8.2.3-2.0"
+        }
+        resources {
+            cpu    = 100
+            memory = 150
+        }
+    } #end log-shipper 
   }
 }
diff --git a/rhodecode-syncldap/rhodecode-syncldap.nomad.tpl b/rhodecode-syncldap/rhodecode-syncldap.nomad.tpl
@@ -12,6 +12,10 @@ job "rhodecode-syncldap" {
   group "rhodecode-syncldap" {
     task "rhodecode-syncldap" {
       driver = "docker"
+
+      # log-shipper
+      leader = true 
+
       config {
         image = "${image}:${tag}"
       }
@@ -26,11 +30,42 @@ RHODECODE_AUTH_TOKEN="{{with secret "forge/rhodecode/api"}}{{.Data.data.auth_tok
         destination = "secrets/file.env"
         change_mode = "restart"
         env         = true
-     }
+      }
       resources {
         cpu    = 100
         memory = 64
       }
     }
+
+    # log-shipper
+    task "log-shipper" {
+        driver = "docker"
+        restart {
+                interval = "3m"
+                attempts = 5
+                delay    = "15s"
+                mode     = "delay"
+        }
+        meta {
+            INSTANCE = "$\u007BNOMAD_ALLOC_NAME\u007D"
+        }
+        template {
+            data = <<EOH
+REDIS_HOSTS = {{ range service "PileELK-redis" }}{{ .Address }}:{{ .Port }}{{ end }}
+PILE_ELK_APPLICATION = RHODECODE 
+EOH
+            destination = "local/file.env"
+            change_mode = "restart"
+            env = true
+        }
+        config {
+            image = "ans/nomad-filebeat:8.2.3-2.0"
+        }
+        resources {
+            cpu    = 100
+            memory = 150
+        }
+    } #end log-shipper 
+
   }
 }
diff --git a/rhodecode-tools/rhodecode-tools.nomad.tpl b/rhodecode-tools/rhodecode-tools.nomad.tpl
@@ -28,6 +28,10 @@ job "rhodecode-tools" {
                 task "rhodecode-index" {
 
                         driver = "docker"
+
+                        # log-shipper
+                        leader = true 
+
                         template {
                                 data = <<EOT
 [DEFAULT]
@@ -96,9 +100,38 @@ EOT
 
                         }
                         resources {
-                                        cpu = 256
-                                        memory = 512
+                                cpu = 256
+                                memory = 512
                         }
                 }
+                # log-shipper
+                task "log-shipper" {
+                        driver = "docker"
+                        restart {
+                                interval = "3m"
+                                attempts = 5
+                                delay    = "15s"
+                                mode     = "delay"
+                        }
+                        meta {
+                                INSTANCE = "$\u007BNOMAD_ALLOC_NAME\u007D"
+                        }
+                        template {
+                                data = <<EOH
+REDIS_HOSTS = {{ range service "PileELK-redis" }}{{ .Address }}:{{ .Port }}{{ end }}
+PILE_ELK_APPLICATION = RHODECODE 
+EOH
+                                destination = "local/file.env"
+                                change_mode = "restart"
+                                env = true
+                        }
+                        config {
+                                image = "ans/nomad-filebeat:8.2.3-2.0"
+                        }
+                        resources {
+                                cpu    = 100
+                                memory = 150
+                        }
+                } #end log-shipper 
         }
-}
+}
diff --git a/rhodecode_backup.sh b/rhodecode_backup.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+echo "Démarrage du script de sauvegarde de Rhodecode"
+#############################################################################
+# Nom du script     : rhodecode-backup.sh
+# Auteur            : E.RIEGEL (QM HENIX)
+# Date de Création  : 15/02/2022
+# Version           : 0.0.5
+# Descritpion       : Script permettant la sauvegarde de la BDD de RhodeCode et de sauvegarder les repos de l'ANS
+#
+# Historique des mises à jour :
+#-----------+--------+-------------+------------------------------------------------------
+#  Version  |   Date   |   Auteur     |  Description
+#-----------+--------+-------------+------------------------------------------------------
+#  0.0.1    | 15/02/22 | E.RIEGEL     | Initialisation du script
+#-----------+--------+-------------+------------------------------------------------------
+#  0.0.2    | 07/03/22 | M.JURAVLIOV  | Ajout de la purge de la table user_logs (entrées de plus de 2 mois)
+#-----------+--------+-------------+------------------------------------------------------
+#  0.0.3    | 11/05/22 | E.RIEGEL     | Adaptation du script pour la version multi-conteneurs de Rhodecode
+#-----------+--------+-------------+------------------------------------------------------
+#  0.0.4    | 21/09/23 | Y.ETRILLARD  | Ajout -task dans la commande nomad exec de rhodecode-postgres
+#-----------+--------+-------------+------------------------------------------------------
+#  0.0.5    | 29/09/23 | Y.ETRILLARD  | Rétention à 3 jours
+#-----------+--------+-------------+------------------------------------------------------
+###############################################################################################
+
+. /root/.bash_profile
+
+# Configuration de base: datestamp e.g. YYYYMMDD
+DATE=$(date +"%Y%m%d")
+
+# Dossier où sauvegarder les backups
+BACKUP_DIR="/var/backup/RHODECODE"
+
+# Dossier de sauvegarde de la table user_logs
+BACKUP_USER_LOGS_DIR="/var/backup/RHODECODE_USER_LOGS"
+
+# Commande NOMAD
+#NOMAD=/usr/local/bin/nomad
+NOMAD=$(which nomad)
+
+#Repo PATH To BACKUP in the container
+REPO_PATH=/var/opt
+#Archive Name of the backup repo directory
+BACKUP_REPO_FILENAME="BACKUP_REPOS_RHODECODE_${DATE}.tar.gz"
+#Name of the dump file (Bdd Rhodecode)
+DUMP_FILENAME="BACKUP_RHODECODE_BDD_${DATE}.dump"
+#Archive Name of the backup user_logs table
+BACKUP_USER_LOGS_FILENAME="BACKUP_USER_LOGS_${DATE}.tar.gz"
+
+
+# Nombre de jours à garder les dossiers (seront effacés après X jours)
+RETENTION=3
+
+# ---- NE RIEN MODIFIER SOUS CETTE LIGNE ------------------------------------------
+#
+# Create a new directory into backup directory location for this date
+mkdir -p $BACKUP_DIR/$DATE
+mkdir -p $BACKUP_USER_LOGS_DIR
+
+# Backup repos
+echo "Starting backup repos..."
+
+$NOMAD exec -job -task rhodecode rhodecode-community tar -cOzv -C $REPO_PATH rhodecode_repo_store > $BACKUP_DIR/$DATE/$BACKUP_REPO_FILENAME
+BACKUP_RESULT=$?
+if [ $BACKUP_RESULT -gt 1 ]
+then
+        echo "Repo backup failed with error code : ${BACKUP_RESULT}"
+        exit 1
+else
+        echo "Repo backup done"
+fi
+
+# Dump rhodecode bdd
+echo "starting rhodecode dump..."
+$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres pg_dump -F c --dbname=postgresql://postgres@localhost/rhodecode > $BACKUP_DIR/$DATE/$DUMP_FILENAME
+
+DUMP_RESULT=$?
+if [ $DUMP_RESULT -gt 0 ]
+then
+        echo "Rhodecode dump failed with error code : ${DUMP_RESULT}"
+        exit 1
+else
+        echo "Rhodecode dump done"
+fi
+
+# Remove files older than X days
+find $BACKUP_DIR/* -mtime +$RETENTION -exec rm -rf {} \;
+
+echo "Backup Rhodecode finished"
+
+echo "Purging data from user_logs"
+$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"copy (select * from user_logs where DATE(action_date) < NOW() - INTERVAL '15 DAY') to '/tmp/rhodecode_user_logs.csv' delimiter ',' CSV HEADER;\"" - postgres
+$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres tar -cOzv /tmp/rhodecode_user_logs.csv > $BACKUP_USER_LOGS_DIR/$BACKUP_USER_LOGS_FILENAME
+$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres rm -f /tmp/rhodecode_user_logs.csv
+$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"delete from user_logs where DATE(action_date) < NOW() - INTERVAL '15 DAY';\"" - postgres
+$NOMAD exec -task rhodecode-postgres -job rhodecode-postgres su -c "psql -d rhodecode -c \"VACUUM (VERBOSE, ANALYZE) user_logs;\"" - postgres
+
+echo "Data purging finished"