Cicd functionality

README.md

@@ -1,21 +1,23 @@
 # What has been done so far
-- Monitoring using Prometheus, Grafana, Node-explorer & CAdvisor
-- api gateway 
+- monitoring using Prometheus, Grafana, Node-explorer & CAdvisor
+- api gateway
 - docker-swarm for deployment
-- Assure that you can create newer images based on BE & FE updates
+- create newer images based on BE & FE updates
+- persistent DB
+- management of Docker Swarm services with Portainer
+- pipeline CI/CD (no stage for testing yet)
 
 # What needs to be done
-- Make DB persistent
 - Loki for logging
 - rabbitMQ
-- pipeline CI/CD
+
 # Contents
 - Kong
 - Prometheus
 - docker-swarm file
 - scripts for running and removing containers
 
-# Ports 
+# Ports
 - FE runs on port 3000
 - BE runs on port 8080
 - Kong routes traffic from localhost:8080 to localhost:8000
@@ -25,5 +27,68 @@
 - Node-explorer runs on port 9100
 
 # Running
-- after running `run.sh` script give it around 2 mins before backend will be up and running
-- use `clean.sh` to stop services and remove pruned containers
+- after running `run.sh` script give it around 5 mins before backend will be up and running
+- use `clean.sh` to stop services and remove pruned containers
+
+# Portainer
+Management of the Docker Swarm infrastructure. It is used as an visual interface to control the container environment.
+
+To start the portainer containers run:
+```
+./run-portainer
+```
+
+To stop the portainer containers run:
+```
+./clean-portainer
+```
+
+Check that the services in the *portainer* stack are fully created, then access it throught `localhost:9010` in the web browser. Log in using the username *admin* and password *adminidp2022*.
+
+Add a new stack with the name *idp_project* and paste the content of the `stack.yml` file, wait until all services are up and running.
+
+
+# Gitlab CI/CD
+Created two separate repositories for the front-end and back-end and add their own `Dockerfile` to create the images accordingly. Added `.gitlab-ci.yml` in both repos. (the address of the webhook needs to be added manually)
+
+Create images for front-end and back-end and add them to the registry:
+```
+docker login gitlab.cs.pub.ro:5050
+docker build -t gitlab.cs.pub.ro:5050/pw-idp-2022/pw-backend .
+docker push gitlab.cs.pub.ro:5050/pw-idp-2022/pw-backend
+
+docker login gitlab.cs.pub.ro:5050
+docker build -t gitlab.cs.pub.ro:5050/pw-idp-2022/pw-frontend .
+docker push gitlab.cs.pub.ro:5050/pw-idp-2022/pw-frontend
+```
+
+Create the Gitlab runner with the following commands:
+```
+docker run -d --name gitlab-runner --restart always -v gitlab-runner-config:/etc/gitlab-runner -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
+
+docker run --rm -it -v gitlab-runner-config:/etc/gitlab-runner gitlab/gitlab-runner register
+
+# to register use de following:
+# https://gitlab.cs.pub.ro/
+# Registration token
+# Name of the runner
+# Some tags
+# docker
+# docker:19.03
+
+docker run -it -v gitlab-runner-config:/test alpine
+```
+
+Modify the `config.toml` file to grant privileged access.
+```
+privileged = true
+volumes = ["/cache", "/var/run/docker.sock:/var/run/docker.sock"]
+
+```
+
+Restart the runner:
+```
+sudo docker restart gitlab-runner
+```
+
+To test, try making changes for the master branch and see if anything changes. :)

clean-portainer.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+docker stack rm portainer
+docker kill $(docker ps -q)
+docker container prune
+docker image prune

gitlab-stack.yml

@@ -0,0 +1,149 @@
+version: "3.8"
+services:
+  frontend:
+    image: gitlab.cs.pub.ro:5050/pw-idp-2022/pw-frontend
+    ports:
+      - "3000:3000"
+    depends_on:
+      - backend
+    networks:
+      - idp_network
+    deploy:
+      replicas: 2
+      placement:
+        max_replicas_per_node: 2
+      update_config:
+        parallelism: 2
+        order: start-first
+      rollback_config:
+        parallelism: 2
+        order: stop-first
+
+  backend:
+    image: gitlab.cs.pub.ro:5050/pw-idp-2022/pw-backend
+    ports:
+      - "8080:8080"
+    depends_on:
+      - db
+    networks:
+      - idp_network
+    environment:
+        SPRING_DATASOURCE_URL: jdbc:mysql://db:3306/proiect?createDatabaseIfNotExist=true
+        SPRING_DATASOURCE_USERNAME: root
+        SPRING_DATASOURCE_PASSWORD: admin
+    deploy:
+      replicas: 2
+      placement:
+        max_replicas_per_node: 2
+      update_config:
+        parallelism: 2
+        order: start-first
+      rollback_config:
+        parallelism: 2
+        order: stop-first
+
+  db:
+    image: mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: admin
+      MYSQL_DATABASE: proiect
+      MYSQL_USER: admin
+    ports:
+      - "3306:3306"
+    networks:
+      - idp_network
+    deploy:
+      placement:
+        constraints:
+          - "node.role==manager"
+    volumes:
+      - db-volume:/var/lib/mysql
+
+  kong:
+    image: kong:latest
+    volumes:
+      - /home/hypersmoit/Desktop/idp_pweb/idp/Kong:/usr/local/kong/declarative
+    environment:
+      KONG_DATABASE: 'off'
+      KONG_DECLARATIVE_CONFIG: /usr/local/kong/declarative/kong.yml
+      KONG_PROXY_ACCESS_LOG: /dev/stdout
+      KONG_ADMIN_ACCESS_LOG: /dev/stdout
+      KONG_PROXY_ERROR_LOG: /dev/stderr
+      KONG_ADMIN_ERROR_LOG: /dev/stderr
+      KONG_ADMIN_LISTEN: 0.0.0.0:8001, 0.0.0.0:8444 ssl
+    ports:
+      - 8000:8000
+      - 8443:8443
+      - 8001:8001
+      - 8444:8444
+    deploy:
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - idp_network
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - grafana-volume:/var/lib/grafana
+    ports:
+      - 3001:3000
+    depends_on:
+      - prometheus
+    deploy:
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - idp_network
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - /home/hypersmoit/Desktop/idp_pweb/idp/prometheus/config.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - 9090:9090
+    deploy:
+      placement:
+        constraints: [node.role == manager]
+    networks:
+      - idp_network
+
+  node_exporter:
+        image: prom/node-exporter
+        deploy:
+            mode: global
+        volumes:
+            - /proc:/host/proc:ro
+            - /sys:/host/sys:ro
+            - /:/rootfs:ro
+        command:
+            - '--path.procfs=/host/proc'
+            - '--path.rootfs=/rootfs'
+            - '--path.sysfs=/host/sys'
+            - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+        ports:
+            - 9100:9100
+        networks:
+            - idp_network
+
+  cadvisor:
+        image: gcr.io/cadvisor/cadvisor
+        deploy:
+            mode: global
+        volumes:
+            - /:/rootfs:ro
+            - /var/run:/var/run:ro
+            - /sys:/sys:ro
+            - /var/lib/docker/:/var/lib/docker:ro
+            - /dev/disk:/dev/disk/:ro
+        ports:
+            - 8081:8080
+        networks:
+            - idp_network
+
+volumes:
+  grafana-volume:
+  db-volume:
+
+networks:
+  idp_network:

portainer-agent-stack.yml

@@ -0,0 +1,39 @@
+version: '3.2'
+
+services:
+  agent:
+    image: portainer/agent:2.11.1
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/volumes:/var/lib/docker/volumes
+    networks:
+      - agent_network
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.platform.os == linux]
+
+  portainer:
+    image: portainer/portainer-ce:2.11.1
+    command: -H tcp://tasks.agent:9001 --tlsskipverify
+    ports:
+      - "9443:9443"
+      - "9000:9000"
+      - "8002:8000"
+    volumes:
+      - portainer_data:/data
+    networks:
+      - agent_network
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+
+networks:
+  agent_network:
+    driver: overlay
+    attachable: true
+
+volumes:
+  portainer_data:

run-portainer.sh

@@ -0,0 +1,4 @@
+#! /bin/bash
+# docker pull portainer/agent
+# docker pull portainer/portainer-ce
+docker stack deploy -c portainer-agent-stack.yml portainer