Cicd

.github/workflows/EC2_deploy.yaml

@@ -0,0 +1,293 @@
+name: Deploy to Amazon EC2
+
+on:
+  push:
+    branches:
+      - cicd  # Trigger deployment on the cicd branch
+
+env:
+  AWS_REGION: us-east-1
+  ECR_REPOSITORY_NAMESPACE: indeq
+  IMAGE_TAG: ${{ github.run_id }}
+  # IMAGE_TAG: "14588128148"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # For OIDC auth (recommended)
+      contents: read
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+
+    # comment out starting here
+    - name: Install Protocol Buffers Compiler
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y protobuf-compiler
+
+    - name: Install Go
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y golang-go
+
+    - name: Install protoc-gen-go
+      run: |
+        go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+        echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV
+
+    - name: Install protoc-gen-go-grpc
+      run: |
+        go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+        echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV
+
+    - name: Check protoc-gen-go Installation
+      run: |
+        protoc-gen-go --version || echo "protoc-gen-go not found"
+    # comment out ending here  
+
+    - name: Create .env file
+      run: |
+        # first delete the .env file
+        rm backend/common/config/.env
+
+        # Use cat with heredoc to prevent command interpretation
+        cat << 'EOL' > backend/common/config/.env
+        ${{ secrets.ENV1 }}
+        ${{ secrets.ENV2 }}
+        ${{ secrets.ENV3 }}
+        ${{ secrets.ENV4 }}
+        ${{ secrets.ENV5 }}
+        ${{ secrets.ENV6 }}
+        ${{ secrets.ENV7 }}
+        EOL
+
+    # comment out starting here
+    - name: Generate Code
+      run: |
+        cd backend/common && make gen
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    # comment out ending here
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ env.AWS_REGION }}
+
+
+    - name: Login to Amazon ECR
+      run: |
+        aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com
+
+    - name: Extract EC2 Key Pair
+      run: |
+        # Extract markers
+        BEGIN_MARKER="-----BEGIN RSA PRIVATE KEY-----"
+        END_MARKER="-----END RSA PRIVATE KEY-----"
+
+        # Create PEM file with proper formatting
+        echo "$BEGIN_MARKER" > private_key.pem
+        echo "${{ secrets.EC2_KEY_PAIR }}" | sed 's/.\{64\}/&\n/g' >> private_key.pem
+        echo "$END_MARKER" >> private_key.pem
+
+        chmod 600 private_key.pem
+
+    - name: Predeploy Cleanup
+      run: |
+
+        #  showing private key
+        cat private_key.pem
+
+        # SSH into EC2 to create directories first for clean up
+        ssh -vvv -o StrictHostKeyChecking=no -i private_key.pem ec2-user@${{ secrets.EC2_PUBLIC_IP }} << 'EOF'
+
+          # Print current directory and list contents
+          pwd
+
+          # Install Docker if not already installed
+          if ! command -v docker &> /dev/null; then
+            echo "Docker not found. Installing..."
+            sudo yum update -y
+            sudo yum install -y docker
+            sudo systemctl start docker
+            sudo systemctl enable docker
+            sudo usermod -aG docker ec2-user
+            echo "Docker installed successfully"
+          fi 
+
+          # Create necessary directories if they don't exist
+          if [ ! -d "/home/ec2-user/common/config" ]; then
+            mkdir -p /home/ec2-user/common/config
+          fi
+
+          # If directory exists, remove .env inside
+          if [ -f "/home/ec2-user/common/config/.env" ]; then
+            rm /home/ec2-user/common/config/.env
+          fi
+
+          # Stop and remove containers safely
+          if [ "$(docker ps -q)" ]; then
+            docker stop $(docker ps -q)
+          fi
+          if [ "$(docker ps -aq)" ]; then
+            docker rm $(docker ps -aq)
+          fi
+
+          docker image prune -f
+          docker network prune -f
+        EOF
+
+    - name: Copy env file to EC2
+      run: |
+        # Copy the env file to EC2
+        scp -o StrictHostKeyChecking=no -i private_key.pem backend/common/config/.env ec2-user@${{ secrets.EC2_PUBLIC_IP }}:/home/ec2-user/common/config/.env
+
+    - name: Build Docker Images
+      run: |
+        authImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG"
+        queryImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG"
+        gatewayImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG"
+        initImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG"
+        embeddingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG"
+        retrievalImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG"
+        vectorImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG"
+        desktopImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG"
+        integrationImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/integration:$IMAGE_TAG"
+        crawlingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG"
+        mqttImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/mqtt:$IMAGE_TAG"
+
+        docker build -t authentication:$IMAGE_TAG -f backend/authentication/Dockerfile backend
+        docker tag authentication:$IMAGE_TAG $authImage
+
+        docker build -t query:$IMAGE_TAG -f backend/query/Dockerfile backend
+        docker tag query:$IMAGE_TAG $queryImage
+
+        docker build -t gateway:$IMAGE_TAG -f backend/gateway/Dockerfile backend
+        docker tag gateway:$IMAGE_TAG $gatewayImage
+
+        docker build -t init:$IMAGE_TAG -f backend/init/Dockerfile backend
+        docker tag init:$IMAGE_TAG $initImage
+
+        docker build -t embedding:$IMAGE_TAG -f backend/embedding/Dockerfile backend
+        docker tag embedding:$IMAGE_TAG $embeddingImage
+
+        docker build -t retrieval:$IMAGE_TAG -f backend/retrieval/Dockerfile backend
+        docker tag retrieval:$IMAGE_TAG $retrievalImage
+
+        docker build -t vector:$IMAGE_TAG -f backend/vector/Dockerfile backend
+        docker tag vector:$IMAGE_TAG $vectorImage
+
+        docker build -t desktop:$IMAGE_TAG -f backend/desktop/Dockerfile backend
+        docker tag desktop:$IMAGE_TAG $desktopImage
+
+        docker build -t integration:$IMAGE_TAG -f backend/integration/Dockerfile backend
+        docker tag integration:$IMAGE_TAG $integrationImage
+
+        docker build -t crawling:$IMAGE_TAG -f backend/crawling/Dockerfile backend
+        docker tag crawling:$IMAGE_TAG $crawlingImage
+
+        docker build -t mqtt:$IMAGE_TAG -f backend/mqtt/Dockerfile backend
+        docker tag mqtt:$IMAGE_TAG $mqttImage
+
+    - name: Push Docker Images
+      run: |
+        authImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG"
+        queryImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG"
+        gatewayImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG"
+        initImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG"
+        embeddingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG"
+        retrievalImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG"
+        vectorImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG"
+        desktopImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG"
+        integrationImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/integration:$IMAGE_TAG"
+        crawlingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG"
+        mqttImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/mqtt:$IMAGE_TAG"
+
+        docker push $authImage
+        docker push $queryImage
+        docker push $gatewayImage
+        docker push $initImage
+        docker push $embeddingImage
+        docker push $retrievalImage
+        docker push $vectorImage
+        docker push $desktopImage
+        docker push $integrationImage
+        docker push $crawlingImage
+        docker push $mqttImage
+
+        # Replace placeholder image references with actual ECR URLs
+        sed -i "s|authImage|$authImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|queryImage|$queryImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|gatewayImage|$gatewayImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|initImage|$initImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|embeddingImage|$embeddingImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|retrievalImage|$retrievalImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|vectorImage|$vectorImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|desktopImage|$desktopImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|integrationImage|$integrationImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|crawlingImage|$crawlingImage|g" backend/docker-compose-ec2.yaml
+        sed -i "s|mqttImage|$mqttImage|g" backend/docker-compose-ec2.yaml
+
+        # Checking if the docker-compose.yaml file is updated
+        cat backend/docker-compose-ec2.yaml
+        echo "TESTING"
+
+    - name: Copy docker-compose file to EC2
+      run: |
+        # Copy the docker-compose file to EC2
+        # first echo file content
+        cat backend/docker-compose-ec2.yaml
+
+        scp -o StrictHostKeyChecking=no -i private_key.pem backend/docker-compose-ec2.yaml ec2-user@${{ secrets.EC2_PUBLIC_IP }}:/home/ec2-user/
+
+    - name: Deploy to EC2
+      run: |
+        # Install AWS CLI if not already installed
+        if ! command -v aws &> /dev/null; then
+          echo "AWS CLI not found. Installing..."
+          sudo apt-get update
+          sudo apt-get install -y awscli
+        fi
+
+        # SSH into the EC2 instance and run the Docker containers
+        ssh -v -o StrictHostKeyChecking=no -i private_key.pem ec2-user@${{ secrets.EC2_PUBLIC_IP }} << 'EOF'
+          # Install Docker if not already installed
+          if ! command -v docker &> /dev/null; then
+            echo "Docker not found. Installing..."
+            sudo yum update -y
+            sudo yum install -y docker
+            sudo systemctl start docker
+            sudo systemctl enable docker
+            sudo usermod -aG docker ec2-user
+            echo "Docker installed successfully"
+          fi
+
+          # Install Docker Compose V2 if not present
+          if ! command -v docker-compose &> /dev/null && ! command -v docker compose &> /dev/null; then
+            echo "Installing Docker Compose V2..."
+            # For Amazon Linux 2023
+            sudo mkdir -p /usr/local/lib/docker/cli-plugins
+            sudo curl -SL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose
+            sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
+          fi
+
+          # Configure AWS CLI
+          aws configure set region us-east-1
+
+          # Verify IAM role
+          aws sts get-caller-identity
+
+          # Login to ECR without credentials (using IAM role)
+          aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com
+
+          # Run docker compose with absolute path
+          docker compose -f /home/ec2-user/docker-compose-ec2.yaml up -d
+        EOF

.gitignore

@@ -34,6 +34,8 @@
 
 # Environment variables
 .env.local
+backend-config.yaml
+root_ca.crt
 
 # Log files
 *.log