diff --git a/.github/deploy.yaml b/.github/deploy.yaml new file mode 100644 index 00000000..d68b593f --- /dev/null +++ b/.github/deploy.yaml @@ -0,0 +1,303 @@ +name: Deploy to Amazon EKS + +on: + push: + branches: + - cicd # Trigger deployment on the cicd branch + +env: + AWS_REGION: us-east-1 + ECR_REPOSITORY_NAMESPACE: indeq + IMAGE_TAG: ${{ github.run_id }} + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + id-token: write # For OIDC auth (recommended) + contents: read + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Install Protocol Buffers Compiler + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - name: Install Go + run: | + sudo apt-get update + sudo apt-get install -y golang-go + + - name: Install protoc-gen-go + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@latest + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV + + - name: Install protoc-gen-go-grpc + run: | + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV + + - name: Check protoc-gen-go Installation + run: | + protoc-gen-go --version || echo "protoc-gen-go not found" + + - name: Generate .env Files + env: + AUTH_PORT: ${{ secrets.AUTH_PORT }} + DEV_PROD: ${{ secrets.DEV_PROD }} + AUTH_ADDRESS: ${{ secrets.AUTH_ADDRESS }} + QUERY_PORT: ${{ secrets.QUERY_PORT }} + QUERY_ADDRESS: ${{ secrets.QUERY_ADDRESS }} + VECTOR_PORT: ${{ secrets.VECTOR_PORT }} + VECTOR_ADDRESS: ${{ secrets.VECTOR_ADDRESS }} + DESKTOP_ADDRESS: ${{ secrets.DESKTOP_ADDRESS }} + DESKTOP_PORT: ${{ secrets.DESKTOP_PORT }} + RETRIEVAL_ADDRESS: ${{ secrets.RETRIEVAL_ADDRESS }} + RETRIEVAL_PORT: ${{ secrets.RETRIEVAL_PORT }} + WAITLIST_ADDRESS: ${{ secrets.WAITLIST_ADDRESS }} + WAITLIST_PORT: ${{ secrets.WAITLIST_PORT }} + INTEGRATION_ADDRESS: ${{ secrets.INTEGRATION_ADDRESS }} + INTEGRATION_PORT: ${{ secrets.INTEGRATION_PORT }} + GATEWAY_ADDRESS: ${{ secrets.GATEWAY_ADDRESS }} + CRAWLING_PORT: ${{ secrets.CRAWLING_PORT }} + CRAWLING_ADDRESS: ${{ secrets.CRAWLING_ADDRESS }} + COUCHDB_ADDRESS: ${{ secrets.COUCHDB_ADDRESS }} + MQTT_ADDRESS: ${{ secrets.MQTT_ADDRESS }} + MQTT_PORT: ${{ secrets.MQTT_PORT }} + EMBEDDING_MODEL_NAME: ${{ secrets.EMBEDDING_MODEL_NAME }} + EMBEDDING_RERANKER: ${{ secrets.EMBEDDING_RERANKER }} + EMBEDDING_PORT: ${{ secrets.EMBEDDING_PORT }} + EMBEDDING_ADDRESS: ${{ secrets.EMBEDDING_ADDRESS }} + EMBEDDING_MAX_WORKERS: ${{ secrets.EMBEDDING_MAX_WORKERS }} + EMBEDDING_GRACEFUL_SHUTDOWN_TIMEOUT: ${{ secrets.EMBEDDING_GRACEFUL_SHUTDOWN_TIMEOUT }} + CRAWLING_GOOGLE_RETRIVAL_MAX_WORKERS: ${{ secrets.CRAWLING_GOOGLE_RETRIVAL_MAX_WORKERS }} + CRAWLING_GMAIL_MAX_WORKERS: ${{ secrets.CRAWLING_GMAIL_MAX_WORKERS }} + CRAWLING_DRIVE_MAX_WORKERS: ${{ secrets.CRAWLING_DRIVE_MAX_WORKERS }} + CRAWLING_GOOGLEDOCS_MAX_WORKERS: ${{ secrets.CRAWLING_GOOGLEDOCS_MAX_WORKERS }} + CRAWLING_GOOGLESLIDES_MAX_WORKERS: ${{ secrets.CRAWLING_GOOGLESLIDES_MAX_WORKERS }} + CRAWLING_GOOGLEPDF_MAX_WORKERS: ${{ secrets.CRAWLING_GOOGLEPDF_MAX_WORKERS }} + CRAWLING_MICROSOFT_RETRIVAL_MAX_WORKERS: ${{ secrets.CRAWLING_MICROSOFT_RETRIVAL_MAX_WORKERS }} + VECTOR_BATCH_SIZE: ${{ secrets.VECTOR_BATCH_SIZE }} + VECTOR_BATCH_TIMEOUT: ${{ secrets.VECTOR_BATCH_TIMEOUT }} + QUERY_NUMBER_OF_SOURCES: ${{ secrets.QUERY_NUMBER_OF_SOURCES }} + QUERY_TTL: ${{ secrets.QUERY_TTL }} + QUERY_QUEUE_TTL: ${{ secrets.QUERY_QUEUE_TTL }} + QUERY_EXPANSION: ${{ secrets.QUERY_EXPANSION }} + QUERY_SUMMARY_UPPER_BOUND: ${{ secrets.QUERY_SUMMARY_UPPER_BOUND }} + QUERY_SUMMARY_LOWER_BOUND: ${{ secrets.QUERY_SUMMARY_LOWER_BOUND }} + QUERY_DEFAULT_MODEL: ${{ secrets.QUERY_DEFAULT_MODEL }} + RETRIEVAL_K_VAL: ${{ secrets.RETRIEVAL_K_VAL }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ secrets.POSTGRES_DB }} + COUCHDB_USER: ${{ secrets.COUCHDB_USER }} + COUCHDB_PASSWORD: ${{ secrets.COUCHDB_PASSWORD }} + JWT_SECRET: ${{ secrets.JWT_SECRET }} + ARGON2_MEMORY: ${{ secrets.ARGON2_MEMORY }} + ARGON2_ITERATIONS: ${{ secrets.ARGON2_ITERATIONS }} + ARGON2_PARALLELISM: ${{ secrets.ARGON2_PARALLELISM }} + ARGON2_SALT_LENGTH: ${{ secrets.ARGON2_SALT_LENGTH }} + ARGON2_KEY_LENGTH: ${{ secrets.ARGON2_KEY_LENGTH }} + MIN_PASSWORD_LENGTH: ${{ secrets.MIN_PASSWORD_LENGTH }} + MAX_PASSWORD_LENGTH: ${{ secrets.MAX_PASSWORD_LENGTH }} + MAX_EMAIL_LENGTH: ${{ secrets.MAX_EMAIL_LENGTH }} + RABBITMQ_URL: ${{ secrets.RABBITMQ_URL }} + RABBITMQ_DEFAULT_USER: ${{ secrets.RABBITMQ_DEFAULT_USER }} + RABBITMQ_DEFAULT_PASS: ${{ secrets.RABBITMQ_DEFAULT_PASS }} + RABBITMQ_LOGS: ${{ secrets.RABBITMQ_LOGS }} + KAFKA_BROKER_ADDRESS: ${{ secrets.KAFKA_BROKER_ADDRESS }} + KAFKA_BROKER_ID: ${{ secrets.KAFKA_BROKER_ID }} + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: ${{ secrets.KAFKA_LISTENER_SECURITY_PROTOCOL_MAP }} + KAFKA_ADVERTISED_LISTENERS: ${{ secrets.KAFKA_ADVERTISED_LISTENERS }} + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: ${{ secrets.KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR }} + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: ${{ secrets.KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS }} + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: ${{ secrets.KAFKA_TRANSACTION_STATE_LOG_MIN_ISR }} + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: ${{ secrets.KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR }} + KAFKA_PROCESS_ROLES: ${{ secrets.KAFKA_PROCESS_ROLES }} + KAFKA_NODE_ID: ${{ secrets.KAFKA_NODE_ID }} + KAFKA_CONTROLLER_QUORUM_VOTERS: ${{ secrets.KAFKA_CONTROLLER_QUORUM_VOTERS }} + KAFKA_LISTENERS: ${{ secrets.KAFKA_LISTENERS }} + KAFKA_INTER_BROKER_LISTENER_NAME: ${{ secrets.KAFKA_INTER_BROKER_LISTENER_NAME }} + KAFKA_CONTROLLER_LISTENER_NAMES: ${{ secrets.KAFKA_CONTROLLER_LISTENER_NAMES }} + CLUSTER_ID: ${{ secrets.CLUSTER_ID }} + KAFKA_LOG4J_ROOT_LOGLEVEL: ${{ secrets.KAFKA_LOG4J_ROOT_LOGLEVEL }} + KAFKA_LOG4J_LOGGERS: ${{ secrets.KAFKA_LOG4J_LOGGERS }} + OLLAMA_URL: ${{ secrets.OLLAMA_URL }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LLM_MODEL: ${{ secrets.LLM_MODEL }} + ZILLIZ_ADDRESS: ${{ secrets.ZILLIZ_ADDRESS }} + ZILLIZ_API_KEY: ${{ secrets.ZILLIZ_API_KEY }} + VECTOR_DIMENSION: ${{ secrets.VECTOR_DIMENSION }} + GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} + GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} + GOOGLE_AUTH_URL: ${{ secrets.GOOGLE_AUTH_URL }} + GOOGLE_REDIRECT_URI: ${{ secrets.GOOGLE_REDIRECT_URI }} + GOOGLE_SCOPES: ${{ secrets.GOOGLE_SCOPES }} + GOOGLE_SSO_CLIENT_ID: ${{ secrets.GOOGLE_SSO_CLIENT_ID }} + GOOGLE_SSO_CLIENT_SECRET: ${{ secrets.GOOGLE_SSO_CLIENT_SECRET }} + GOOGLE_SSO_REDIRECT_URI: ${{ secrets.GOOGLE_SSO_REDIRECT_URI }} + GOOGLE_SSO_SCOPES: ${{ secrets.GOOGLE_SSO_SCOPES }} + MICROSOFT_CLIENT_ID: ${{ secrets.MICROSOFT_CLIENT_ID }} + MICROSOFT_CLIENT_SECRET: ${{ secrets.MICROSOFT_CLIENT_SECRET }} + MICROSOFT_AUTH_URL: ${{ secrets.MICROSOFT_AUTH_URL }} + MICROSOFT_REDIRECT_URI: ${{ secrets.MICROSOFT_REDIRECT_URI }} + MICROSOFT_SCOPES: ${{ secrets.MICROSOFT_SCOPES }} + NOTION_CLIENT_ID: ${{ secrets.NOTION_CLIENT_ID }} + NOTION_CLIENT_SECRET: ${{ secrets.NOTION_CLIENT_SECRET }} + NOTION_REDIRECT_URI: ${{ secrets.NOTION_REDIRECT_URI }} + NOTION_AUTH_URL: ${{ secrets.NOTION_AUTH_URL }} + REDIS_ADDRESS: ${{ secrets.REDIS_ADDRESS }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} + CA_CRT: ${{ secrets.CA_CRT }} + CA_KEY: ${{ secrets.CA_KEY }} + QUERY_CRT: ${{ secrets.QUERY_CRT }} + QUERY_KEY: ${{ secrets.QUERY_KEY }} + AUTH_CRT: ${{ secrets.AUTH_CRT }} + AUTH_KEY: ${{ secrets.AUTH_KEY }} + VECTOR_CRT: ${{ secrets.VECTOR_CRT }} + VECTOR_KEY: ${{ secrets.VECTOR_KEY }} + GATEWAY_CRT: ${{ secrets.GATEWAY_CRT }} + GATEWAY_KEY: ${{ secrets.GATEWAY_KEY }} + DESKTOP_CRT: ${{ secrets.DESKTOP_CRT }} + DESKTOP_KEY: ${{ secrets.DESKTOP_KEY }} + EMBEDDING_CRT: ${{ secrets.EMBEDDING_CRT }} + EMBEDDING_KEY: ${{ secrets.EMBEDDING_KEY }} + RETRIEVAL_CRT: ${{ secrets.RETRIEVAL_CRT }} + RETRIEVAL_KEY: ${{ secrets.RETRIEVAL_KEY }} + MQTT_CRT: ${{ secrets.MQTT_CRT }} + MQTT_KEY: ${{ secrets.MQTT_KEY }} + WAITLIST_CRT: ${{ secrets.WAITLIST_CRT }} + WAITLIST_KEY: ${{ secrets.WAITLIST_KEY }} + INTEGRATION_CRT: ${{ secrets.INTEGRATION_CRT }} + INTEGRATION_KEY: ${{ secrets.INTEGRATION_KEY }} + CRAWLING_CRT: ${{ secrets.CRAWLING_CRT }} + CRAWLING_KEY: ${{ secrets.CRAWLING_KEY }} + ALLOWED_CLIENT_IP: ${{ secrets.ALLOWED_CLIENT_IP }} + SMTP_FROM: ${{ secrets.SMTP_FROM }} + SMTP_USER: ${{ secrets.SMTP_USER }} + SMTP_PASS: ${{ secrets.SMTP_PASS }} + SMTP_HOST: ${{ secrets.SMTP_HOST }} + SMTP_PORT: ${{ secrets.SMTP_PORT }} + EC2_KEY_PAIR: ${{ secrets.EC2_KEY_PAIR }} + EC2_PUBLIC_IP: ${{ secrets.EC2_PUBLIC_IP }} + + run: | + chmod +x ./generate_env.sh + ./generate_env.sh + + - name: Generate Code + run: | + cd backend/common && make gen + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + run: | + aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com + + - name: Build and Push Docker Images + run: | + docker build -t authentication:$IMAGE_TAG -f backend/authentication/Dockerfile backend + docker tag authentication:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG + + docker build -t query:$IMAGE_TAG -f backend/query/Dockerfile backend + docker tag query:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG + + docker build -t gateway:$IMAGE_TAG -f backend/gateway/Dockerfile backend + docker tag gateway:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG + + docker build -t waitlist:$IMAGE_TAG -f backend/waitlist/Dockerfile backend + docker tag waitlist:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/waitlist:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/waitlist:$IMAGE_TAG + + docker build -t init:$IMAGE_TAG -f backend/init/Dockerfile backend + docker tag init:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG + + docker build -t embedding:$IMAGE_TAG -f backend/embedding/Dockerfile backend + docker tag embedding:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG + + docker build -t retrieval:$IMAGE_TAG -f backend/retrieval/Dockerfile backend + docker tag retrieval:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG + + docker build -t vector:$IMAGE_TAG -f backend/vector/Dockerfile backend + docker tag vector:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG + + docker build -t desktop:$IMAGE_TAG -f backend/desktop/Dockerfile backend + docker tag desktop:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG + + docker build -t crawling:$IMAGE_TAG -f backend/crawling/Dockerfile backend + docker tag crawling:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG + + docker build -t integration:$IMAGE_TAG -f backend/integration/Dockerfile backend + docker tag integration:$IMAGE_TAG ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/integration:$IMAGE_TAG + docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/integration:$IMAGE_TAG + + - name: Deploy to EC2 + run: | + echo ${{ secrets.EC2_KEY_PAIR }} | tr -d '\r' > private_key.pem + chmod 600 private_key.pem + echo "length of private key: $(wc -c < private_key.pem)" + # Install AWS CLI if not already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + sudo apt-get update + sudo apt-get install -y awscli + fi + + # Copy the Docker image to the EC2 instance + aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/waitlist:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG + docker pull ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/integration:$IMAGE_TAG + + echo "Content of the PEM file:" + + # SSH into the EC2 instance and run the Docker containers + ssh -v -o StrictHostKeyChecking=no -i private_key.pem ec2-user@${{ secrets.EC2_PUBLIC_IP }} << 'EOF' + docker run -d --restart always --name authentication -p 80:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG + docker run -d --restart always --name query -p 81:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG + docker run -d --restart always --name gateway -p 82:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG + docker run -d --restart always --name waitlist -p 83:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/waitlist:$IMAGE_TAG + docker run -d --restart always --name init -p 84:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG + docker run -d --restart always --name embedding -p 85:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG + docker run -d --restart always --name retrieval -p 86:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG + docker run -d --restart always --name vector -p 87:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG + docker run -d --restart always --name desktop -p 88:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG + docker run -d --restart always --name crawling -p 89:80 ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG + EOF + diff --git a/.github/workflows/EC2_deploy.yaml b/.github/workflows/EC2_deploy.yaml new file mode 100644 index 00000000..12f131fe --- /dev/null +++ b/.github/workflows/EC2_deploy.yaml @@ -0,0 +1,293 @@ +name: Deploy to Amazon EC2 + +on: + push: + branches: + - cicd # Trigger deployment on the cicd branch + +env: + AWS_REGION: us-east-1 + ECR_REPOSITORY_NAMESPACE: indeq + IMAGE_TAG: ${{ github.run_id }} + # IMAGE_TAG: "14588128148" + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + id-token: write # For OIDC auth (recommended) + contents: read + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + # comment out starting here + - name: Install Protocol Buffers Compiler + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - name: Install Go + run: | + sudo apt-get update + sudo apt-get install -y golang-go + + - name: Install protoc-gen-go + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@latest + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV + + - name: Install protoc-gen-go-grpc + run: | + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV + + - name: Check protoc-gen-go Installation + run: | + protoc-gen-go --version || echo "protoc-gen-go not found" + # comment out ending here + + - name: Create .env file + run: | + # first delete the .env file + rm backend/common/config/.env + + # Use cat with heredoc to prevent command interpretation + cat << 'EOL' > backend/common/config/.env + ${{ secrets.ENV1 }} + ${{ secrets.ENV2 }} + ${{ secrets.ENV3 }} + ${{ secrets.ENV4 }} + ${{ secrets.ENV5 }} + ${{ secrets.ENV6 }} + ${{ secrets.ENV7 }} + EOL + + # comment out starting here + - name: Generate Code + run: | + cd backend/common && make gen + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + # comment out ending here + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + + - name: Login to Amazon ECR + run: | + aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com + + - name: Extract EC2 Key Pair + run: | + # Extract markers + BEGIN_MARKER="-----BEGIN RSA PRIVATE KEY-----" + END_MARKER="-----END RSA PRIVATE KEY-----" + + # Create PEM file with proper formatting + echo "$BEGIN_MARKER" > private_key.pem + echo "${{ secrets.EC2_KEY_PAIR }}" | sed 's/.\{64\}/&\n/g' >> private_key.pem + echo "$END_MARKER" >> private_key.pem + + chmod 600 private_key.pem + + - name: Predeploy Cleanup + run: | + + # showing private key + cat private_key.pem + + # SSH into EC2 to create directories first for clean up + ssh -vvv -o StrictHostKeyChecking=no -i private_key.pem ec2-user@${{ secrets.EC2_PUBLIC_IP }} << 'EOF' + + # Print current directory and list contents + pwd + + # Install Docker if not already installed + if ! command -v docker &> /dev/null; then + echo "Docker not found. Installing..." + sudo yum update -y + sudo yum install -y docker + sudo systemctl start docker + sudo systemctl enable docker + sudo usermod -aG docker ec2-user + echo "Docker installed successfully" + fi + + # Create necessary directories if they don't exist + if [ ! -d "/home/ec2-user/common/config" ]; then + mkdir -p /home/ec2-user/common/config + fi + + # If directory exists, remove .env inside + if [ -f "/home/ec2-user/common/config/.env" ]; then + rm /home/ec2-user/common/config/.env + fi + + # Stop and remove containers safely + if [ "$(docker ps -q)" ]; then + docker stop $(docker ps -q) + fi + if [ "$(docker ps -aq)" ]; then + docker rm $(docker ps -aq) + fi + + docker image prune -f + docker network prune -f + EOF + + - name: Copy env file to EC2 + run: | + # Copy the env file to EC2 + scp -o StrictHostKeyChecking=no -i private_key.pem backend/common/config/.env ec2-user@${{ secrets.EC2_PUBLIC_IP }}:/home/ec2-user/common/config/.env + + - name: Build Docker Images + run: | + authImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG" + queryImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG" + gatewayImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG" + initImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG" + embeddingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG" + retrievalImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG" + vectorImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG" + desktopImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG" + integrationImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/integration:$IMAGE_TAG" + crawlingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG" + mqttImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/mqtt:$IMAGE_TAG" + + docker build -t authentication:$IMAGE_TAG -f backend/authentication/Dockerfile backend + docker tag authentication:$IMAGE_TAG $authImage + + docker build -t query:$IMAGE_TAG -f backend/query/Dockerfile backend + docker tag query:$IMAGE_TAG $queryImage + + docker build -t gateway:$IMAGE_TAG -f backend/gateway/Dockerfile backend + docker tag gateway:$IMAGE_TAG $gatewayImage + + docker build -t init:$IMAGE_TAG -f backend/init/Dockerfile backend + docker tag init:$IMAGE_TAG $initImage + + docker build -t embedding:$IMAGE_TAG -f backend/embedding/Dockerfile backend + docker tag embedding:$IMAGE_TAG $embeddingImage + + docker build -t retrieval:$IMAGE_TAG -f backend/retrieval/Dockerfile backend + docker tag retrieval:$IMAGE_TAG $retrievalImage + + docker build -t vector:$IMAGE_TAG -f backend/vector/Dockerfile backend + docker tag vector:$IMAGE_TAG $vectorImage + + docker build -t desktop:$IMAGE_TAG -f backend/desktop/Dockerfile backend + docker tag desktop:$IMAGE_TAG $desktopImage + + docker build -t integration:$IMAGE_TAG -f backend/integration/Dockerfile backend + docker tag integration:$IMAGE_TAG $integrationImage + + docker build -t crawling:$IMAGE_TAG -f backend/crawling/Dockerfile backend + docker tag crawling:$IMAGE_TAG $crawlingImage + + docker build -t mqtt:$IMAGE_TAG -f backend/mqtt/Dockerfile backend + docker tag mqtt:$IMAGE_TAG $mqttImage + + - name: Push Docker Images + run: | + authImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/authentication:$IMAGE_TAG" + queryImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/query:$IMAGE_TAG" + gatewayImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/gateway:$IMAGE_TAG" + initImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/init:$IMAGE_TAG" + embeddingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/embedding:$IMAGE_TAG" + retrievalImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/retrieval:$IMAGE_TAG" + vectorImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/vector:$IMAGE_TAG" + desktopImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/desktop:$IMAGE_TAG" + integrationImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/integration:$IMAGE_TAG" + crawlingImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/crawling:$IMAGE_TAG" + mqttImage="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPOSITORY_NAMESPACE/mqtt:$IMAGE_TAG" + + docker push $authImage + docker push $queryImage + docker push $gatewayImage + docker push $initImage + docker push $embeddingImage + docker push $retrievalImage + docker push $vectorImage + docker push $desktopImage + docker push $integrationImage + docker push $crawlingImage + docker push $mqttImage + + # Replace placeholder image references with actual ECR URLs + sed -i "s|authImage|$authImage|g" backend/docker-compose-ec2.yaml + sed -i "s|queryImage|$queryImage|g" backend/docker-compose-ec2.yaml + sed -i "s|gatewayImage|$gatewayImage|g" backend/docker-compose-ec2.yaml + sed -i "s|initImage|$initImage|g" backend/docker-compose-ec2.yaml + sed -i "s|embeddingImage|$embeddingImage|g" backend/docker-compose-ec2.yaml + sed -i "s|retrievalImage|$retrievalImage|g" backend/docker-compose-ec2.yaml + sed -i "s|vectorImage|$vectorImage|g" backend/docker-compose-ec2.yaml + sed -i "s|desktopImage|$desktopImage|g" backend/docker-compose-ec2.yaml + sed -i "s|integrationImage|$integrationImage|g" backend/docker-compose-ec2.yaml + sed -i "s|crawlingImage|$crawlingImage|g" backend/docker-compose-ec2.yaml + sed -i "s|mqttImage|$mqttImage|g" backend/docker-compose-ec2.yaml + + # Checking if the docker-compose.yaml file is updated + cat backend/docker-compose-ec2.yaml + echo "TESTING" + + - name: Copy docker-compose file to EC2 + run: | + # Copy the docker-compose file to EC2 + # first echo file content + cat backend/docker-compose-ec2.yaml + + scp -o StrictHostKeyChecking=no -i private_key.pem backend/docker-compose-ec2.yaml ec2-user@${{ secrets.EC2_PUBLIC_IP }}:/home/ec2-user/ + + - name: Deploy to EC2 + run: | + # Install AWS CLI if not already installed + if ! command -v aws &> /dev/null; then + echo "AWS CLI not found. Installing..." + sudo apt-get update + sudo apt-get install -y awscli + fi + + # SSH into the EC2 instance and run the Docker containers + ssh -v -o StrictHostKeyChecking=no -i private_key.pem ec2-user@${{ secrets.EC2_PUBLIC_IP }} << 'EOF' + # Install Docker if not already installed + if ! command -v docker &> /dev/null; then + echo "Docker not found. Installing..." + sudo yum update -y + sudo yum install -y docker + sudo systemctl start docker + sudo systemctl enable docker + sudo usermod -aG docker ec2-user + echo "Docker installed successfully" + fi + + # Install Docker Compose V2 if not present + if ! command -v docker-compose &> /dev/null && ! command -v docker compose &> /dev/null; then + echo "Installing Docker Compose V2..." + # For Amazon Linux 2023 + sudo mkdir -p /usr/local/lib/docker/cli-plugins + sudo curl -SL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + fi + + # Configure AWS CLI + aws configure set region us-east-1 + + # Verify IAM role + aws sts get-caller-identity + + # Login to ECR without credentials (using IAM role) + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com + + # Run docker compose with absolute path + docker compose -f /home/ec2-user/docker-compose-ec2.yaml up -d + EOF diff --git a/.gitignore b/.gitignore index 7a5ebef5..42b1332f 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,8 @@ # Environment variables .env.local +backend-config.yaml +root_ca.crt # Log files *.log diff --git a/backend/common/config/.env b/backend/common/config/.env index 362901d4..8cf9e8c3 100644 Binary files a/backend/common/config/.env and b/backend/common/config/.env differ diff --git a/backend/docker-compose-ec2.yaml b/backend/docker-compose-ec2.yaml new file mode 100644 index 00000000..081758b5 --- /dev/null +++ b/backend/docker-compose-ec2.yaml @@ -0,0 +1,225 @@ +services: + rabbitmq: + image: rabbitmq:4.0-management + container_name: rabbitmq + env_file: + - /home/ec2-user/common/config/.env + ports: + - 5672:5672 # AMQP protocol port + - 15672:15672 # Management UI port + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "check_port_connectivity"] + interval: 5s + timeout: 5s + retries: 5 + networks: + - indeq-net + + kafka: + image: apache/kafka:3.9.0 + container_name: kafka + env_file: + - /home/ec2-user/common/config/.env + networks: + - indeq-net + healthcheck: + test: + [ + "CMD-SHELL", + "/opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 --list", + ] + interval: 5s + timeout: 5s + retries: 5 + + init: + image: initImage + restart: "no" + env_file: + - /home/ec2-user/common/config/.env + depends_on: + kafka: + condition: service_healthy + couchdb: + condition: service_healthy + networks: + - indeq-net + + embedding: + image: embeddingImage + env_file: + - /home/ec2-user/common/config/.env + volumes: + - embedding_data:/app/model_cache + networks: + - indeq-net + + mqtt: + image: mqttImage + container_name: mqtt-service + ports: + - "8883:8883" + env_file: + - /home/ec2-user/common/config/.env + networks: + - indeq-net + + authentication: + image: authImage + env_file: + - /home/ec2-user/common/config/.env + depends_on: + appDB: + condition: service_healthy + vector: + condition: service_started + networks: + - indeq-net + + retrieval: + image: retrievalImage + depends_on: + desktop: + condition: service_started + crawling: + condition: service_started + vector: + condition: service_started + networks: + - indeq-net + + vector: + image: vectorImage + build: + context: . + dockerfile: vector/Dockerfile + depends_on: + init: + condition: service_completed_successfully + networks: + - indeq-net + + gateway: + image: gatewayImage + env_file: + - /home/ec2-user/common/config/.env + ports: + - "8080:8080" + depends_on: + query: + condition: service_started + rabbitmq: + condition: service_healthy + authentication: + condition: service_started + desktop: + condition: service_started + init: + condition: service_completed_successfully + networks: + - indeq-net + + desktop: + image: desktopImage + ports: + - "8081:8080" + depends_on: + init: + condition: service_completed_successfully + mqtt: + condition: service_started + networks: + - indeq-net + + query: + image: queryImage + depends_on: + rabbitmq: + condition: service_healthy + networks: + - indeq-net + + crawling: + image: crawlingImage + env_file: + - /home/ec2-user/common/config/.env + depends_on: + init: + condition: service_completed_successfully + appDB: + condition: service_healthy + networks: + - indeq-net + + integration: + image: integrationImage + env_file: /home/ec2-user/common/config/.env + depends_on: + appDB: + condition: service_healthy + redis: + condition: service_started + networks: + - indeq-net + + redis: + image: redis:alpine + env_file: + - /home/ec2-user/common/config/.env + command: ["sh", "-c", 'exec redis-server --requirepass "$$REDIS_PASSWORD"'] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "-a", "$$REDIS_PASSWORD", "ping"] + interval: 5s + timeout: 3s + retries: 3 + networks: + - indeq-net + + appDB: + image: postgres:latest + env_file: + - /home/ec2-user/common/config/.env + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 5s + retries: 5 + command: + - -c + - ssl=off + expose: + - "5432" + volumes: + - app_data:/var/lib/postgresql/data + networks: + - indeq-net + + couchdb: + image: couchdb + env_file: + - /home/ec2-user/common/config/.env + restart: always + volumes: + - couch_data:/opt/couchdb/data + healthcheck: + test: ["CMD-SHELL", "curl --silent --fail http://localhost:5984 || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + networks: + - indeq-net + +volumes: + app_data: + embedding_data: + redis_data: + couch_data: + +networks: + indeq-net: + driver: bridge + labels: + com.docker.network.dns.name: docker.internal + diff --git a/backend/k8s-manifest.yml b/backend/k8s-manifest.yml new file mode 100644 index 00000000..69d69b66 --- /dev/null +++ b/backend/k8s-manifest.yml @@ -0,0 +1,288 @@ +# Ingress + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: indeq-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + rules: + - http: + paths: + - path: /gateway + pathType: Prefix + backend: + service: + name: gateway-service + port: + number: 8080 + - path: /query + pathType: Prefix + backend: + service: + name: query-service + port: + number: 8080 + - path: /auth + pathType: Prefix + backend: + service: + name: authentication-service + port: + number: 8080 + - path: /ollama + pathType: Prefix + backend: + service: + name: ollama-service + port: + number: 11434 + +--- +# ConfigMap for environment variables +apiVersion: v1 +kind: ConfigMap +metadata: + name: backend-config +data: + POSTGRES_USER: your_user + POSTGRES_DB: your_db + POSTGRES_PASSWORD: your_password + +--- +# Gateway Service +apiVersion: v1 +kind: Service +metadata: + name: gateway-service +spec: + selector: + app: gateway + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gateway-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: gateway + template: + metadata: + labels: + app: gateway + spec: + containers: + - name: gateway + image: gateway-service + # imagePullPolicy: Never + ports: + - containerPort: 8080 + +--- +# Query Service +apiVersion: v1 +kind: Service +metadata: + name: query-service +spec: + selector: + app: query + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: query-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: query + template: + metadata: + labels: + app: query + spec: + containers: + - name: query + image: query-service + # imagePullPolicy: Never + ports: + - containerPort: 8080 + +--- +# Authentication Service +apiVersion: v1 +kind: Service +metadata: + name: authentication-service +spec: + selector: + app: authentication + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: authentication-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: authentication + template: + metadata: + labels: + app: authentication + spec: + containers: + - name: authentication + image: authentication-service + # imagePullPolicy: Never + ports: + - containerPort: 8080 + +--- +# Ollama Service +apiVersion: v1 +kind: Service +metadata: + name: ollama-service +spec: + selector: + app: ollama + ports: + - port: 11434 + targetPort: 11434 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ollama-deployment +spec: + replicas: 1 # Single replica since this is a stateful workload + selector: + matchLabels: + app: ollama + template: + metadata: + labels: + app: ollama + spec: + containers: + - name: ollama + image: ollama/ollama:latest + # imagePullPolicy: Never + ports: + - containerPort: 11434 + resources: + limits: + memory: "16Gi" + env: + - name: OLLAMA_NUM_PARALLEL + value: "4" + volumeMounts: + - name: ollama-models + mountPath: /root/.ollama + volumes: + - name: ollama-models + emptyDir: {} + +--- +# RabbitMQ Service +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq-service +spec: + selector: + app: rabbitmq + ports: + - name: amqp + port: 5672 + targetPort: 5672 + - name: management + port: 15672 + targetPort: 15672 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq-deployment +spec: + replicas: 1 # Single replica since this is a stateful workload + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:3-management + # imagePullPolicy: Never + ports: + - containerPort: 5672 + - containerPort: 15672 + volumeMounts: + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + volumes: + - name: rabbitmq-data + emptyDir: {} + +--- +# PostgreSQL Service +apiVersion: v1 +kind: Service +metadata: + name: postgres-service +spec: + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment +spec: + replicas: 1 # Single replica since this is a stateful workload + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:latest + # imagePullPolicy: Never + ports: + - containerPort: 5432 + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + envFrom: + - configMapRef: + name: backend-config # Reference the ConfigMap here + volumes: + - name: postgres-data + emptyDir: {} + diff --git a/backend/powershell/deletesecrets.ps1 b/backend/powershell/deletesecrets.ps1 new file mode 100644 index 00000000..e3ce2637 --- /dev/null +++ b/backend/powershell/deletesecrets.ps1 @@ -0,0 +1,47 @@ +# List all secrets and delete them +function Remove-AllGitHubSecrets { + Write-Host "Fetching all secrets..." -ForegroundColor Yellow + + # Get all secrets + $secrets = gh secret list 2>&1 + + if ($LASTEXITCODE -ne 0) { + Write-Host "Error fetching secrets: $secrets" -ForegroundColor Red + exit 1 + } + + # Counter for tracking progress + $total = ($secrets | Measure-Object -Line).Lines + $current = 0 + + # Process each secret + foreach ($secret in $secrets) { + # Skip empty lines + if ([string]::IsNullOrWhiteSpace($secret)) { + continue + } + + # Extract secret name (first column from gh secret list output) + $secretName = ($secret -split '\s+')[0] + $current++ + + Write-Host "($current/$total) Deleting secret: $secretName" -ForegroundColor Yellow + + try { + $result = gh secret delete $secretName 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Successfully deleted secret: $secretName" -ForegroundColor Green + } else { + Write-Host "Failed to delete secret: $secretName - $result" -ForegroundColor Red + } + } + catch { + Write-Host "Error deleting secret: $secretName - $_" -ForegroundColor Red + } + } + + Write-Host "Completed deleting secrets" -ForegroundColor Green +} + +# Run the function +Remove-AllGitHubSecrets \ No newline at end of file diff --git a/backend/powershell/setsecrets.ps1 b/backend/powershell/setsecrets.ps1 new file mode 100644 index 00000000..f433e72a --- /dev/null +++ b/backend/powershell/setsecrets.ps1 @@ -0,0 +1,56 @@ +# Load the YAML file and extract the data section +$data = yq eval '.' ..\backend-config.yaml + +# Check if any secrets were found +if ($data.Count -eq 0) { + Write-Host "No secrets found in backend-config.yaml." + exit +} + +# Loop through each secret +foreach ($secret in $data) { + # Extract the raw secret string + $rawSecret = $secret + + # Trim the string and extract key and value + $trimmedSecret = $rawSecret.Trim() + $keyValue = $trimmedSecret -split ': ' + + # Check if the keyValue array has the expected number of elements + if ($keyValue.Count -lt 2) { + Write-Host "Warning: The entry '$rawSecret' does not contain a valid key-value pair. Skipping..." + continue + } + + # Extract the key and value + $name = $keyValue[0].Trim() # This will be the key + $value = $keyValue[1].Trim() # This will be the value if it exists + + # For EC2_KEY_PAIR, ensure proper line endings + if ($name -eq 'EC2_KEY_PAIR') { + # Read the key file + $keyContent = $value + + # Split the key into parts + $beginMarker = "-----BEGIN RSA PRIVATE KEY-----" + $endMarker = "-----END RSA PRIVATE KEY-----" + $keyBody = $keyContent -replace $beginMarker, "" -replace $endMarker, "" -replace "\s+", "" + + + # Reconstruct the key with proper format + $value = $keyBody + } + # Debug output to check the extracted values + $value = $value -replace '^"|"$', '' # Remove extra outer double quotes from the value + Write-Host "Extracted secret: Name='$name', Value='$value'" + # Check if the value is empty or commented out + if (-not $value -or $value -match '^\s*#') { + Write-Host "Warning: The value for secret '$name' is empty or commented out. Skipping..." + continue + } + + + # Set the secret in GitHub + gh secret set $name --body $value + +} \ No newline at end of file diff --git a/generate_env.sh b/generate_env.sh new file mode 100644 index 00000000..c86f4f13 --- /dev/null +++ b/generate_env.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Create .env files in the specified directories +echo "Creating .env files..." + +# Define the directories +declare -a dirs=("backend/gateway" "backend/authentication" "backend/query" "backend/common/config") + +# Loop through each directory and create the .env file +for dir in "${dirs[@]}"; do + echo "Creating .env in $dir" + { + echo "AUTH_PORT=${AUTH_PORT}" + echo "AUTH_ADDRESS=${AUTH_ADDRESS}" + echo "QUERY_PORT=${QUERY_PORT}" + echo "QUERY_ADDRESS=${QUERY_ADDRESS}" + echo "VECTOR_PORT=${VECTOR_PORT}" + echo "VECTOR_ADDRESS=${VECTOR_ADDRESS}" + echo "GATEWAY_ADDRESS=${GATEWAY_ADDRESS}" + + # Database parameters + echo "DATABASE_URL=${DATABASE_URL}" + echo "POSTGRES_USER=${POSTGRES_USER}" + echo "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}" + echo "POSTGRES_DB=${POSTGRES_DB}" + + echo "JWT_SECRET=${JWT_SECRET}" + + # Argon2 Parameters + echo "ARGON2_MEMORY=${ARGON2_MEMORY}" + echo "ARGON2_ITERATIONS=${ARGON2_ITERATIONS}" + echo "ARGON2_PARALLELISM=${ARGON2_PARALLELISM}" + echo "ARGON2_SALT_LENGTH=${ARGON2_SALT_LENGTH}" + echo "ARGON2_KEY_LENGTH=${ARGON2_KEY_LENGTH}" + + # Password and Email Constraints + echo "MIN_PASSWORD_LENGTH=${MIN_PASSWORD_LENGTH}" + echo "MAX_PASSWORD_LENGTH=${MAX_PASSWORD_LENGTH}" + echo "MAX_EMAIL_LENGTH=${MAX_EMAIL_LENGTH}" + + # RabbitMQ parameters + echo "RABBITMQ_URL=${RABBITMQ_URL}" + echo "RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}" + echo "RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}" + echo "RABBITMQ_LOGS=${RABBITMQ_LOGS}" + + # Ollama parameters + echo "OLLAMA_URL=${OLLAMA_URL}" + echo "LLM_MODEL=${LLM_MODEL}" + + # Zilliz parameters + echo "ZILLIZ_ADDRESS=${ZILLIZ_ADDRESS}" + echo "ZILLIZ_API_KEY=${ZILLIZ_API_KEY}" + + # TLS parameters in base 64 + echo "CA_CRT=${CA_CRT}" + echo "QUERY_CRT=${QUERY_CRT}" + echo "QUERY_KEY=${QUERY_KEY}" + echo "AUTH_CRT=${AUTH_CRT}" + echo "AUTH_KEY=${AUTH_KEY}" + echo "GATEWAY_CRT=${GATEWAY_CRT}" + echo "GATEWAY_KEY=${GATEWAY_KEY}" + + # CORS parameters + echo "ALLOWED_CLIENT_IP=${ALLOWED_CLIENT_IP}" + } > "$dir/.env" +done + +echo ".env files created successfully." \ No newline at end of file