Fix Gemini Handling Youtube URLs

.github/workflows/deploy.yml

@@ -0,0 +1,183 @@
+name: Build and Deploy Cortex
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-cortex:
+    name: Build Cortex Image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,prefix=
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build and push Cortex image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  build-file-handler:
+    name: Build File Handler Image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-file-handler
+          tags: |
+            type=sha,prefix=
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build and push File Handler image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./helper-apps/cortex-file-handler
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  deploy:
+    name: Deploy to Hetzner
+    runs-on: ubuntu-latest
+    needs: [build-cortex, build-file-handler]
+    environment: production
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Copy docker-compose to server
+        uses: appleboy/scp-action@v0.1.7
+        with:
+          host: ${{ secrets.DEPLOY_HOST }}
+          username: ${{ secrets.DEPLOY_USER }}
+          key: ${{ secrets.DEPLOY_SSH_KEY }}
+          source: "docker-compose.prod.yml"
+          target: "/opt/cortex"
+
+      - name: Deploy to server
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.DEPLOY_HOST }}
+          username: ${{ secrets.DEPLOY_USER }}
+          key: ${{ secrets.DEPLOY_SSH_KEY }}
+          script: |
+            cd /opt/cortex
+
+            # Repository name must be lowercase for Docker
+            REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
+
+            # Login to GitHub Container Registry
+            echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+            # Pull the new images
+            docker pull ghcr.io/${REPO_NAME}:latest
+            docker pull ghcr.io/${REPO_NAME}-file-handler:latest
+
+            # Update the stack
+            export IMAGE_TAG=latest
+            export GITHUB_REPOSITORY_CORTEX=${REPO_NAME}
+
+            # Load environment variables
+            set -a
+            source .env
+            set +a
+
+            # Deploy with docker-compose
+            docker compose -f docker-compose.prod.yml up -d --remove-orphans
+
+            # Clean up old images
+            docker image prune -f
+
+      - name: Health check
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.DEPLOY_HOST }}
+          username: ${{ secrets.DEPLOY_USER }}
+          key: ${{ secrets.DEPLOY_SSH_KEY }}
+          script: |
+            echo "Waiting for services to start..."
+            sleep 15
+
+            # Check Cortex
+            if docker ps | grep -q cortex; then
+              echo "✅ Cortex is running"
+            else
+              echo "❌ Cortex failed to start"
+              docker logs cortex --tail 30
+              exit 1
+            fi
+
+            # Check File Handler
+            if docker ps | grep -q cortex-file-handler; then
+              echo "✅ File Handler is running"
+            else
+              echo "❌ File Handler failed to start"
+              docker logs cortex-file-handler --tail 30
+              exit 1
+            fi
+
+            # Check Redis
+            if docker ps | grep -q redis; then
+              echo "✅ Redis is running"
+            else
+              echo "❌ Redis failed to start"
+              exit 1
+            fi
+
+

.gitignore

@@ -1,6 +1,8 @@
 .DS_Store
 node_modules/
 .env
+.env.*
+deploy/env.production
 .vscode/
 **/__pycache__
 **/.venv

Dockerfile

@@ -0,0 +1,28 @@
+FROM node:18-alpine
+
+WORKDIR /app
+
+# Install build dependencies
+RUN apk add --no-cache python3 make g++
+
+# Copy package files
+COPY package*.json ./
+
+# Install production dependencies only
+RUN npm ci --omit=dev
+
+# Copy application code
+COPY . .
+
+# Remove devDependencies build tools
+RUN apk del python3 make g++
+
+# Expose GraphQL port
+EXPOSE 4000
+
+# Health check - uses /healthcheck which bypasses API key auth
+HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
+    CMD wget -qO- http://localhost:4000/healthcheck || exit 1
+
+CMD ["npm", "start"]
+

FILE_SYSTEM_DOCUMENTATION.md

@@ -650,10 +650,11 @@ RemoveFileFromCollection Tool
 - **Lifecycle**: Azure automatically deletes `retention=temporary` files after 30 days
 
 #### Google Cloud Storage (Optional)
-- **Enabled**: If `GCP_SERVICE_ACCOUNT_KEY` configured
+- **Enabled**: If `GCP_SERVICE_ACCOUNT_EMAIL` (recommended, uses impersonation) or `GCP_SERVICE_ACCOUNT_KEY` (legacy) is configured
 - **URL Format**: `gs://bucket/path`
 - **Usage**: Media file chunks, converted files
 - **No short-lived URLs**: GCS URLs are permanent (no SAS equivalent)
+- **Authentication**: See `GCP_IMPERSONATION_SETUP.md` for setup instructions using service account impersonation (recommended method)
 
 #### Local Storage (Fallback)
 - **Used**: If Azure not configured

README.md

@@ -861,7 +861,8 @@ The following properties can be configured through environment variables or the
 - `enableDuplicateRequests`: Enable sending duplicate requests if not completed after timeout. Default is true.
 - `enableGraphqlCache`: Enable GraphQL query caching. Default is false.
 - `enableRestEndpoints`: Create REST endpoints for pathways as well as GraphQL queries. Default is false.
-- `gcpServiceAccountKey`: GCP service account key for authentication. Default is null.
+- `gcpServiceAccountKey`: GCP service account key for authentication (legacy method). Default is null.
+- `gcpServiceAccountEmail`: GCP service account email for impersonation (recommended method). Default is null. See `GCP_IMPERSONATION_SETUP.md` for setup instructions.
 - `models`: Object containing the different models used by the project.
 - `pathways`: Object containing pathways for the project.
 - `pathwaysPath`: Path to custom pathways. Default is './pathways'.

config.js

@@ -130,6 +130,12 @@ var config = convict({
         env: 'GCP_SERVICE_ACCOUNT_KEY',
         sensitive: true
     },
+    gcpServiceAccountEmail: {
+        format: String,
+        default: null,
+        env: 'GCP_SERVICE_ACCOUNT_EMAIL',
+        sensitive: false
+    },
     azureServicePrincipalCredentials: {
         format: String,
         default: null,
@@ -941,7 +947,7 @@ if (config.get('entityConstants') && defaultEntityConstants) {
     config.set('entityConstants', { ...defaultEntityConstants, ...config.get('entityConstants') });
 }
 
-if (config.get('gcpServiceAccountKey')) {
+if (config.get('gcpServiceAccountEmail') || config.get('gcpServiceAccountKey')) {
     const gcpAuthTokenHelper = new GcpAuthTokenHelper(config.getProperties());
     config.set('gcpAuthTokenHelper', gcpAuthTokenHelper);
 }