diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..fda45bde --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,183 @@ +name: Build and Deploy Cortex + +on: + push: + branches: + - main + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-cortex: + name: Build Cortex Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,prefix= + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Cortex image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-file-handler: + name: Build File Handler Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-file-handler + tags: | + type=sha,prefix= + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push File Handler image + uses: docker/build-push-action@v5 + with: + context: ./helper-apps/cortex-file-handler + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + name: Deploy to Hetzner + runs-on: ubuntu-latest + needs: [build-cortex, build-file-handler] + environment: production + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Copy docker-compose to server + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "docker-compose.prod.yml" + target: "/opt/cortex" + + - name: Deploy to server + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + cd /opt/cortex + + # Repository name must be lowercase for Docker + REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + + # Login to GitHub Container Registry + echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + # Pull the new images + docker pull ghcr.io/${REPO_NAME}:latest + docker pull ghcr.io/${REPO_NAME}-file-handler:latest + + # Update the stack + export IMAGE_TAG=latest + export GITHUB_REPOSITORY_CORTEX=${REPO_NAME} + + # Load environment variables + set -a + source .env + set +a + + # Deploy with docker-compose + docker compose -f docker-compose.prod.yml up -d --remove-orphans + + # Clean up old images + docker image prune -f + + - name: Health check + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + echo "Waiting for services to start..." + sleep 15 + + # Check Cortex + if docker ps | grep -q cortex; then + echo "✅ Cortex is running" + else + echo "❌ Cortex failed to start" + docker logs cortex --tail 30 + exit 1 + fi + + # Check File Handler + if docker ps | grep -q cortex-file-handler; then + echo "✅ File Handler is running" + else + echo "❌ File Handler failed to start" + docker logs cortex-file-handler --tail 30 + exit 1 + fi + + # Check Redis + if docker ps | grep -q redis; then + echo "✅ Redis is running" + else + echo "❌ Redis failed to start" + exit 1 + fi + + diff --git a/.gitignore b/.gitignore index c2a8d18f..d87d5d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .DS_Store node_modules/ .env +.env.* +deploy/env.production .vscode/ **/__pycache__ **/.venv diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..14fa4154 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM node:18-alpine + +WORKDIR /app + +# Install build dependencies +RUN apk add --no-cache python3 make g++ + +# Copy package files +COPY package*.json ./ + +# Install production dependencies only +RUN npm ci --omit=dev + +# Copy application code +COPY . . + +# Remove devDependencies build tools +RUN apk del python3 make g++ + +# Expose GraphQL port +EXPOSE 4000 + +# Health check - uses /healthcheck which bypasses API key auth +HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ + CMD wget -qO- http://localhost:4000/healthcheck || exit 1 + +CMD ["npm", "start"] + diff --git a/FILE_SYSTEM_DOCUMENTATION.md b/FILE_SYSTEM_DOCUMENTATION.md index 8b552390..a1f35840 100644 --- a/FILE_SYSTEM_DOCUMENTATION.md +++ b/FILE_SYSTEM_DOCUMENTATION.md @@ -650,10 +650,11 @@ RemoveFileFromCollection Tool - **Lifecycle**: Azure automatically deletes `retention=temporary` files after 30 days #### Google Cloud Storage (Optional) -- **Enabled**: If `GCP_SERVICE_ACCOUNT_KEY` configured +- **Enabled**: If `GCP_SERVICE_ACCOUNT_EMAIL` (recommended, uses impersonation) or `GCP_SERVICE_ACCOUNT_KEY` (legacy) is configured - **URL Format**: `gs://bucket/path` - **Usage**: Media file chunks, converted files - **No short-lived URLs**: GCS URLs are permanent (no SAS equivalent) +- **Authentication**: See `GCP_IMPERSONATION_SETUP.md` for setup instructions using service account impersonation (recommended method) #### Local Storage (Fallback) - **Used**: If Azure not configured diff --git a/README.md b/README.md index 4c5896f0..8d6cc918 100644 --- a/README.md +++ b/README.md @@ -861,7 +861,8 @@ The following properties can be configured through environment variables or the - `enableDuplicateRequests`: Enable sending duplicate requests if not completed after timeout. Default is true. - `enableGraphqlCache`: Enable GraphQL query caching. Default is false. - `enableRestEndpoints`: Create REST endpoints for pathways as well as GraphQL queries. Default is false. -- `gcpServiceAccountKey`: GCP service account key for authentication. Default is null. +- `gcpServiceAccountKey`: GCP service account key for authentication (legacy method). Default is null. +- `gcpServiceAccountEmail`: GCP service account email for impersonation (recommended method). Default is null. See `GCP_IMPERSONATION_SETUP.md` for setup instructions. - `models`: Object containing the different models used by the project. - `pathways`: Object containing pathways for the project. - `pathwaysPath`: Path to custom pathways. Default is './pathways'. diff --git a/config.js b/config.js index 924f62c8..2325f55e 100644 --- a/config.js +++ b/config.js @@ -130,6 +130,12 @@ var config = convict({ env: 'GCP_SERVICE_ACCOUNT_KEY', sensitive: true }, + gcpServiceAccountEmail: { + format: String, + default: null, + env: 'GCP_SERVICE_ACCOUNT_EMAIL', + sensitive: false + }, azureServicePrincipalCredentials: { format: String, default: null, @@ -941,7 +947,7 @@ if (config.get('entityConstants') && defaultEntityConstants) { config.set('entityConstants', { ...defaultEntityConstants, ...config.get('entityConstants') }); } -if (config.get('gcpServiceAccountKey')) { +if (config.get('gcpServiceAccountEmail') || config.get('gcpServiceAccountKey')) { const gcpAuthTokenHelper = new GcpAuthTokenHelper(config.getProperties()); config.set('gcpAuthTokenHelper', gcpAuthTokenHelper); } diff --git a/config/default.json b/config/default.json index 9e26dfee..e0b116cb 100644 --- a/config/default.json +++ b/config/default.json @@ -1 +1,680 @@ -{} \ No newline at end of file +{ + "defaultModelName": "oai-gpt41", + "models": { + "xai-grok-4": { + "type": "GROK-VISION", + "emulateOpenAIChatModel": "grok-4", + "restStreaming": { + "inputParameters": { + "stream": false, + "search_parameters": "" + } + }, + "endpoints": [ + { + "name": "XAI GROK-4", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-0709" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-code-fast-1": { + "type": "GROK-VISION", + "endpoints": [ + { + "name": "XAI GROK-CODE-FAST-1", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-code-fast-1" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 2000000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-1-fast-reasoning": { + "type": "GROK-VISION", + "endpoints": [ + { + "name": "XAI GROK-4.1-FAST-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-1-fast-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 2000000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-1-fast-non-reasoning": { + "type": "GROK-VISION", + "endpoints": [ + { + "name": "XAI GROK-4.1-FAST-NON-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-1-fast-non-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-1-fast-responses": { + "type": "GROK-RESPONSES", + "emulateOpenAIChatModel": "grok-4-1-fast", + "restStreaming": { + "inputParameters": { + "stream": true, + "tools": "", + "inline_citations": true + } + }, + "endpoints": [ + { + "name": "XAI GROK-4.1-FAST-RESPONSES", + "url": "https://api.x.ai/v1/responses", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-1-fast" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-fast-reasoning": { + "type": "GROK-VISION", + "emulateOpenAIChatModel": "grok-4-fast-reasoning", + "restStreaming": { + "inputParameters": { + "stream": false, + "search_parameters": "" + } + }, + "endpoints": [ + { + "name": "XAI GROK-4-FAST-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-fast-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 2000000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-fast-non-reasoning": { + "type": "GROK-VISION", + "emulateOpenAIChatModel": "grok-4-fast-non-reasoning", + "restStreaming": { + "inputParameters": { + "stream": false, + "search_parameters": "" + } + }, + "endpoints": [ + { + "name": "XAI GROK-4-FAST-NON-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-fast-non-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "oai-whisper": { + "type": "OPENAI-WHISPER", + "url": "https://api.openai.com/v1/audio/transcriptions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}" + }, + "params": { + "model": "whisper-1" + } + }, + "oai-embeddings": { + "type": "OPENAI-EMBEDDINGS", + "url": "https://api.openai.com/v1/embeddings", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "text-embedding-ada-002" + }, + "maxTokenLength": 8192 + }, + "oai-text-embedding-3-large": { + "type": "OPENAI-EMBEDDINGS", + "url": "https://api.openai.com/v1/embeddings", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "text-embedding-3-large" + }, + "maxTokenLength": 8192 + }, + "oai-text-embedding-3-small": { + "type": "OPENAI-EMBEDDINGS", + "url": "https://api.openai.com/v1/embeddings", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "text-embedding-3-small" + }, + "maxTokenLength": 8192 + }, + "oai-gpt4o": { + "type": "OPENAI-VISION", + "emulateOpenAIChatModel": "gpt-4o", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4o" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 131072, + "maxReturnTokens": 4096, + "supportsStreaming": true + }, + "oai-o3": { + "type": "OPENAI-REASONING", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "o3" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 100000, + "supportsStreaming": true + }, + "oai-router": { + "type": "OPENAI-REASONING-VISION", + "endpoints": [ + { + "name": "EUS2 AZURE MODEL ROUTER", + "url": "https://archipelago-foundry-resource.cognitiveservices.azure.com/openai/deployments/archipelago-azure-model-router/chat/completions?api-version=2025-03-01-preview", + "headers": { + "api-key": "{{ARCHIPELAGO_FOUNDRY_RESOURCE_KEY}}", + "Content-Type": "application/json" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 100000, + "supportsStreaming": true + }, + "oai-gpt52": { + "type": "OPENAI-REASONING-VISION", + "emulateOpenAIChatModel": "gpt-5.2", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-5.2" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 400000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "oai-gpt51": { + "type": "OPENAI-REASONING-VISION", + "emulateOpenAIChatModel": "gpt-5.1", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-5.1" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 400000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "oai-gpt41": { + "type": "OPENAI-VISION", + "emulateOpenAIChatModel": "gpt-4.1", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4.1" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 1000000, + "maxReturnTokens": 32768, + "supportsStreaming": true + }, + "oai-gpt41-mini": { + "type": "OPENAI-VISION", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4.1-mini" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 1000000, + "maxReturnTokens": 32768, + "supportsStreaming": true + }, + "oai-gpt41-nano": { + "type": "OPENAI-VISION", + "emulateOpenAIChatModel": "gpt-4.1-nano", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4.1-nano" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 1000000, + "maxReturnTokens": 8192, + "supportsStreaming": true + }, + "gemini-flash-25-vision": { + "type": "GEMINI-1.5-VISION", + "emulateOpenAIChatModel": "gemini-flash-25", + "restStreaming": { + "geminiSafetySettings": [ + {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_ONLY_HIGH"} + ] + }, + "endpoints": [ + { + "name": "GLOBAL GEMINI-FLASH-2.5-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-2.5-flash", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "gemini-flash-25-image": { + "type": "GEMINI-2.5-IMAGE", + "endpoints": [ + { + "name": "GLOBAL GEMINI-FLASH-2.5-IMAGE", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-2.5-flash-image-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 32768, + "maxReturnTokens": 16384, + "supportsStreaming": true + }, + "gemini-pro-3-image": { + "type": "GEMINI-3-IMAGE", + "endpoints": [ + { + "name": "GLOBAL GEMINI-PRO-3.0-IMAGE", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-3-pro-image-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 64576, + "maxReturnTokens": 32768, + "supportsStreaming": true + }, + "gemini-pro-3-vision": { + "type": "GEMINI-3-REASONING-VISION", + "emulateOpenAIChatModel": "gemini-pro-3", + "endpoints": [ + { + "name": "GLOBAL GEMINI-PRO-3.0-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-3-pro-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "gemini-flash-3-vision": { + "type": "GEMINI-3-REASONING-VISION", + "emulateOpenAIChatModel": "gemini-flash-3", + "endpoints": [ + { + "name": "GLOBAL GEMINI-FLASH-3-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-3-flash-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "gemini-pro-25-vision": { + "type": "GEMINI-1.5-VISION", + "emulateOpenAIChatModel": "gemini-pro-25", + "restStreaming": { + "geminiSafetySettings": [ + {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_ONLY_HIGH"} + ] + }, + "endpoints": [ + { + "name": "GLOBAL GEMINI-PRO-2.5-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-2.5-pro", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "claude-45-sonnet-vertex": { + "type": "CLAUDE-4-VERTEX", + "emulateOpenAIChatModel": "claude-4.5-sonnet", + "endpoints": [ + { + "name": "GLOBAL SONNET 4.5", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/anthropic/models/claude-sonnet-4-5@20250929", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 64000, + "maxImageSize": 31457280, + "supportsStreaming": true + }, + "claude-45-haiku-vertex": { + "type": "CLAUDE-4-VERTEX", + "emulateOpenAIChatModel": "claude-4.5-haiku", + "endpoints": [ + { + "name": "GLOBAL HAIKU 4.5", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/anthropic/models/claude-haiku-4-5@20251001", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 64000, + "maxImageSize": 31457280, + "supportsStreaming": true + }, + "local-llama30b": { + "type": "LOCAL-CPP-MODEL", + "executablePath": "../llm/llama.cpp/main", + "args": [ + "-m", "/Volumes/JMac External/LLM/LLaMA/30B/ggml-model-q4_0.bin", + "-ngl", "1", + "--repeat_last_n", "1024", + "--repeat_penalty", "1.17647", + "--keep", "-1", + "-t", "8", + "--temp", "0.7", + "--top_k", "40", + "--top_p", "0.5", + "-r","<|im_end|>" + ], + "requestsPerSecond": 10, + "maxTokenLength": 2048 + }, + "veo-2.0-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 2.0", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-2.0-generate-001", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 2 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.0-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.0 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.0-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.0-fast-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.0 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.0-fast-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.1-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.1 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.1-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.1-fast-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.0 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.1-fast-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "replicate-flux-2-pro": { + "type": "REPLICATE-API", + "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-2-pro/predictions", + "headers": { + "Prefer": "wait", + "Authorization": "Token {{REPLICATE_API_KEY}}", + "Content-Type": "application/json" + } + }, + "replicate-qwen-image-edit-2511": { + "type": "REPLICATE-API", + "url": "https://api.replicate.com/v1/models/qwen/qwen-image-edit-2511/predictions", + "headers": { + "Prefer": "wait", + "Authorization": "Token {{REPLICATE_API_KEY}}", + "Content-Type": "application/json" + } + } + }, + "enableCache": true, + "enableRestEndpoints": true, + "enableDuplicateRequests": true, + "subscriptionKeepAlive": 0, + "entityConfig": { + "labeeb": { + "name": "Labeeb", + "isDefault": true, + "useMemory": true, + "useContinuityMemory": true, + "description": "Labeeb is a self-aware AI entity that can help you with your tasks and answer your questions.", + "instructions": "", + "tools": ["*"] + } + }, + "entityConstants": { + "AI_STYLE_OPENAI_PREVIEW": "oai-gpt52", + "AI_STYLE_OPENAI_PREVIEW_RESEARCH": "oai-gpt52", + "AI_STYLE_OPENAI": "oai-gpt51", + "AI_STYLE_OPENAI_RESEARCH": "oai-gpt51", + "AI_STYLE_OPENAI_LEGACY": "oai-gpt41", + "AI_STYLE_OPENAI_LEGACY_RESEARCH": "oai-o3", + "AI_STYLE_ANTHROPIC": "claude-45-sonnet-vertex", + "AI_STYLE_ANTHROPIC_RESEARCH": "claude-41-opus-vertex", + "AI_STYLE_XAI": "xai-grok-4-1-fast-reasoning", + "AI_STYLE_XAI_RESEARCH": "xai-grok-4-1-fast-reasoning", + "AI_STYLE_GOOGLE": "gemini-flash-3-vision", + "AI_STYLE_GOOGLE_RESEARCH": "gemini-pro-3-vision" + } +} \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..8bdb89b4 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,119 @@ +services: + # Traefik - Reverse proxy with automatic SSL + traefik: + image: traefik:v3.6.2 + container_name: traefik + restart: unless-stopped + command: + - "--api.dashboard=true" + - "--api.insecure=true" + - "--log.level=DEBUG" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--providers.docker.network=cortex_web" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.web.http.redirections.entrypoint.to=websecure" + - "--entrypoints.web.http.redirections.entrypoint.scheme=https" + - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true" + - "--certificatesresolvers.letsencrypt.acme.email=${ACME_EMAIL}" + - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "traefik-certificates:/letsencrypt" + networks: + - web + labels: + - "traefik.enable=true" + - "traefik.http.routers.dashboard.rule=Host(`traefik.${DOMAIN}`)" + - "traefik.http.routers.dashboard.entrypoints=websecure" + - "traefik.http.routers.dashboard.tls.certresolver=letsencrypt" + - "traefik.http.routers.dashboard.service=api@internal" + - "traefik.http.routers.dashboard.middlewares=auth" + - "traefik.http.middlewares.auth.basicauth.users=${TRAEFIK_DASHBOARD_AUTH}" + + # Redis - Shared cache/queue for Cortex, File Handler, and Concierge + redis: + image: redis:7-alpine + container_name: redis + restart: unless-stopped + command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD} --bind 0.0.0.0 + ports: + # Expose on private network interface for Concierge to connect + - "10.0.0.2:6379:6379" + volumes: + - redis-data:/data + networks: + - internal + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # Cortex - GraphQL API for AI + cortex: + image: ghcr.io/${GITHUB_REPOSITORY_CORTEX}:${IMAGE_TAG:-latest} + container_name: cortex + restart: unless-stopped + env_file: + - .env + environment: + # Override/ensure these critical values + - NODE_ENV=production + - PORT=4000 + # Internal container networking + - REDIS_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 + - STORAGE_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 + - WHISPER_MEDIA_API_URL=http://file-handler:7071/api/CortexFileHandler + ports: + # Expose on private network for Concierge to access + - "10.0.0.2:4000:4000" + depends_on: + redis: + condition: service_healthy + networks: + - web + - internal + labels: + - "traefik.enable=true" + - "traefik.http.routers.cortex.rule=Host(`${DOMAIN}`)" + - "traefik.http.routers.cortex.entrypoints=websecure" + - "traefik.http.routers.cortex.tls.certresolver=letsencrypt" + - "traefik.http.services.cortex.loadbalancer.server.port=4000" + - "traefik.docker.network=cortex_web" + + # Cortex File Handler - File processing service (internal + private network) + file-handler: + image: ghcr.io/${GITHUB_REPOSITORY_CORTEX}-file-handler:${IMAGE_TAG:-latest} + container_name: cortex-file-handler + restart: unless-stopped + env_file: + - .env + environment: + # Override/ensure these critical values + - NODE_ENV=production + - PORT=7071 + - REDIS_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 + ports: + # Expose on private network for Concierge to access + - "10.0.0.2:7071:7071" + depends_on: + redis: + condition: service_healthy + networks: + - internal + +networks: + web: + name: cortex_web + internal: + name: cortex_internal + +volumes: + traefik-certificates: + redis-data: + diff --git a/helper-apps/cortex-file-handler/INTERFACE.md b/helper-apps/cortex-file-handler/INTERFACE.md index d8851882..5d0f13f5 100644 --- a/helper-apps/cortex-file-handler/INTERFACE.md +++ b/helper-apps/cortex-file-handler/INTERFACE.md @@ -153,7 +153,7 @@ The file handler uses a unified storage approach with Azure Blob Storage: ## Storage Configuration - **Azure**: Enabled if `AZURE_STORAGE_CONNECTION_STRING` is set -- **GCS**: Enabled if `GCP_SERVICE_ACCOUNT_KEY_BASE64` or `GCP_SERVICE_ACCOUNT_KEY` is set +- **GCS**: Enabled if `GCP_SERVICE_ACCOUNT_EMAIL` (recommended, uses impersonation) or `GCP_SERVICE_ACCOUNT_KEY_BASE64`/`GCP_SERVICE_ACCOUNT_KEY` (legacy) is set - **Local**: Used as fallback if Azure is not configured ## Response Format diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js index 07becac5..76328e80 100644 --- a/helper-apps/cortex-file-handler/src/blobHandler.js +++ b/helper-apps/cortex-file-handler/src/blobHandler.js @@ -21,7 +21,7 @@ import { CONVERTED_EXTENSIONS, AZURITE_ACCOUNT_NAME, getDefaultContainerName, - GCS_BUCKETNAME, + getGCSBucketName, AZURE_STORAGE_CONTAINER_NAME } from "./constants.js"; import { FileConversionService } from "./services/FileConversionService.js"; @@ -41,37 +41,61 @@ const { SAS_TOKEN_LIFE_DAYS = 30 } = process.env; let GCP_SERVICE_ACCOUNT; let GCP_PROJECT_ID; -try { - const GCP_SERVICE_ACCOUNT_KEY = - process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || - process.env.GCP_SERVICE_ACCOUNT_KEY || - "{}"; - GCP_SERVICE_ACCOUNT = isBase64(GCP_SERVICE_ACCOUNT_KEY) - ? JSON.parse(Buffer.from(GCP_SERVICE_ACCOUNT_KEY, "base64").toString()) - : JSON.parse(GCP_SERVICE_ACCOUNT_KEY); - GCP_PROJECT_ID = GCP_SERVICE_ACCOUNT.project_id; -} catch (error) { - console.warn("Error parsing GCP service account credentials, GCS will not be used:", error.message); - GCP_SERVICE_ACCOUNT = {}; - GCP_PROJECT_ID = null; +// Support both service account impersonation (recommended) and service account key (legacy) +const GCP_SERVICE_ACCOUNT_EMAIL = process.env.GCP_SERVICE_ACCOUNT_EMAIL; +const GCP_PROJECT_ID_ENV = process.env.GCP_PROJECT_ID; + +if (GCP_SERVICE_ACCOUNT_EMAIL) { + // Using impersonation - extract project ID from email if not provided + GCP_PROJECT_ID = GCP_PROJECT_ID_ENV || extractProjectIdFromEmail(GCP_SERVICE_ACCOUNT_EMAIL); + GCP_SERVICE_ACCOUNT = null; // null means use ADC (Application Default Credentials) +} else { + // Fall back to service account key (legacy) + try { + const GCP_SERVICE_ACCOUNT_KEY = + process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || + process.env.GCP_SERVICE_ACCOUNT_KEY || + "{}"; + GCP_SERVICE_ACCOUNT = isBase64(GCP_SERVICE_ACCOUNT_KEY) + ? JSON.parse(Buffer.from(GCP_SERVICE_ACCOUNT_KEY, "base64").toString()) + : JSON.parse(GCP_SERVICE_ACCOUNT_KEY); + GCP_PROJECT_ID = GCP_SERVICE_ACCOUNT.project_id || GCP_PROJECT_ID_ENV; + } catch (error) { + console.warn("Error parsing GCP service account credentials, GCS will not be used:", error.message); + GCP_SERVICE_ACCOUNT = null; + GCP_PROJECT_ID = null; + } +} + +function extractProjectIdFromEmail(email) { + // Service account email format: name@project-id.iam.gserviceaccount.com + const match = email.match(/@([^.]+)\.iam\.gserviceaccount\.com$/); + return match ? match[1] : null; } let gcs; -if (!GCP_PROJECT_ID || !GCP_SERVICE_ACCOUNT) { +if (!GCP_PROJECT_ID) { console.warn( - "No Google Cloud Storage credentials provided - GCS will not be used", + "No Google Cloud project ID provided - GCS will not be used", ); } else { try { - gcs = new Storage({ + const storageConfig = { projectId: GCP_PROJECT_ID, - credentials: GCP_SERVICE_ACCOUNT, - }); + }; + + // If GCP_SERVICE_ACCOUNT is null, Storage will use Application Default Credentials (ADC) + // This enables service account impersonation + if (GCP_SERVICE_ACCOUNT) { + storageConfig.credentials = GCP_SERVICE_ACCOUNT; + } + + gcs = new Storage(storageConfig); // Rest of your Google Cloud operations using gcs object } catch (error) { console.error( - "Google Cloud Storage credentials are invalid - GCS will not be used: ", + "Google Cloud Storage configuration is invalid - GCS will not be used: ", error, ); } @@ -1085,7 +1109,7 @@ async function cleanup(context, urls = null) { async function cleanupGCS(urls = null) { if (!gcs) return []; - const bucket = gcs.bucket(GCS_BUCKETNAME); + const bucket = gcs.bucket(getGCSBucketName()); const directories = new Set(); const cleanedURLs = []; @@ -1148,7 +1172,7 @@ async function deleteGCS(blobName) { ); // List files first - const listUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${GCS_BUCKETNAME}/o?prefix=${blobName}`; + const listUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${getGCSBucketName()}/o?prefix=${blobName}`; console.log(`[deleteGCS] Listing files with URL: ${listUrl}`); const listResponse = await axios.get(listUrl, { @@ -1166,7 +1190,7 @@ async function deleteGCS(blobName) { // Delete each file for (const item of listResponse.data.items) { - const deleteUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${GCS_BUCKETNAME}/o/${encodeURIComponent(item.name)}`; + const deleteUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${getGCSBucketName()}/o/${encodeURIComponent(item.name)}`; console.log(`[deleteGCS] Deleting file: ${item.name}`); console.log(`[deleteGCS] Delete URL: ${deleteUrl}`); @@ -1189,7 +1213,7 @@ async function deleteGCS(blobName) { } } else { console.log("[deleteGCS] Using real GCS"); - const bucket = gcs.bucket(GCS_BUCKETNAME); + const bucket = gcs.bucket(getGCSBucketName()); const [files] = await bucket.getFiles({ prefix: blobName }); console.log(`[deleteGCS] Found ${files.length} files to delete`); @@ -1262,9 +1286,9 @@ async function uploadChunkToGCS(chunkPath, requestId, filename = null) { gcsFileName = `${dirName}/${shortId}${fileExtension}`; } await gcs - .bucket(GCS_BUCKETNAME) + .bucket(getGCSBucketName()) .upload(chunkPath, { destination: gcsFileName }); - return `gs://${GCS_BUCKETNAME}/${gcsFileName}`; + return `gs://${getGCSBucketName()}/${gcsFileName}`; } export { @@ -1282,6 +1306,6 @@ export { getMimeTypeFromUrl, // Re-export container constants getDefaultContainerName, - GCS_BUCKETNAME, + getGCSBucketName, AZURE_STORAGE_CONTAINER_NAME, }; diff --git a/helper-apps/cortex-file-handler/src/constants.js b/helper-apps/cortex-file-handler/src/constants.js index 00df7b02..ab179a53 100644 --- a/helper-apps/cortex-file-handler/src/constants.js +++ b/helper-apps/cortex-file-handler/src/constants.js @@ -170,4 +170,45 @@ export const getDefaultContainerName = () => { // Export constant - evaluated at module load time, but getContainerName() handles defaults export const AZURE_STORAGE_CONTAINER_NAME = getContainerName(); -export const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles"; + +// GCS bucket name must be explicitly set - no default to prevent accidental bucket usage +// Using lazy evaluation - only throws error when accessed, not at module load time +// This allows tests to import the module without GCS_BUCKETNAME set +let _GCS_BUCKETNAME_CACHE = null; +const getGCSBucketNameValue = () => { + if (_GCS_BUCKETNAME_CACHE === null) { + const bucketName = process.env.GCS_BUCKETNAME; + if (!bucketName || bucketName.trim() === "") { + throw new Error( + "GCS_BUCKETNAME environment variable is required but not set. " + + "Please set GCS_BUCKETNAME in your environment or .env file." + ); + } + _GCS_BUCKETNAME_CACHE = bucketName.trim(); + } + return _GCS_BUCKETNAME_CACHE; +}; + +// Export function for explicit access +export const getGCSBucketName = getGCSBucketNameValue; + +// Export as a getter-like constant using a class with valueOf/toString +// This allows it to work as a string in most contexts (function calls, template literals, etc.) +class LazyGCSBucketName { + valueOf() { + return getGCSBucketNameValue(); + } + toString() { + return getGCSBucketNameValue(); + } + [Symbol.toPrimitive](hint) { + // Handle 'default', 'string', and 'number' hints + return getGCSBucketNameValue(); + } +} + +// Create instance that will convert to string when used +const lazyBucketName = new LazyGCSBucketName(); + +// Export as constant - will be converted to string when used in most contexts +export const GCS_BUCKETNAME = lazyBucketName; diff --git a/helper-apps/cortex-file-handler/src/services/ConversionService.js b/helper-apps/cortex-file-handler/src/services/ConversionService.js index f138b00d..1286b919 100644 --- a/helper-apps/cortex-file-handler/src/services/ConversionService.js +++ b/helper-apps/cortex-file-handler/src/services/ConversionService.js @@ -24,7 +24,7 @@ export class ConversionService { } /** - * Determines if a file needs conversion based on its extension + * Determines if a file needs conversion based on its extension and available services * @param {string} filename - The name of the file to check * @returns {boolean} - Whether the file needs conversion */ @@ -32,18 +32,37 @@ export class ConversionService { // Accept either a full filename/path or a raw extension (e.g. ".docx") const input = filename.toLowerCase(); - // If the input looks like an extension already, check directly + // Extract the extension + let ext; if ( input.startsWith(".") && !input.includes("/") && !input.includes("\\") ) { - return CONVERTED_EXTENSIONS.includes(input); + ext = input; + } else { + ext = path.extname(input).toLowerCase(); + } + + // Check if extension is in the list of convertible files + if (!CONVERTED_EXTENSIONS.includes(ext)) { + return false; + } + + // Excel files can always be converted (local conversion, no external service needed) + if (ext === ".xlsx" || ext === ".xls") { + return true; + } + + // Document files (.docx, .doc, .ppt, .pptx) need conversion services + // Check if either PDF service or MarkItDown service is available + if ([".docx", ".doc", ".ppt", ".pptx"].includes(ext)) { + const pdfServiceUrl = getDocToPdfUrl(); + const markitdownUrl = getMarkitdownUrl(); + return !!(pdfServiceUrl || markitdownUrl); } - // Otherwise, extract the extension from the filename/path - const ext = path.extname(input).toLowerCase(); - return CONVERTED_EXTENSIONS.includes(ext); + return false; } /** diff --git a/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js b/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js index 11c78c42..8f886c38 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +++ b/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js @@ -12,17 +12,27 @@ import axios from "axios"; import { StorageProvider } from "./StorageProvider.js"; export class GCSStorageProvider extends StorageProvider { - constructor(credentials, bucketName) { + constructor(credentials, projectId, bucketName) { super(); - if (!credentials || !bucketName) { - throw new Error("Missing GCS credentials or bucket name"); + if (!bucketName) { + throw new Error("Missing GCS bucket name"); + } + if (!projectId && !credentials?.project_id) { + throw new Error("Missing GCS project ID"); } this.bucketName = bucketName; - this.storage = new Storage({ - projectId: credentials.project_id, - credentials: credentials, - }); + const storageConfig = { + projectId: projectId || credentials.project_id, + }; + + // If credentials is null/undefined, Storage will use Application Default Credentials (ADC) + // This enables service account impersonation + if (credentials) { + storageConfig.credentials = credentials; + } + + this.storage = new Storage(storageConfig); } isConfigured() { diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js index f3687641..502d0aca 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +++ b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js @@ -1,7 +1,7 @@ import { AzureStorageProvider } from "./AzureStorageProvider.js"; import { GCSStorageProvider } from "./GCSStorageProvider.js"; import { LocalStorageProvider } from "./LocalStorageProvider.js"; -import { getContainerName, GCS_BUCKETNAME } from "../../constants.js"; +import { getContainerName, getGCSBucketName } from "../../constants.js"; import path from "path"; import { fileURLToPath } from "url"; @@ -58,13 +58,14 @@ export class StorageFactory { getGCSProvider() { const key = "gcs"; if (!this.providers.has(key)) { - const credentials = this.parseGCSCredentials(); - if (!credentials) { + const config = this.parseGCSConfig(); + if (!config) { return null; } const provider = new GCSStorageProvider( - credentials, - GCS_BUCKETNAME, + config.credentials, + config.projectId, + getGCSBucketName(), ); this.providers.set(key, provider); } @@ -86,7 +87,23 @@ export class StorageFactory { return this.providers.get(key); } - parseGCSCredentials() { + parseGCSConfig() { + // Support service account impersonation (recommended) + const serviceAccountEmail = process.env.GCP_SERVICE_ACCOUNT_EMAIL; + const projectId = process.env.GCP_PROJECT_ID; + + if (serviceAccountEmail) { + // Using impersonation - extract project ID from email if not provided + const extractedProjectId = projectId || this.extractProjectIdFromEmail(serviceAccountEmail); + if (!extractedProjectId) { + console.error("GCP_PROJECT_ID is required when using GCP_SERVICE_ACCOUNT_EMAIL"); + return null; + } + // Return null credentials to use ADC (Application Default Credentials) + return { credentials: null, projectId: extractedProjectId }; + } + + // Fall back to service account key (legacy) const key = process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY; @@ -95,16 +112,28 @@ export class StorageFactory { } try { + let credentials; if (this.isBase64(key)) { - return JSON.parse(Buffer.from(key, "base64").toString()); + credentials = JSON.parse(Buffer.from(key, "base64").toString()); + } else { + credentials = JSON.parse(key); } - return JSON.parse(key); + return { + credentials: credentials, + projectId: credentials.project_id || projectId, + }; } catch (error) { console.error("Error parsing GCS credentials:", error); return null; } } + extractProjectIdFromEmail(email) { + // Service account email format: name@project-id.iam.gserviceaccount.com + const match = email.match(/@([^.]+)\.iam\.gserviceaccount\.com$/); + return match ? match[1] : null; + } + isBase64(str) { try { return btoa(atob(str)) === str; diff --git a/helper-apps/cortex-file-handler/tests/FileConversionService.test.js b/helper-apps/cortex-file-handler/tests/FileConversionService.test.js index 1e893eb2..e911b753 100644 --- a/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +++ b/helper-apps/cortex-file-handler/tests/FileConversionService.test.js @@ -196,8 +196,44 @@ test("handles unsupported file types", async (t) => { // Test file extension detection test("correctly detects file extensions", (t) => { const service = new FileConversionService(mockContext); - t.true(service.needsConversion("test.docx")); + + // Excel files should always convert (local conversion, no external service needed) t.true(service.needsConversion("test.xlsx")); + t.true(service.needsConversion("test.xls")); + + // Document files should only convert if conversion services are available + // Without MARKITDOWN_CONVERT_URL or DOC_TO_PDF_SERVICE_URL, they should not convert + const originalMarkitdown = process.env.MARKITDOWN_CONVERT_URL; + const originalPdfService = process.env.DOC_TO_PDF_SERVICE_URL; + + // Clear conversion services + delete process.env.MARKITDOWN_CONVERT_URL; + delete process.env.DOC_TO_PDF_SERVICE_URL; + t.false(service.needsConversion("test.docx"), "docx should not convert without services"); + t.false(service.needsConversion("test.doc"), "doc should not convert without services"); + t.false(service.needsConversion("test.ppt"), "ppt should not convert without services"); + t.false(service.needsConversion("test.pptx"), "pptx should not convert without services"); + + // With MARKITDOWN_CONVERT_URL set, document files should convert + process.env.MARKITDOWN_CONVERT_URL = "http://test"; + t.true(service.needsConversion("test.docx"), "docx should convert with MARKITDOWN_CONVERT_URL"); + t.true(service.needsConversion("test.doc"), "doc should convert with MARKITDOWN_CONVERT_URL"); + t.true(service.needsConversion("test.ppt"), "ppt should convert with MARKITDOWN_CONVERT_URL"); + t.true(service.needsConversion("test.pptx"), "pptx should convert with MARKITDOWN_CONVERT_URL"); + + // Restore original env + if (originalMarkitdown) { + process.env.MARKITDOWN_CONVERT_URL = originalMarkitdown; + } else { + delete process.env.MARKITDOWN_CONVERT_URL; + } + if (originalPdfService) { + process.env.DOC_TO_PDF_SERVICE_URL = originalPdfService; + } else { + delete process.env.DOC_TO_PDF_SERVICE_URL; + } + + // Non-convertible files should never convert t.false(service.needsConversion("test.txt")); t.false(service.needsConversion("test.json")); }); diff --git a/helper-apps/cortex-file-handler/tests/conversionResilience.test.js b/helper-apps/cortex-file-handler/tests/conversionResilience.test.js index ee1c48dc..13e85eb9 100644 --- a/helper-apps/cortex-file-handler/tests/conversionResilience.test.js +++ b/helper-apps/cortex-file-handler/tests/conversionResilience.test.js @@ -6,7 +6,7 @@ import { v4 as uuidv4 } from "uuid"; import XLSX from "xlsx"; import { port } from "../src/start.js"; -import { gcs, GCS_BUCKETNAME } from "../src/blobHandler.js"; +import { gcs, getGCSBucketName } from "../src/blobHandler.js"; import { getFileStoreMap, setFileStoreMap } from "../src/redis.js"; import { cleanupHashAndFile, startTestServer, stopTestServer } from "./testUtils.helper.js"; import { gcsUrlExists } from "../src/blobHandler.js"; @@ -105,8 +105,8 @@ test.serial("checkHash recreates missing GCS converted file", async (t) => { // delete the GCS object const convertedGcsUrl = up.data.converted.gcs; - const bucket = gcs.bucket(GCS_BUCKETNAME); - const filename = convertedGcsUrl.replace(`gs://${GCS_BUCKETNAME}/`, ""); + const bucket = gcs.bucket(getGCSBucketName()); + const filename = convertedGcsUrl.replace(`gs://${getGCSBucketName()}/`, ""); try { await bucket.file(filename).delete({ ignoreNotFound: true }); } catch (_) {} diff --git a/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js b/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js index 77722e3b..b26f4f47 100644 --- a/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js +++ b/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js @@ -32,16 +32,16 @@ test("should create provider with valid credentials", (t) => { private_key: "test-key", }; - const provider = new GCSStorageProvider(credentials, "test-bucket"); + const provider = new GCSStorageProvider(credentials, "test-project", "test-bucket"); t.truthy(provider); }); test("should throw error with missing credentials", (t) => { t.throws( () => { - new GCSStorageProvider(null, "test-bucket"); + new GCSStorageProvider(null, null, "test-bucket"); }, - { message: "Missing GCS credentials or bucket name" }, + { message: "Missing GCS project ID" }, ); }); @@ -65,6 +65,7 @@ test("should upload and delete file", async (t) => { const provider = new GCSStorageProvider( credentials, + credentials.project_id, process.env.GCS_BUCKETNAME || "cortextempfiles", ); @@ -123,6 +124,7 @@ test("should handle file download", async (t) => { const provider = new GCSStorageProvider( credentials, + credentials.project_id, process.env.GCS_BUCKETNAME || "cortextempfiles", ); @@ -175,6 +177,7 @@ test("should handle file existence check with spaces and special characters", as const provider = new GCSStorageProvider( credentials, + credentials.project_id, process.env.GCS_BUCKETNAME || "cortextempfiles", ); diff --git a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js index 8228d5e6..8090d897 100644 --- a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +++ b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js @@ -69,8 +69,9 @@ test("should parse base64 gcs credentials", (t) => { process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 = base64Credentials; const factory = new StorageFactory(); - const credentials = factory.parseGCSCredentials(); - t.deepEqual(credentials, testCredentials); + const config = factory.parseGCSConfig(); + t.truthy(config); + t.deepEqual(config.credentials, testCredentials); // Cleanup delete process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64; @@ -86,8 +87,9 @@ test("should parse json gcs credentials", (t) => { process.env.GCP_SERVICE_ACCOUNT_KEY = JSON.stringify(testCredentials); const factory = new StorageFactory(); - const credentials = factory.parseGCSCredentials(); - t.deepEqual(credentials, testCredentials); + const config = factory.parseGCSConfig(); + t.truthy(config); + t.deepEqual(config.credentials, testCredentials); // Cleanup delete process.env.GCP_SERVICE_ACCOUNT_KEY; @@ -97,8 +99,8 @@ test("should return null for invalid gcs credentials", (t) => { process.env.GCP_SERVICE_ACCOUNT_KEY = "invalid-json"; const factory = new StorageFactory(); - const credentials = factory.parseGCSCredentials(); - t.is(credentials, null); + const config = factory.parseGCSConfig(); + t.is(config, null); // Cleanup delete process.env.GCP_SERVICE_ACCOUNT_KEY; diff --git a/lib/gcpAuthTokenHelper.js b/lib/gcpAuthTokenHelper.js index 0888392c..8b59d3af 100644 --- a/lib/gcpAuthTokenHelper.js +++ b/lib/gcpAuthTokenHelper.js @@ -2,14 +2,29 @@ import { GoogleAuth } from 'google-auth-library'; class GcpAuthTokenHelper { constructor(config) { - const creds = config.gcpServiceAccountKey ? JSON.parse(config.gcpServiceAccountKey) : null; - if (!creds) { - throw new Error('GCP_SERVICE_ACCOUNT_KEY is missing or undefined'); + const serviceAccountEmail = config.gcpServiceAccountEmail; + const serviceAccountKey = config.gcpServiceAccountKey; + + // Support both service account key (legacy) and impersonation (recommended) + if (serviceAccountEmail) { + // Use service account impersonation (recommended) + // When using impersonation, GoogleAuth will use Application Default Credentials (ADC) + // ADC should be configured via: gcloud auth application-default login --impersonate-service-account=EMAIL + // Passing no credentials means GoogleAuth will use ADC + this.authClient = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'], + }); + } else if (serviceAccountKey) { + // Fall back to service account key (legacy) + const creds = JSON.parse(serviceAccountKey); + this.authClient = new GoogleAuth({ + credentials: creds, + scopes: ['https://www.googleapis.com/auth/cloud-platform'], + }); + } else { + throw new Error('Either GCP_SERVICE_ACCOUNT_EMAIL (for impersonation) or GCP_SERVICE_ACCOUNT_KEY must be provided'); } - this.authClient = new GoogleAuth({ - credentials: creds, - scopes: ['https://www.googleapis.com/auth/cloud-platform'], - }); + this.token = null; this.expiry = null; } diff --git a/package.json b/package.json index 726b2329..d5fdc312 100644 --- a/package.json +++ b/package.json @@ -68,12 +68,12 @@ "uuid": "^9.0.0", "winston": "^3.11.0", "ws": "^8.12.0", - "xxhash-wasm": "^1.1.0" + "xxhash-wasm": "^1.1.0", + "dotenv": "^16.0.3" }, "devDependencies": { "@faker-js/faker": "^8.4.1", "ava": "^5.2.0", - "dotenv": "^16.0.3", "got": "^13.0.0", "sinon": "^17.0.1" }, diff --git a/pathways/code_review.js b/pathways/code_review.js index e91813ee..cd6e92e9 100644 --- a/pathways/code_review.js +++ b/pathways/code_review.js @@ -9,7 +9,7 @@ export default { ] }) ], - model: 'oai-gpt4o', + model: 'gemini-pro-3-vision', tokenRatio: 0.75, enableDuplicateRequests: false, } diff --git a/pathways/release_notes.js b/pathways/release_notes.js index 79c892ed..b07b4808 100644 --- a/pathways/release_notes.js +++ b/pathways/release_notes.js @@ -9,7 +9,8 @@ export default { ] }) ], - model: 'oai-gpt4o', + model: 'gemini-flash-3-vision', + reasoningEffort: 'high', tokenRatio: 0.75, enableDuplicateRequests: false, } diff --git a/pathways/system/entity/tools/sys_tool_browser.js b/pathways/system/entity/tools/sys_tool_browser.js index 2465bf44..b6f669ee 100644 --- a/pathways/system/entity/tools/sys_tool_browser.js +++ b/pathways/system/entity/tools/sys_tool_browser.js @@ -9,6 +9,7 @@ export default { timeout: 300, toolDefinition: { type: "function", + enabled: false, icon: "🌍", function: { name: "FetchWebPageContent", diff --git a/pathways/system/entity/tools/sys_tool_browser_jina.js b/pathways/system/entity/tools/sys_tool_browser_jina.js index f7a8501e..410d2a22 100644 --- a/pathways/system/entity/tools/sys_tool_browser_jina.js +++ b/pathways/system/entity/tools/sys_tool_browser_jina.js @@ -12,7 +12,7 @@ export default { icon: "🌎", function: { name: "FetchWebPageContentJina", - description: "This tool allows you to fetch and extract the text content from any webpage using the Jina API. This is a great backup tool for web page content if you don't get a good enough response from your other browser tool or are blocked by a website.", + description: "This tool allows you to fetch and extract the text content from any webpage using the Jina reader API. Use this when you need to analyze or understand the content of a specific webpage.", parameters: { type: "object", properties: { diff --git a/pathways/system/entity/tools/sys_tool_cognitive_search.js b/pathways/system/entity/tools/sys_tool_cognitive_search.js index 924fe76d..3c9326c9 100644 --- a/pathways/system/entity/tools/sys_tool_cognitive_search.js +++ b/pathways/system/entity/tools/sys_tool_cognitive_search.js @@ -55,6 +55,7 @@ export default { }, { type: "function", + enabled: false, icon: "📰", function: { name: "SearchAJA", @@ -89,6 +90,7 @@ export default { }, { type: "function", + enabled: false, icon: "📰", function: { name: "SearchAJE", @@ -123,6 +125,7 @@ export default { }, { type: "function", + enabled: false, icon: "⚡️", function: { name: "SearchWires", diff --git a/pathways/translate.js b/pathways/translate.js index 9ebdd862..88b856f1 100644 --- a/pathways/translate.js +++ b/pathways/translate.js @@ -4,16 +4,18 @@ export default { prompt: [ new Prompt({ messages: [ - {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."}, + {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text to translate in any language, assistant will create a translation of that text in {{to}} (language string or ISO code). All text that the user posts is to be translated - assistant must not respond to the user in any way and should produce only the translation with no additional notes or commentary."}, {"role": "user", "content": "{{{text}}}"} ]}), ], inputParameters: { to: `Arabic`, tokenRatio: 0.2, + model: 'oai-gpt5-chat', }, - inputChunkSize: 500, - model: 'oai-gpt4o', + inputChunkSize: 1000, enableDuplicateRequests: false, + useParallelChunkProcessing: true, + enableCache: true, } \ No newline at end of file diff --git a/server/plugins/gemini15VisionPlugin.js b/server/plugins/gemini15VisionPlugin.js index 6fb83afb..19272584 100644 --- a/server/plugins/gemini15VisionPlugin.js +++ b/server/plugins/gemini15VisionPlugin.js @@ -44,8 +44,9 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin { try { // First try to parse as JSON if it's a string const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart; - const {type, text, image_url, gcs} = part; - let fileUrl = gcs || image_url?.url; + const {type, text, image_url, gcs, url} = part; + // Check for URL in multiple places: gcs, image_url.url, or direct url property + let fileUrl = gcs || image_url?.url || url; if (typeof part === 'string') { return { text: inputPart }; @@ -72,9 +73,12 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin { if (!base64Data) { return null; } + // Extract MIME type from data URL if available + const mimeMatch = fileUrl.match(/data:([^;]+);base64,/); + const mimeType = mimeMatch ? mimeMatch[1] : 'image/jpeg'; return { inlineData: { - mimeType: 'image/jpeg', + mimeType: mimeType, data: base64Data } }; @@ -85,6 +89,15 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin { fileUri: fileUrl } }; + } else if (fileUrl.startsWith('http://') || fileUrl.startsWith('https://')) { + // Gemini can read directly from HTTP/HTTPS URLs using fileData with fileUri + // No need to fetch and convert to base64 + return { + fileData: { + mimeType: mime.lookup(fileUrl) || 'image/jpeg', + fileUri: fileUrl + } + }; } return null; } diff --git a/start.js b/start.js index af6c1e28..0a50937a 100644 --- a/start.js +++ b/start.js @@ -1,3 +1,4 @@ +import 'dotenv/config'; import startServerFactory from './index.js'; (async () => {