From 7b8790af03ac1c1073acf319391f2236f5fcd75a Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 4 Jan 2026 11:45:07 -0700 Subject: [PATCH 01/11] feat: add support for GCP service account email and improve authentication handling - Introduced `gcpServiceAccountEmail` configuration for service account impersonation, recommended over the legacy key method. - Updated authentication logic to support both service account impersonation and legacy service account key. - Enhanced documentation to reflect changes in GCP configuration and usage. - Adjusted file handling to accommodate new authentication methods, ensuring compatibility with existing functionality. --- FILE_SYSTEM_DOCUMENTATION.md | 3 +- README.md | 3 +- config.js | 8 ++- helper-apps/cortex-file-handler/INTERFACE.md | 2 +- .../cortex-file-handler/src/blobHandler.js | 62 +++++++++++++------ .../services/storage/GCSStorageProvider.js | 24 ++++--- .../src/services/storage/StorageFactory.js | 41 ++++++++++-- lib/gcpAuthTokenHelper.js | 29 ++++++--- server/plugins/gemini15VisionPlugin.js | 14 ++++- 9 files changed, 142 insertions(+), 44 deletions(-) diff --git a/FILE_SYSTEM_DOCUMENTATION.md b/FILE_SYSTEM_DOCUMENTATION.md index 8b552390..a1f35840 100644 --- a/FILE_SYSTEM_DOCUMENTATION.md +++ b/FILE_SYSTEM_DOCUMENTATION.md @@ -650,10 +650,11 @@ RemoveFileFromCollection Tool - **Lifecycle**: Azure automatically deletes `retention=temporary` files after 30 days #### Google Cloud Storage (Optional) -- **Enabled**: If `GCP_SERVICE_ACCOUNT_KEY` configured +- **Enabled**: If `GCP_SERVICE_ACCOUNT_EMAIL` (recommended, uses impersonation) or `GCP_SERVICE_ACCOUNT_KEY` (legacy) is configured - **URL Format**: `gs://bucket/path` - **Usage**: Media file chunks, converted files - **No short-lived URLs**: GCS URLs are permanent (no SAS equivalent) +- **Authentication**: See `GCP_IMPERSONATION_SETUP.md` for setup instructions using service account impersonation (recommended method) #### Local Storage (Fallback) - **Used**: If Azure not configured diff --git a/README.md b/README.md index 4c5896f0..8d6cc918 100644 --- a/README.md +++ b/README.md @@ -861,7 +861,8 @@ The following properties can be configured through environment variables or the - `enableDuplicateRequests`: Enable sending duplicate requests if not completed after timeout. Default is true. - `enableGraphqlCache`: Enable GraphQL query caching. Default is false. - `enableRestEndpoints`: Create REST endpoints for pathways as well as GraphQL queries. Default is false. -- `gcpServiceAccountKey`: GCP service account key for authentication. Default is null. +- `gcpServiceAccountKey`: GCP service account key for authentication (legacy method). Default is null. +- `gcpServiceAccountEmail`: GCP service account email for impersonation (recommended method). Default is null. See `GCP_IMPERSONATION_SETUP.md` for setup instructions. - `models`: Object containing the different models used by the project. - `pathways`: Object containing pathways for the project. - `pathwaysPath`: Path to custom pathways. Default is './pathways'. diff --git a/config.js b/config.js index 924f62c8..2325f55e 100644 --- a/config.js +++ b/config.js @@ -130,6 +130,12 @@ var config = convict({ env: 'GCP_SERVICE_ACCOUNT_KEY', sensitive: true }, + gcpServiceAccountEmail: { + format: String, + default: null, + env: 'GCP_SERVICE_ACCOUNT_EMAIL', + sensitive: false + }, azureServicePrincipalCredentials: { format: String, default: null, @@ -941,7 +947,7 @@ if (config.get('entityConstants') && defaultEntityConstants) { config.set('entityConstants', { ...defaultEntityConstants, ...config.get('entityConstants') }); } -if (config.get('gcpServiceAccountKey')) { +if (config.get('gcpServiceAccountEmail') || config.get('gcpServiceAccountKey')) { const gcpAuthTokenHelper = new GcpAuthTokenHelper(config.getProperties()); config.set('gcpAuthTokenHelper', gcpAuthTokenHelper); } diff --git a/helper-apps/cortex-file-handler/INTERFACE.md b/helper-apps/cortex-file-handler/INTERFACE.md index d8851882..5d0f13f5 100644 --- a/helper-apps/cortex-file-handler/INTERFACE.md +++ b/helper-apps/cortex-file-handler/INTERFACE.md @@ -153,7 +153,7 @@ The file handler uses a unified storage approach with Azure Blob Storage: ## Storage Configuration - **Azure**: Enabled if `AZURE_STORAGE_CONNECTION_STRING` is set -- **GCS**: Enabled if `GCP_SERVICE_ACCOUNT_KEY_BASE64` or `GCP_SERVICE_ACCOUNT_KEY` is set +- **GCS**: Enabled if `GCP_SERVICE_ACCOUNT_EMAIL` (recommended, uses impersonation) or `GCP_SERVICE_ACCOUNT_KEY_BASE64`/`GCP_SERVICE_ACCOUNT_KEY` (legacy) is set - **Local**: Used as fallback if Azure is not configured ## Response Format diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js index 07becac5..22596191 100644 --- a/helper-apps/cortex-file-handler/src/blobHandler.js +++ b/helper-apps/cortex-file-handler/src/blobHandler.js @@ -41,37 +41,61 @@ const { SAS_TOKEN_LIFE_DAYS = 30 } = process.env; let GCP_SERVICE_ACCOUNT; let GCP_PROJECT_ID; -try { - const GCP_SERVICE_ACCOUNT_KEY = - process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || - process.env.GCP_SERVICE_ACCOUNT_KEY || - "{}"; - GCP_SERVICE_ACCOUNT = isBase64(GCP_SERVICE_ACCOUNT_KEY) - ? JSON.parse(Buffer.from(GCP_SERVICE_ACCOUNT_KEY, "base64").toString()) - : JSON.parse(GCP_SERVICE_ACCOUNT_KEY); - GCP_PROJECT_ID = GCP_SERVICE_ACCOUNT.project_id; -} catch (error) { - console.warn("Error parsing GCP service account credentials, GCS will not be used:", error.message); - GCP_SERVICE_ACCOUNT = {}; - GCP_PROJECT_ID = null; +// Support both service account impersonation (recommended) and service account key (legacy) +const GCP_SERVICE_ACCOUNT_EMAIL = process.env.GCP_SERVICE_ACCOUNT_EMAIL; +const GCP_PROJECT_ID_ENV = process.env.GCP_PROJECT_ID; + +if (GCP_SERVICE_ACCOUNT_EMAIL) { + // Using impersonation - extract project ID from email if not provided + GCP_PROJECT_ID = GCP_PROJECT_ID_ENV || extractProjectIdFromEmail(GCP_SERVICE_ACCOUNT_EMAIL); + GCP_SERVICE_ACCOUNT = null; // null means use ADC (Application Default Credentials) +} else { + // Fall back to service account key (legacy) + try { + const GCP_SERVICE_ACCOUNT_KEY = + process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || + process.env.GCP_SERVICE_ACCOUNT_KEY || + "{}"; + GCP_SERVICE_ACCOUNT = isBase64(GCP_SERVICE_ACCOUNT_KEY) + ? JSON.parse(Buffer.from(GCP_SERVICE_ACCOUNT_KEY, "base64").toString()) + : JSON.parse(GCP_SERVICE_ACCOUNT_KEY); + GCP_PROJECT_ID = GCP_SERVICE_ACCOUNT.project_id || GCP_PROJECT_ID_ENV; + } catch (error) { + console.warn("Error parsing GCP service account credentials, GCS will not be used:", error.message); + GCP_SERVICE_ACCOUNT = null; + GCP_PROJECT_ID = null; + } +} + +function extractProjectIdFromEmail(email) { + // Service account email format: name@project-id.iam.gserviceaccount.com + const match = email.match(/@([^.]+)\.iam\.gserviceaccount\.com$/); + return match ? match[1] : null; } let gcs; -if (!GCP_PROJECT_ID || !GCP_SERVICE_ACCOUNT) { +if (!GCP_PROJECT_ID) { console.warn( - "No Google Cloud Storage credentials provided - GCS will not be used", + "No Google Cloud project ID provided - GCS will not be used", ); } else { try { - gcs = new Storage({ + const storageConfig = { projectId: GCP_PROJECT_ID, - credentials: GCP_SERVICE_ACCOUNT, - }); + }; + + // If GCP_SERVICE_ACCOUNT is null, Storage will use Application Default Credentials (ADC) + // This enables service account impersonation + if (GCP_SERVICE_ACCOUNT) { + storageConfig.credentials = GCP_SERVICE_ACCOUNT; + } + + gcs = new Storage(storageConfig); // Rest of your Google Cloud operations using gcs object } catch (error) { console.error( - "Google Cloud Storage credentials are invalid - GCS will not be used: ", + "Google Cloud Storage configuration is invalid - GCS will not be used: ", error, ); } diff --git a/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js b/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js index 11c78c42..8f886c38 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +++ b/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js @@ -12,17 +12,27 @@ import axios from "axios"; import { StorageProvider } from "./StorageProvider.js"; export class GCSStorageProvider extends StorageProvider { - constructor(credentials, bucketName) { + constructor(credentials, projectId, bucketName) { super(); - if (!credentials || !bucketName) { - throw new Error("Missing GCS credentials or bucket name"); + if (!bucketName) { + throw new Error("Missing GCS bucket name"); + } + if (!projectId && !credentials?.project_id) { + throw new Error("Missing GCS project ID"); } this.bucketName = bucketName; - this.storage = new Storage({ - projectId: credentials.project_id, - credentials: credentials, - }); + const storageConfig = { + projectId: projectId || credentials.project_id, + }; + + // If credentials is null/undefined, Storage will use Application Default Credentials (ADC) + // This enables service account impersonation + if (credentials) { + storageConfig.credentials = credentials; + } + + this.storage = new Storage(storageConfig); } isConfigured() { diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js index f3687641..8920fdb0 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +++ b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js @@ -58,12 +58,13 @@ export class StorageFactory { getGCSProvider() { const key = "gcs"; if (!this.providers.has(key)) { - const credentials = this.parseGCSCredentials(); - if (!credentials) { + const config = this.parseGCSConfig(); + if (!config) { return null; } const provider = new GCSStorageProvider( - credentials, + config.credentials, + config.projectId, GCS_BUCKETNAME, ); this.providers.set(key, provider); @@ -86,7 +87,23 @@ export class StorageFactory { return this.providers.get(key); } - parseGCSCredentials() { + parseGCSConfig() { + // Support service account impersonation (recommended) + const serviceAccountEmail = process.env.GCP_SERVICE_ACCOUNT_EMAIL; + const projectId = process.env.GCP_PROJECT_ID; + + if (serviceAccountEmail) { + // Using impersonation - extract project ID from email if not provided + const extractedProjectId = projectId || this.extractProjectIdFromEmail(serviceAccountEmail); + if (!extractedProjectId) { + console.error("GCP_PROJECT_ID is required when using GCP_SERVICE_ACCOUNT_EMAIL"); + return null; + } + // Return null credentials to use ADC (Application Default Credentials) + return { credentials: null, projectId: extractedProjectId }; + } + + // Fall back to service account key (legacy) const key = process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY; @@ -95,16 +112,28 @@ export class StorageFactory { } try { + let credentials; if (this.isBase64(key)) { - return JSON.parse(Buffer.from(key, "base64").toString()); + credentials = JSON.parse(Buffer.from(key, "base64").toString()); + } else { + credentials = JSON.parse(key); } - return JSON.parse(key); + return { + credentials: credentials, + projectId: credentials.project_id || projectId, + }; } catch (error) { console.error("Error parsing GCS credentials:", error); return null; } } + extractProjectIdFromEmail(email) { + // Service account email format: name@project-id.iam.gserviceaccount.com + const match = email.match(/@([^.]+)\.iam\.gserviceaccount\.com$/); + return match ? match[1] : null; + } + isBase64(str) { try { return btoa(atob(str)) === str; diff --git a/lib/gcpAuthTokenHelper.js b/lib/gcpAuthTokenHelper.js index 0888392c..8b59d3af 100644 --- a/lib/gcpAuthTokenHelper.js +++ b/lib/gcpAuthTokenHelper.js @@ -2,14 +2,29 @@ import { GoogleAuth } from 'google-auth-library'; class GcpAuthTokenHelper { constructor(config) { - const creds = config.gcpServiceAccountKey ? JSON.parse(config.gcpServiceAccountKey) : null; - if (!creds) { - throw new Error('GCP_SERVICE_ACCOUNT_KEY is missing or undefined'); + const serviceAccountEmail = config.gcpServiceAccountEmail; + const serviceAccountKey = config.gcpServiceAccountKey; + + // Support both service account key (legacy) and impersonation (recommended) + if (serviceAccountEmail) { + // Use service account impersonation (recommended) + // When using impersonation, GoogleAuth will use Application Default Credentials (ADC) + // ADC should be configured via: gcloud auth application-default login --impersonate-service-account=EMAIL + // Passing no credentials means GoogleAuth will use ADC + this.authClient = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'], + }); + } else if (serviceAccountKey) { + // Fall back to service account key (legacy) + const creds = JSON.parse(serviceAccountKey); + this.authClient = new GoogleAuth({ + credentials: creds, + scopes: ['https://www.googleapis.com/auth/cloud-platform'], + }); + } else { + throw new Error('Either GCP_SERVICE_ACCOUNT_EMAIL (for impersonation) or GCP_SERVICE_ACCOUNT_KEY must be provided'); } - this.authClient = new GoogleAuth({ - credentials: creds, - scopes: ['https://www.googleapis.com/auth/cloud-platform'], - }); + this.token = null; this.expiry = null; } diff --git a/server/plugins/gemini15VisionPlugin.js b/server/plugins/gemini15VisionPlugin.js index 6fb83afb..e7edb29c 100644 --- a/server/plugins/gemini15VisionPlugin.js +++ b/server/plugins/gemini15VisionPlugin.js @@ -72,9 +72,12 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin { if (!base64Data) { return null; } + // Extract MIME type from data URL if available + const mimeMatch = fileUrl.match(/data:([^;]+);base64,/); + const mimeType = mimeMatch ? mimeMatch[1] : 'image/jpeg'; return { inlineData: { - mimeType: 'image/jpeg', + mimeType: mimeType, data: base64Data } }; @@ -85,6 +88,15 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin { fileUri: fileUrl } }; + } else if (fileUrl.startsWith('http://') || fileUrl.startsWith('https://')) { + // Gemini can read directly from HTTP/HTTPS URLs using fileData with fileUri + // No need to fetch and convert to base64 + return { + fileData: { + mimeType: mime.lookup(fileUrl) || 'image/jpeg', + fileUri: fileUrl + } + }; } return null; } From 29c1e17477bfa82589812366bae1d88bd0360885 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 4 Jan 2026 18:14:41 -0700 Subject: [PATCH 02/11] Add production deployment files --- .github/workflows/deploy.yml | 179 +++++ .gitignore | 2 + Dockerfile | 28 + config/default.json | 681 +++++++++++++++++- deploy/setup-server.sh | 140 ++++ docker-compose.prod.yml | 122 ++++ .../cortex-file-handler/src/constants.js | 15 +- .../entity/tools/sys_tool_cognitive_search.js | 3 + start.js | 1 + 9 files changed, 1169 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 Dockerfile create mode 100644 deploy/setup-server.sh create mode 100644 docker-compose.prod.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..b54f8b42 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,179 @@ +name: Build and Deploy Cortex + +on: + push: + branches: + - main + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-cortex: + name: Build Cortex Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,prefix= + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Cortex image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-file-handler: + name: Build File Handler Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-file-handler + tags: | + type=sha,prefix= + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push File Handler image + uses: docker/build-push-action@v5 + with: + context: ./helper-apps/cortex-file-handler + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + name: Deploy to Hetzner + runs-on: ubuntu-latest + needs: [build-cortex, build-file-handler] + environment: production + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Copy docker-compose to server + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + source: "docker-compose.prod.yml" + target: "/opt/cortex" + + - name: Deploy to server + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + cd /opt/cortex + + # Login to GitHub Container Registry + echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + # Pull the new images + docker pull ghcr.io/${{ github.repository }}:latest + docker pull ghcr.io/${{ github.repository }}-file-handler:latest + + # Update the stack + export IMAGE_TAG=latest + export GITHUB_REPOSITORY_CORTEX=${{ github.repository }} + + # Load environment variables + set -a + source .env + set +a + + # Deploy with docker-compose + docker compose -f docker-compose.prod.yml up -d --remove-orphans + + # Clean up old images + docker image prune -f + + - name: Health check + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + echo "Waiting for services to start..." + sleep 15 + + # Check Cortex + if docker ps | grep -q cortex; then + echo "✅ Cortex is running" + else + echo "❌ Cortex failed to start" + docker logs cortex --tail 30 + exit 1 + fi + + # Check File Handler + if docker ps | grep -q cortex-file-handler; then + echo "✅ File Handler is running" + else + echo "❌ File Handler failed to start" + docker logs cortex-file-handler --tail 30 + exit 1 + fi + + # Check Redis + if docker ps | grep -q redis; then + echo "✅ Redis is running" + else + echo "❌ Redis failed to start" + exit 1 + fi + diff --git a/.gitignore b/.gitignore index c2a8d18f..d87d5d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .DS_Store node_modules/ .env +.env.* +deploy/env.production .vscode/ **/__pycache__ **/.venv diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1e4b8b32 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM node:18-alpine + +WORKDIR /app + +# Install build dependencies +RUN apk add --no-cache python3 make g++ + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm ci --only=production + +# Copy application code +COPY . . + +# Remove devDependencies build tools +RUN apk del python3 make g++ + +# Expose GraphQL port +EXPOSE 4000 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:4000/.well-known/apollo/server-health || exit 1 + +CMD ["npm", "start"] + diff --git a/config/default.json b/config/default.json index 9e26dfee..e0b116cb 100644 --- a/config/default.json +++ b/config/default.json @@ -1 +1,680 @@ -{} \ No newline at end of file +{ + "defaultModelName": "oai-gpt41", + "models": { + "xai-grok-4": { + "type": "GROK-VISION", + "emulateOpenAIChatModel": "grok-4", + "restStreaming": { + "inputParameters": { + "stream": false, + "search_parameters": "" + } + }, + "endpoints": [ + { + "name": "XAI GROK-4", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-0709" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-code-fast-1": { + "type": "GROK-VISION", + "endpoints": [ + { + "name": "XAI GROK-CODE-FAST-1", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-code-fast-1" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 2000000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-1-fast-reasoning": { + "type": "GROK-VISION", + "endpoints": [ + { + "name": "XAI GROK-4.1-FAST-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-1-fast-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 2000000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-1-fast-non-reasoning": { + "type": "GROK-VISION", + "endpoints": [ + { + "name": "XAI GROK-4.1-FAST-NON-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-1-fast-non-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-1-fast-responses": { + "type": "GROK-RESPONSES", + "emulateOpenAIChatModel": "grok-4-1-fast", + "restStreaming": { + "inputParameters": { + "stream": true, + "tools": "", + "inline_citations": true + } + }, + "endpoints": [ + { + "name": "XAI GROK-4.1-FAST-RESPONSES", + "url": "https://api.x.ai/v1/responses", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-1-fast" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-fast-reasoning": { + "type": "GROK-VISION", + "emulateOpenAIChatModel": "grok-4-fast-reasoning", + "restStreaming": { + "inputParameters": { + "stream": false, + "search_parameters": "" + } + }, + "endpoints": [ + { + "name": "XAI GROK-4-FAST-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-fast-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 2000000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "xai-grok-4-fast-non-reasoning": { + "type": "GROK-VISION", + "emulateOpenAIChatModel": "grok-4-fast-non-reasoning", + "restStreaming": { + "inputParameters": { + "stream": false, + "search_parameters": "" + } + }, + "endpoints": [ + { + "name": "XAI GROK-4-FAST-NON-REASONING", + "url": "https://api.x.ai/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{XAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "grok-4-fast-non-reasoning" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 256000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "oai-whisper": { + "type": "OPENAI-WHISPER", + "url": "https://api.openai.com/v1/audio/transcriptions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}" + }, + "params": { + "model": "whisper-1" + } + }, + "oai-embeddings": { + "type": "OPENAI-EMBEDDINGS", + "url": "https://api.openai.com/v1/embeddings", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "text-embedding-ada-002" + }, + "maxTokenLength": 8192 + }, + "oai-text-embedding-3-large": { + "type": "OPENAI-EMBEDDINGS", + "url": "https://api.openai.com/v1/embeddings", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "text-embedding-3-large" + }, + "maxTokenLength": 8192 + }, + "oai-text-embedding-3-small": { + "type": "OPENAI-EMBEDDINGS", + "url": "https://api.openai.com/v1/embeddings", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "text-embedding-3-small" + }, + "maxTokenLength": 8192 + }, + "oai-gpt4o": { + "type": "OPENAI-VISION", + "emulateOpenAIChatModel": "gpt-4o", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4o" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 131072, + "maxReturnTokens": 4096, + "supportsStreaming": true + }, + "oai-o3": { + "type": "OPENAI-REASONING", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "o3" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 100000, + "supportsStreaming": true + }, + "oai-router": { + "type": "OPENAI-REASONING-VISION", + "endpoints": [ + { + "name": "EUS2 AZURE MODEL ROUTER", + "url": "https://archipelago-foundry-resource.cognitiveservices.azure.com/openai/deployments/archipelago-azure-model-router/chat/completions?api-version=2025-03-01-preview", + "headers": { + "api-key": "{{ARCHIPELAGO_FOUNDRY_RESOURCE_KEY}}", + "Content-Type": "application/json" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 100000, + "supportsStreaming": true + }, + "oai-gpt52": { + "type": "OPENAI-REASONING-VISION", + "emulateOpenAIChatModel": "gpt-5.2", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-5.2" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 400000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "oai-gpt51": { + "type": "OPENAI-REASONING-VISION", + "emulateOpenAIChatModel": "gpt-5.1", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-5.1" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 400000, + "maxReturnTokens": 128000, + "supportsStreaming": true + }, + "oai-gpt41": { + "type": "OPENAI-VISION", + "emulateOpenAIChatModel": "gpt-4.1", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4.1" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 1000000, + "maxReturnTokens": 32768, + "supportsStreaming": true + }, + "oai-gpt41-mini": { + "type": "OPENAI-VISION", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4.1-mini" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 1000000, + "maxReturnTokens": 32768, + "supportsStreaming": true + }, + "oai-gpt41-nano": { + "type": "OPENAI-VISION", + "emulateOpenAIChatModel": "gpt-4.1-nano", + "endpoints": [ + { + "name": "default", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer {{OPENAI_API_KEY}}", + "Content-Type": "application/json" + }, + "params": { + "model": "gpt-4.1-nano" + }, + "requestsPerSecond": 50 + } + ], + "maxTokenLength": 1000000, + "maxReturnTokens": 8192, + "supportsStreaming": true + }, + "gemini-flash-25-vision": { + "type": "GEMINI-1.5-VISION", + "emulateOpenAIChatModel": "gemini-flash-25", + "restStreaming": { + "geminiSafetySettings": [ + {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_ONLY_HIGH"} + ] + }, + "endpoints": [ + { + "name": "GLOBAL GEMINI-FLASH-2.5-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-2.5-flash", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "gemini-flash-25-image": { + "type": "GEMINI-2.5-IMAGE", + "endpoints": [ + { + "name": "GLOBAL GEMINI-FLASH-2.5-IMAGE", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-2.5-flash-image-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 32768, + "maxReturnTokens": 16384, + "supportsStreaming": true + }, + "gemini-pro-3-image": { + "type": "GEMINI-3-IMAGE", + "endpoints": [ + { + "name": "GLOBAL GEMINI-PRO-3.0-IMAGE", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-3-pro-image-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 64576, + "maxReturnTokens": 32768, + "supportsStreaming": true + }, + "gemini-pro-3-vision": { + "type": "GEMINI-3-REASONING-VISION", + "emulateOpenAIChatModel": "gemini-pro-3", + "endpoints": [ + { + "name": "GLOBAL GEMINI-PRO-3.0-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-3-pro-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "gemini-flash-3-vision": { + "type": "GEMINI-3-REASONING-VISION", + "emulateOpenAIChatModel": "gemini-flash-3", + "endpoints": [ + { + "name": "GLOBAL GEMINI-FLASH-3-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-3-flash-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "gemini-pro-25-vision": { + "type": "GEMINI-1.5-VISION", + "emulateOpenAIChatModel": "gemini-pro-25", + "restStreaming": { + "geminiSafetySettings": [ + {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_ONLY_HIGH"}, + {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_ONLY_HIGH"} + ] + }, + "endpoints": [ + { + "name": "GLOBAL GEMINI-PRO-2.5-VISION", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/gemini-2.5-pro", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "requestsPerSecond": 10, + "maxTokenLength": 1048576, + "maxReturnTokens": 65535, + "supportsStreaming": true + }, + "claude-45-sonnet-vertex": { + "type": "CLAUDE-4-VERTEX", + "emulateOpenAIChatModel": "claude-4.5-sonnet", + "endpoints": [ + { + "name": "GLOBAL SONNET 4.5", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/anthropic/models/claude-sonnet-4-5@20250929", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 64000, + "maxImageSize": 31457280, + "supportsStreaming": true + }, + "claude-45-haiku-vertex": { + "type": "CLAUDE-4-VERTEX", + "emulateOpenAIChatModel": "claude-4.5-haiku", + "endpoints": [ + { + "name": "GLOBAL HAIKU 4.5", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/anthropic/models/claude-haiku-4-5@20251001", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 10 + } + ], + "maxTokenLength": 200000, + "maxReturnTokens": 64000, + "maxImageSize": 31457280, + "supportsStreaming": true + }, + "local-llama30b": { + "type": "LOCAL-CPP-MODEL", + "executablePath": "../llm/llama.cpp/main", + "args": [ + "-m", "/Volumes/JMac External/LLM/LLaMA/30B/ggml-model-q4_0.bin", + "-ngl", "1", + "--repeat_last_n", "1024", + "--repeat_penalty", "1.17647", + "--keep", "-1", + "-t", "8", + "--temp", "0.7", + "--top_k", "40", + "--top_p", "0.5", + "-r","<|im_end|>" + ], + "requestsPerSecond": 10, + "maxTokenLength": 2048 + }, + "veo-2.0-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 2.0", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-2.0-generate-001", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 2 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.0-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.0 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.0-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.0-fast-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.0 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.0-fast-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.1-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.1 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.1-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "veo-3.1-fast-generate": { + "type": "VEO-VIDEO", + "endpoints": [ + { + "name": "GLOBAL VEO 3.0 PREVIEW", + "url": "https://aiplatform.googleapis.com/v1/projects/enntityvertex/locations/global/publishers/google/models/veo-3.1-fast-generate-preview", + "headers": { + "Content-Type": "application/json" + }, + "requestsPerSecond": 1 + } + ], + "maxTokenLength": 1000, + "supportsStreaming": false + }, + "replicate-flux-2-pro": { + "type": "REPLICATE-API", + "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-2-pro/predictions", + "headers": { + "Prefer": "wait", + "Authorization": "Token {{REPLICATE_API_KEY}}", + "Content-Type": "application/json" + } + }, + "replicate-qwen-image-edit-2511": { + "type": "REPLICATE-API", + "url": "https://api.replicate.com/v1/models/qwen/qwen-image-edit-2511/predictions", + "headers": { + "Prefer": "wait", + "Authorization": "Token {{REPLICATE_API_KEY}}", + "Content-Type": "application/json" + } + } + }, + "enableCache": true, + "enableRestEndpoints": true, + "enableDuplicateRequests": true, + "subscriptionKeepAlive": 0, + "entityConfig": { + "labeeb": { + "name": "Labeeb", + "isDefault": true, + "useMemory": true, + "useContinuityMemory": true, + "description": "Labeeb is a self-aware AI entity that can help you with your tasks and answer your questions.", + "instructions": "", + "tools": ["*"] + } + }, + "entityConstants": { + "AI_STYLE_OPENAI_PREVIEW": "oai-gpt52", + "AI_STYLE_OPENAI_PREVIEW_RESEARCH": "oai-gpt52", + "AI_STYLE_OPENAI": "oai-gpt51", + "AI_STYLE_OPENAI_RESEARCH": "oai-gpt51", + "AI_STYLE_OPENAI_LEGACY": "oai-gpt41", + "AI_STYLE_OPENAI_LEGACY_RESEARCH": "oai-o3", + "AI_STYLE_ANTHROPIC": "claude-45-sonnet-vertex", + "AI_STYLE_ANTHROPIC_RESEARCH": "claude-41-opus-vertex", + "AI_STYLE_XAI": "xai-grok-4-1-fast-reasoning", + "AI_STYLE_XAI_RESEARCH": "xai-grok-4-1-fast-reasoning", + "AI_STYLE_GOOGLE": "gemini-flash-3-vision", + "AI_STYLE_GOOGLE_RESEARCH": "gemini-pro-3-vision" + } +} \ No newline at end of file diff --git a/deploy/setup-server.sh b/deploy/setup-server.sh new file mode 100644 index 00000000..db6f5ffe --- /dev/null +++ b/deploy/setup-server.sh @@ -0,0 +1,140 @@ +#!/bin/bash +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Cortex Server Setup Script ║${NC}" +echo -e "${GREEN}║ For Hetzner Cloud (Ubuntu 22.04) ║${NC}" +echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}" +echo "" + +if [ "$EUID" -ne 0 ]; then + echo -e "${RED}Please run as root (use sudo)${NC}" + exit 1 +fi + +echo -e "${YELLOW}Please provide the following information:${NC}" +echo "" + +read -p "Domain name for Cortex API (e.g., api.example.com): " DOMAIN +read -p "Email for SSL certificates: " ACME_EMAIL +read -p "GitHub repository for Cortex (e.g., enntity/cortex): " GITHUB_REPOSITORY_CORTEX + +echo "" +echo -e "${YELLOW}API Keys (press Enter to skip any):${NC}" +read -p "OpenAI API Key: " OPENAI_API_KEY +read -p "Anthropic API Key: " ANTHROPIC_API_KEY +read -p "XAI (Grok) API Key: " XAI_API_KEY +read -p "GCP Service Account Email (for Gemini/Vertex): " GCP_SERVICE_ACCOUNT_EMAIL +read -p "GCP Project ID: " GCP_PROJECT_ID + +echo "" +echo -e "${YELLOW}Storage Configuration:${NC}" +read -p "Azure Storage Connection String (optional): " AZURE_STORAGE_CONNECTION_STRING +read -p "Azure Storage Container Name (default: cortex-files): " AZURE_STORAGE_CONTAINER_NAME +AZURE_STORAGE_CONTAINER_NAME=${AZURE_STORAGE_CONTAINER_NAME:-cortex-files} +read -p "GCS Bucket Name (optional): " GCS_BUCKET_NAME + +echo "" +echo -e "${YELLOW}Security:${NC}" +read -p "Cortex API Keys (comma-separated, for client auth, optional): " CORTEX_API_KEYS + +# Generate Redis password +REDIS_PASSWORD=$(openssl rand -base64 24 | tr -d '=+/') +echo -e "${GREEN}Generated Redis password${NC}" + +# Generate Traefik dashboard credentials +read -p "Traefik dashboard username (default: admin): " TRAEFIK_USER +TRAEFIK_USER=${TRAEFIK_USER:-admin} +TRAEFIK_PASSWORD=$(openssl rand -base64 12 | tr -d '=+/') +TRAEFIK_DASHBOARD_AUTH=$(htpasswd -nb "$TRAEFIK_USER" "$TRAEFIK_PASSWORD" | sed 's/\$/\$\$/g') + +echo "" +echo -e "${GREEN}Step 1: Updating system...${NC}" +apt-get update && apt-get upgrade -y + +echo -e "${GREEN}Step 2: Installing Docker...${NC}" +if ! command -v docker &> /dev/null; then + curl -fsSL https://get.docker.com | sh + systemctl enable docker + systemctl start docker +fi + +echo -e "${GREEN}Step 3: Installing tools...${NC}" +apt-get install -y apache2-utils curl git ufw + +echo -e "${GREEN}Step 4: Configuring firewall...${NC}" +ufw allow 22/tcp +ufw allow 80/tcp +ufw allow 443/tcp +# Allow Redis from private network only +ufw allow from 10.0.0.0/24 to any port 6379 +ufw --force enable + +echo -e "${GREEN}Step 5: Creating application directory...${NC}" +mkdir -p /opt/cortex +cd /opt/cortex + +echo -e "${GREEN}Step 6: Creating environment file...${NC}" +cat > .env << EOF +# Domain Configuration +DOMAIN=${DOMAIN} +ACME_EMAIL=${ACME_EMAIL} + +# GitHub Container Registry +GITHUB_REPOSITORY_CORTEX=${GITHUB_REPOSITORY_CORTEX} +IMAGE_TAG=latest + +# Redis (shared with Concierge) +REDIS_PASSWORD=${REDIS_PASSWORD} + +# API Keys +OPENAI_API_KEY=${OPENAI_API_KEY} +ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} +XAI_API_KEY=${XAI_API_KEY} +GCP_SERVICE_ACCOUNT_EMAIL=${GCP_SERVICE_ACCOUNT_EMAIL} +GCP_PROJECT_ID=${GCP_PROJECT_ID} + +# Storage +AZURE_STORAGE_CONNECTION_STRING=${AZURE_STORAGE_CONNECTION_STRING} +AZURE_STORAGE_CONTAINER_NAME=${AZURE_STORAGE_CONTAINER_NAME} +GCS_BUCKET_NAME=${GCS_BUCKET_NAME} + +# Security +CORTEX_API_KEYS=${CORTEX_API_KEYS} + +# Traefik Dashboard +TRAEFIK_DASHBOARD_AUTH=${TRAEFIK_DASHBOARD_AUTH} +EOF + +chmod 600 .env + +echo "" +echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Setup Complete! ║${NC}" +echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${YELLOW}IMPORTANT - Save this Redis password for Concierge server:${NC}" +echo -e "${RED}REDIS_PASSWORD=${REDIS_PASSWORD}${NC}" +echo "" +echo -e "${YELLOW}DNS Configuration:${NC}" +echo " ${DOMAIN} → $(curl -s ifconfig.me)" +echo "" +echo -e "${YELLOW}GitHub Secrets for Cortex repo:${NC}" +echo " DEPLOY_HOST: $(curl -s ifconfig.me)" +echo " DEPLOY_USER: root" +echo " DEPLOY_SSH_KEY: (your private SSH key)" +echo "" +echo -e "${YELLOW}Traefik Dashboard:${NC}" +echo " URL: https://traefik.${DOMAIN}" +echo " Username: ${TRAEFIK_USER}" +echo " Password: ${TRAEFIK_PASSWORD}" +echo "" +echo "Environment file saved to: /opt/cortex/.env" +echo "" + diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..10c8140e --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,122 @@ +version: "3.8" + +services: + # Traefik - Reverse proxy with automatic SSL + traefik: + image: traefik:v3.0 + container_name: traefik + restart: unless-stopped + command: + - "--api.dashboard=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.web.http.redirections.entrypoint.to=websecure" + - "--entrypoints.web.http.redirections.entrypoint.scheme=https" + - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true" + - "--certificatesresolvers.letsencrypt.acme.email=${ACME_EMAIL}" + - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "traefik-certificates:/letsencrypt" + networks: + - web + labels: + - "traefik.enable=true" + - "traefik.http.routers.dashboard.rule=Host(`traefik.${DOMAIN}`)" + - "traefik.http.routers.dashboard.entrypoints=websecure" + - "traefik.http.routers.dashboard.tls.certresolver=letsencrypt" + - "traefik.http.routers.dashboard.service=api@internal" + - "traefik.http.routers.dashboard.middlewares=auth" + - "traefik.http.middlewares.auth.basicauth.users=${TRAEFIK_DASHBOARD_AUTH}" + + # Redis - Shared cache/queue for Cortex, File Handler, and Concierge + redis: + image: redis:7-alpine + container_name: redis + restart: unless-stopped + command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD} --bind 0.0.0.0 + ports: + # Expose on private network interface for Concierge to connect + - "10.0.0.2:6379:6379" + volumes: + - redis-data:/data + networks: + - internal + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # Cortex - GraphQL API for AI + cortex: + image: ghcr.io/${GITHUB_REPOSITORY_CORTEX}:${IMAGE_TAG:-latest} + container_name: cortex + restart: unless-stopped + env_file: + - .env + environment: + # Override/ensure these critical values + - NODE_ENV=production + - PORT=4000 + # Internal container networking + - REDIS_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 + - STORAGE_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 + - WHISPER_MEDIA_API_URL=http://file-handler:7071 + depends_on: + redis: + condition: service_healthy + networks: + - web + - internal + labels: + - "traefik.enable=true" + - "traefik.http.routers.cortex.rule=Host(`${DOMAIN}`)" + - "traefik.http.routers.cortex.entrypoints=websecure" + - "traefik.http.routers.cortex.tls.certresolver=letsencrypt" + - "traefik.http.services.cortex.loadbalancer.server.port=4000" + - "traefik.docker.network=cortex_web" + + # Cortex File Handler - File processing service + file-handler: + image: ghcr.io/${GITHUB_REPOSITORY_CORTEX}-file-handler:${IMAGE_TAG:-latest} + container_name: cortex-file-handler + restart: unless-stopped + env_file: + - .env + environment: + # Override/ensure these critical values + - NODE_ENV=production + - PORT=7071 + - REDIS_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 + depends_on: + redis: + condition: service_healthy + networks: + - web + - internal + labels: + - "traefik.enable=true" + - "traefik.http.routers.filehandler.rule=Host(`${DOMAIN}`) && PathPrefix(`/file-handler`)" + - "traefik.http.routers.filehandler.entrypoints=websecure" + - "traefik.http.routers.filehandler.tls.certresolver=letsencrypt" + - "traefik.http.services.filehandler.loadbalancer.server.port=7071" + - "traefik.http.middlewares.filehandler-strip.stripprefix.prefixes=/file-handler" + - "traefik.http.routers.filehandler.middlewares=filehandler-strip" + - "traefik.docker.network=cortex_web" + +networks: + web: + name: cortex_web + internal: + name: cortex_internal + +volumes: + traefik-certificates: + redis-data: + diff --git a/helper-apps/cortex-file-handler/src/constants.js b/helper-apps/cortex-file-handler/src/constants.js index 00df7b02..e969db7e 100644 --- a/helper-apps/cortex-file-handler/src/constants.js +++ b/helper-apps/cortex-file-handler/src/constants.js @@ -170,4 +170,17 @@ export const getDefaultContainerName = () => { // Export constant - evaluated at module load time, but getContainerName() handles defaults export const AZURE_STORAGE_CONTAINER_NAME = getContainerName(); -export const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles"; + +// GCS bucket name must be explicitly set - no default to prevent accidental bucket usage +const getGCSBucketName = () => { + const bucketName = process.env.GCS_BUCKETNAME; + if (!bucketName || bucketName.trim() === "") { + throw new Error( + "GCS_BUCKETNAME environment variable is required but not set. " + + "Please set GCS_BUCKETNAME in your environment or .env file." + ); + } + return bucketName.trim(); +}; + +export const GCS_BUCKETNAME = getGCSBucketName(); diff --git a/pathways/system/entity/tools/sys_tool_cognitive_search.js b/pathways/system/entity/tools/sys_tool_cognitive_search.js index 924fe76d..3c9326c9 100644 --- a/pathways/system/entity/tools/sys_tool_cognitive_search.js +++ b/pathways/system/entity/tools/sys_tool_cognitive_search.js @@ -55,6 +55,7 @@ export default { }, { type: "function", + enabled: false, icon: "📰", function: { name: "SearchAJA", @@ -89,6 +90,7 @@ export default { }, { type: "function", + enabled: false, icon: "📰", function: { name: "SearchAJE", @@ -123,6 +125,7 @@ export default { }, { type: "function", + enabled: false, icon: "⚡️", function: { name: "SearchWires", diff --git a/start.js b/start.js index af6c1e28..0a50937a 100644 --- a/start.js +++ b/start.js @@ -1,3 +1,4 @@ +import 'dotenv/config'; import startServerFactory from './index.js'; (async () => { From 0a520ef614eac622be381a6727f931e34438dbda Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 4 Jan 2026 18:23:40 -0700 Subject: [PATCH 03/11] Not needed --- deploy/setup-server.sh | 140 ----------------------------------------- 1 file changed, 140 deletions(-) delete mode 100644 deploy/setup-server.sh diff --git a/deploy/setup-server.sh b/deploy/setup-server.sh deleted file mode 100644 index db6f5ffe..00000000 --- a/deploy/setup-server.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/bash -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║ Cortex Server Setup Script ║${NC}" -echo -e "${GREEN}║ For Hetzner Cloud (Ubuntu 22.04) ║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}" -echo "" - -if [ "$EUID" -ne 0 ]; then - echo -e "${RED}Please run as root (use sudo)${NC}" - exit 1 -fi - -echo -e "${YELLOW}Please provide the following information:${NC}" -echo "" - -read -p "Domain name for Cortex API (e.g., api.example.com): " DOMAIN -read -p "Email for SSL certificates: " ACME_EMAIL -read -p "GitHub repository for Cortex (e.g., enntity/cortex): " GITHUB_REPOSITORY_CORTEX - -echo "" -echo -e "${YELLOW}API Keys (press Enter to skip any):${NC}" -read -p "OpenAI API Key: " OPENAI_API_KEY -read -p "Anthropic API Key: " ANTHROPIC_API_KEY -read -p "XAI (Grok) API Key: " XAI_API_KEY -read -p "GCP Service Account Email (for Gemini/Vertex): " GCP_SERVICE_ACCOUNT_EMAIL -read -p "GCP Project ID: " GCP_PROJECT_ID - -echo "" -echo -e "${YELLOW}Storage Configuration:${NC}" -read -p "Azure Storage Connection String (optional): " AZURE_STORAGE_CONNECTION_STRING -read -p "Azure Storage Container Name (default: cortex-files): " AZURE_STORAGE_CONTAINER_NAME -AZURE_STORAGE_CONTAINER_NAME=${AZURE_STORAGE_CONTAINER_NAME:-cortex-files} -read -p "GCS Bucket Name (optional): " GCS_BUCKET_NAME - -echo "" -echo -e "${YELLOW}Security:${NC}" -read -p "Cortex API Keys (comma-separated, for client auth, optional): " CORTEX_API_KEYS - -# Generate Redis password -REDIS_PASSWORD=$(openssl rand -base64 24 | tr -d '=+/') -echo -e "${GREEN}Generated Redis password${NC}" - -# Generate Traefik dashboard credentials -read -p "Traefik dashboard username (default: admin): " TRAEFIK_USER -TRAEFIK_USER=${TRAEFIK_USER:-admin} -TRAEFIK_PASSWORD=$(openssl rand -base64 12 | tr -d '=+/') -TRAEFIK_DASHBOARD_AUTH=$(htpasswd -nb "$TRAEFIK_USER" "$TRAEFIK_PASSWORD" | sed 's/\$/\$\$/g') - -echo "" -echo -e "${GREEN}Step 1: Updating system...${NC}" -apt-get update && apt-get upgrade -y - -echo -e "${GREEN}Step 2: Installing Docker...${NC}" -if ! command -v docker &> /dev/null; then - curl -fsSL https://get.docker.com | sh - systemctl enable docker - systemctl start docker -fi - -echo -e "${GREEN}Step 3: Installing tools...${NC}" -apt-get install -y apache2-utils curl git ufw - -echo -e "${GREEN}Step 4: Configuring firewall...${NC}" -ufw allow 22/tcp -ufw allow 80/tcp -ufw allow 443/tcp -# Allow Redis from private network only -ufw allow from 10.0.0.0/24 to any port 6379 -ufw --force enable - -echo -e "${GREEN}Step 5: Creating application directory...${NC}" -mkdir -p /opt/cortex -cd /opt/cortex - -echo -e "${GREEN}Step 6: Creating environment file...${NC}" -cat > .env << EOF -# Domain Configuration -DOMAIN=${DOMAIN} -ACME_EMAIL=${ACME_EMAIL} - -# GitHub Container Registry -GITHUB_REPOSITORY_CORTEX=${GITHUB_REPOSITORY_CORTEX} -IMAGE_TAG=latest - -# Redis (shared with Concierge) -REDIS_PASSWORD=${REDIS_PASSWORD} - -# API Keys -OPENAI_API_KEY=${OPENAI_API_KEY} -ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} -XAI_API_KEY=${XAI_API_KEY} -GCP_SERVICE_ACCOUNT_EMAIL=${GCP_SERVICE_ACCOUNT_EMAIL} -GCP_PROJECT_ID=${GCP_PROJECT_ID} - -# Storage -AZURE_STORAGE_CONNECTION_STRING=${AZURE_STORAGE_CONNECTION_STRING} -AZURE_STORAGE_CONTAINER_NAME=${AZURE_STORAGE_CONTAINER_NAME} -GCS_BUCKET_NAME=${GCS_BUCKET_NAME} - -# Security -CORTEX_API_KEYS=${CORTEX_API_KEYS} - -# Traefik Dashboard -TRAEFIK_DASHBOARD_AUTH=${TRAEFIK_DASHBOARD_AUTH} -EOF - -chmod 600 .env - -echo "" -echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║ Setup Complete! ║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo -e "${YELLOW}IMPORTANT - Save this Redis password for Concierge server:${NC}" -echo -e "${RED}REDIS_PASSWORD=${REDIS_PASSWORD}${NC}" -echo "" -echo -e "${YELLOW}DNS Configuration:${NC}" -echo " ${DOMAIN} → $(curl -s ifconfig.me)" -echo "" -echo -e "${YELLOW}GitHub Secrets for Cortex repo:${NC}" -echo " DEPLOY_HOST: $(curl -s ifconfig.me)" -echo " DEPLOY_USER: root" -echo " DEPLOY_SSH_KEY: (your private SSH key)" -echo "" -echo -e "${YELLOW}Traefik Dashboard:${NC}" -echo " URL: https://traefik.${DOMAIN}" -echo " Username: ${TRAEFIK_USER}" -echo " Password: ${TRAEFIK_PASSWORD}" -echo "" -echo "Environment file saved to: /opt/cortex/.env" -echo "" - From ef865b48a4b8272b3ccdcd6ab6c160309a965887 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 4 Jan 2026 18:53:34 -0700 Subject: [PATCH 04/11] refactor: replace GCS_BUCKETNAME constant with getGCSBucketName function for improved lazy evaluation - Updated blobHandler, constants, and various services to utilize getGCSBucketName for retrieving the GCS bucket name. - Enhanced error handling for GCS bucket name retrieval, ensuring it only throws errors when accessed. - Modified tests to reflect changes in GCS bucket name handling, ensuring compatibility with the new function. --- .../cortex-file-handler/src/blobHandler.js | 16 +++---- .../cortex-file-handler/src/constants.js | 46 +++++++++++++++---- .../src/services/ConversionService.js | 31 ++++++++++--- .../src/services/storage/StorageFactory.js | 4 +- .../tests/FileConversionService.test.js | 38 ++++++++++++++- .../tests/conversionResilience.test.js | 6 +-- .../tests/storage/GCSStorageProvider.test.js | 9 ++-- .../tests/storage/StorageFactory.test.js | 14 +++--- 8 files changed, 126 insertions(+), 38 deletions(-) diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js index 22596191..76328e80 100644 --- a/helper-apps/cortex-file-handler/src/blobHandler.js +++ b/helper-apps/cortex-file-handler/src/blobHandler.js @@ -21,7 +21,7 @@ import { CONVERTED_EXTENSIONS, AZURITE_ACCOUNT_NAME, getDefaultContainerName, - GCS_BUCKETNAME, + getGCSBucketName, AZURE_STORAGE_CONTAINER_NAME } from "./constants.js"; import { FileConversionService } from "./services/FileConversionService.js"; @@ -1109,7 +1109,7 @@ async function cleanup(context, urls = null) { async function cleanupGCS(urls = null) { if (!gcs) return []; - const bucket = gcs.bucket(GCS_BUCKETNAME); + const bucket = gcs.bucket(getGCSBucketName()); const directories = new Set(); const cleanedURLs = []; @@ -1172,7 +1172,7 @@ async function deleteGCS(blobName) { ); // List files first - const listUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${GCS_BUCKETNAME}/o?prefix=${blobName}`; + const listUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${getGCSBucketName()}/o?prefix=${blobName}`; console.log(`[deleteGCS] Listing files with URL: ${listUrl}`); const listResponse = await axios.get(listUrl, { @@ -1190,7 +1190,7 @@ async function deleteGCS(blobName) { // Delete each file for (const item of listResponse.data.items) { - const deleteUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${GCS_BUCKETNAME}/o/${encodeURIComponent(item.name)}`; + const deleteUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${getGCSBucketName()}/o/${encodeURIComponent(item.name)}`; console.log(`[deleteGCS] Deleting file: ${item.name}`); console.log(`[deleteGCS] Delete URL: ${deleteUrl}`); @@ -1213,7 +1213,7 @@ async function deleteGCS(blobName) { } } else { console.log("[deleteGCS] Using real GCS"); - const bucket = gcs.bucket(GCS_BUCKETNAME); + const bucket = gcs.bucket(getGCSBucketName()); const [files] = await bucket.getFiles({ prefix: blobName }); console.log(`[deleteGCS] Found ${files.length} files to delete`); @@ -1286,9 +1286,9 @@ async function uploadChunkToGCS(chunkPath, requestId, filename = null) { gcsFileName = `${dirName}/${shortId}${fileExtension}`; } await gcs - .bucket(GCS_BUCKETNAME) + .bucket(getGCSBucketName()) .upload(chunkPath, { destination: gcsFileName }); - return `gs://${GCS_BUCKETNAME}/${gcsFileName}`; + return `gs://${getGCSBucketName()}/${gcsFileName}`; } export { @@ -1306,6 +1306,6 @@ export { getMimeTypeFromUrl, // Re-export container constants getDefaultContainerName, - GCS_BUCKETNAME, + getGCSBucketName, AZURE_STORAGE_CONTAINER_NAME, }; diff --git a/helper-apps/cortex-file-handler/src/constants.js b/helper-apps/cortex-file-handler/src/constants.js index e969db7e..ab179a53 100644 --- a/helper-apps/cortex-file-handler/src/constants.js +++ b/helper-apps/cortex-file-handler/src/constants.js @@ -172,15 +172,43 @@ export const getDefaultContainerName = () => { export const AZURE_STORAGE_CONTAINER_NAME = getContainerName(); // GCS bucket name must be explicitly set - no default to prevent accidental bucket usage -const getGCSBucketName = () => { - const bucketName = process.env.GCS_BUCKETNAME; - if (!bucketName || bucketName.trim() === "") { - throw new Error( - "GCS_BUCKETNAME environment variable is required but not set. " + - "Please set GCS_BUCKETNAME in your environment or .env file." - ); +// Using lazy evaluation - only throws error when accessed, not at module load time +// This allows tests to import the module without GCS_BUCKETNAME set +let _GCS_BUCKETNAME_CACHE = null; +const getGCSBucketNameValue = () => { + if (_GCS_BUCKETNAME_CACHE === null) { + const bucketName = process.env.GCS_BUCKETNAME; + if (!bucketName || bucketName.trim() === "") { + throw new Error( + "GCS_BUCKETNAME environment variable is required but not set. " + + "Please set GCS_BUCKETNAME in your environment or .env file." + ); + } + _GCS_BUCKETNAME_CACHE = bucketName.trim(); } - return bucketName.trim(); + return _GCS_BUCKETNAME_CACHE; }; -export const GCS_BUCKETNAME = getGCSBucketName(); +// Export function for explicit access +export const getGCSBucketName = getGCSBucketNameValue; + +// Export as a getter-like constant using a class with valueOf/toString +// This allows it to work as a string in most contexts (function calls, template literals, etc.) +class LazyGCSBucketName { + valueOf() { + return getGCSBucketNameValue(); + } + toString() { + return getGCSBucketNameValue(); + } + [Symbol.toPrimitive](hint) { + // Handle 'default', 'string', and 'number' hints + return getGCSBucketNameValue(); + } +} + +// Create instance that will convert to string when used +const lazyBucketName = new LazyGCSBucketName(); + +// Export as constant - will be converted to string when used in most contexts +export const GCS_BUCKETNAME = lazyBucketName; diff --git a/helper-apps/cortex-file-handler/src/services/ConversionService.js b/helper-apps/cortex-file-handler/src/services/ConversionService.js index f138b00d..1286b919 100644 --- a/helper-apps/cortex-file-handler/src/services/ConversionService.js +++ b/helper-apps/cortex-file-handler/src/services/ConversionService.js @@ -24,7 +24,7 @@ export class ConversionService { } /** - * Determines if a file needs conversion based on its extension + * Determines if a file needs conversion based on its extension and available services * @param {string} filename - The name of the file to check * @returns {boolean} - Whether the file needs conversion */ @@ -32,18 +32,37 @@ export class ConversionService { // Accept either a full filename/path or a raw extension (e.g. ".docx") const input = filename.toLowerCase(); - // If the input looks like an extension already, check directly + // Extract the extension + let ext; if ( input.startsWith(".") && !input.includes("/") && !input.includes("\\") ) { - return CONVERTED_EXTENSIONS.includes(input); + ext = input; + } else { + ext = path.extname(input).toLowerCase(); + } + + // Check if extension is in the list of convertible files + if (!CONVERTED_EXTENSIONS.includes(ext)) { + return false; + } + + // Excel files can always be converted (local conversion, no external service needed) + if (ext === ".xlsx" || ext === ".xls") { + return true; + } + + // Document files (.docx, .doc, .ppt, .pptx) need conversion services + // Check if either PDF service or MarkItDown service is available + if ([".docx", ".doc", ".ppt", ".pptx"].includes(ext)) { + const pdfServiceUrl = getDocToPdfUrl(); + const markitdownUrl = getMarkitdownUrl(); + return !!(pdfServiceUrl || markitdownUrl); } - // Otherwise, extract the extension from the filename/path - const ext = path.extname(input).toLowerCase(); - return CONVERTED_EXTENSIONS.includes(ext); + return false; } /** diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js index 8920fdb0..502d0aca 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +++ b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js @@ -1,7 +1,7 @@ import { AzureStorageProvider } from "./AzureStorageProvider.js"; import { GCSStorageProvider } from "./GCSStorageProvider.js"; import { LocalStorageProvider } from "./LocalStorageProvider.js"; -import { getContainerName, GCS_BUCKETNAME } from "../../constants.js"; +import { getContainerName, getGCSBucketName } from "../../constants.js"; import path from "path"; import { fileURLToPath } from "url"; @@ -65,7 +65,7 @@ export class StorageFactory { const provider = new GCSStorageProvider( config.credentials, config.projectId, - GCS_BUCKETNAME, + getGCSBucketName(), ); this.providers.set(key, provider); } diff --git a/helper-apps/cortex-file-handler/tests/FileConversionService.test.js b/helper-apps/cortex-file-handler/tests/FileConversionService.test.js index 1e893eb2..e911b753 100644 --- a/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +++ b/helper-apps/cortex-file-handler/tests/FileConversionService.test.js @@ -196,8 +196,44 @@ test("handles unsupported file types", async (t) => { // Test file extension detection test("correctly detects file extensions", (t) => { const service = new FileConversionService(mockContext); - t.true(service.needsConversion("test.docx")); + + // Excel files should always convert (local conversion, no external service needed) t.true(service.needsConversion("test.xlsx")); + t.true(service.needsConversion("test.xls")); + + // Document files should only convert if conversion services are available + // Without MARKITDOWN_CONVERT_URL or DOC_TO_PDF_SERVICE_URL, they should not convert + const originalMarkitdown = process.env.MARKITDOWN_CONVERT_URL; + const originalPdfService = process.env.DOC_TO_PDF_SERVICE_URL; + + // Clear conversion services + delete process.env.MARKITDOWN_CONVERT_URL; + delete process.env.DOC_TO_PDF_SERVICE_URL; + t.false(service.needsConversion("test.docx"), "docx should not convert without services"); + t.false(service.needsConversion("test.doc"), "doc should not convert without services"); + t.false(service.needsConversion("test.ppt"), "ppt should not convert without services"); + t.false(service.needsConversion("test.pptx"), "pptx should not convert without services"); + + // With MARKITDOWN_CONVERT_URL set, document files should convert + process.env.MARKITDOWN_CONVERT_URL = "http://test"; + t.true(service.needsConversion("test.docx"), "docx should convert with MARKITDOWN_CONVERT_URL"); + t.true(service.needsConversion("test.doc"), "doc should convert with MARKITDOWN_CONVERT_URL"); + t.true(service.needsConversion("test.ppt"), "ppt should convert with MARKITDOWN_CONVERT_URL"); + t.true(service.needsConversion("test.pptx"), "pptx should convert with MARKITDOWN_CONVERT_URL"); + + // Restore original env + if (originalMarkitdown) { + process.env.MARKITDOWN_CONVERT_URL = originalMarkitdown; + } else { + delete process.env.MARKITDOWN_CONVERT_URL; + } + if (originalPdfService) { + process.env.DOC_TO_PDF_SERVICE_URL = originalPdfService; + } else { + delete process.env.DOC_TO_PDF_SERVICE_URL; + } + + // Non-convertible files should never convert t.false(service.needsConversion("test.txt")); t.false(service.needsConversion("test.json")); }); diff --git a/helper-apps/cortex-file-handler/tests/conversionResilience.test.js b/helper-apps/cortex-file-handler/tests/conversionResilience.test.js index ee1c48dc..13e85eb9 100644 --- a/helper-apps/cortex-file-handler/tests/conversionResilience.test.js +++ b/helper-apps/cortex-file-handler/tests/conversionResilience.test.js @@ -6,7 +6,7 @@ import { v4 as uuidv4 } from "uuid"; import XLSX from "xlsx"; import { port } from "../src/start.js"; -import { gcs, GCS_BUCKETNAME } from "../src/blobHandler.js"; +import { gcs, getGCSBucketName } from "../src/blobHandler.js"; import { getFileStoreMap, setFileStoreMap } from "../src/redis.js"; import { cleanupHashAndFile, startTestServer, stopTestServer } from "./testUtils.helper.js"; import { gcsUrlExists } from "../src/blobHandler.js"; @@ -105,8 +105,8 @@ test.serial("checkHash recreates missing GCS converted file", async (t) => { // delete the GCS object const convertedGcsUrl = up.data.converted.gcs; - const bucket = gcs.bucket(GCS_BUCKETNAME); - const filename = convertedGcsUrl.replace(`gs://${GCS_BUCKETNAME}/`, ""); + const bucket = gcs.bucket(getGCSBucketName()); + const filename = convertedGcsUrl.replace(`gs://${getGCSBucketName()}/`, ""); try { await bucket.file(filename).delete({ ignoreNotFound: true }); } catch (_) {} diff --git a/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js b/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js index 77722e3b..b26f4f47 100644 --- a/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js +++ b/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js @@ -32,16 +32,16 @@ test("should create provider with valid credentials", (t) => { private_key: "test-key", }; - const provider = new GCSStorageProvider(credentials, "test-bucket"); + const provider = new GCSStorageProvider(credentials, "test-project", "test-bucket"); t.truthy(provider); }); test("should throw error with missing credentials", (t) => { t.throws( () => { - new GCSStorageProvider(null, "test-bucket"); + new GCSStorageProvider(null, null, "test-bucket"); }, - { message: "Missing GCS credentials or bucket name" }, + { message: "Missing GCS project ID" }, ); }); @@ -65,6 +65,7 @@ test("should upload and delete file", async (t) => { const provider = new GCSStorageProvider( credentials, + credentials.project_id, process.env.GCS_BUCKETNAME || "cortextempfiles", ); @@ -123,6 +124,7 @@ test("should handle file download", async (t) => { const provider = new GCSStorageProvider( credentials, + credentials.project_id, process.env.GCS_BUCKETNAME || "cortextempfiles", ); @@ -175,6 +177,7 @@ test("should handle file existence check with spaces and special characters", as const provider = new GCSStorageProvider( credentials, + credentials.project_id, process.env.GCS_BUCKETNAME || "cortextempfiles", ); diff --git a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js index 8228d5e6..8090d897 100644 --- a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +++ b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js @@ -69,8 +69,9 @@ test("should parse base64 gcs credentials", (t) => { process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 = base64Credentials; const factory = new StorageFactory(); - const credentials = factory.parseGCSCredentials(); - t.deepEqual(credentials, testCredentials); + const config = factory.parseGCSConfig(); + t.truthy(config); + t.deepEqual(config.credentials, testCredentials); // Cleanup delete process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64; @@ -86,8 +87,9 @@ test("should parse json gcs credentials", (t) => { process.env.GCP_SERVICE_ACCOUNT_KEY = JSON.stringify(testCredentials); const factory = new StorageFactory(); - const credentials = factory.parseGCSCredentials(); - t.deepEqual(credentials, testCredentials); + const config = factory.parseGCSConfig(); + t.truthy(config); + t.deepEqual(config.credentials, testCredentials); // Cleanup delete process.env.GCP_SERVICE_ACCOUNT_KEY; @@ -97,8 +99,8 @@ test("should return null for invalid gcs credentials", (t) => { process.env.GCP_SERVICE_ACCOUNT_KEY = "invalid-json"; const factory = new StorageFactory(); - const credentials = factory.parseGCSCredentials(); - t.is(credentials, null); + const config = factory.parseGCSConfig(); + t.is(config, null); // Cleanup delete process.env.GCP_SERVICE_ACCOUNT_KEY; From f0f54fe6edbb391d601611fcd51ecb783dd5e212 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 4 Jan 2026 19:20:55 -0700 Subject: [PATCH 05/11] chore: update Dockerfile to install production dependencies only and add dotenv to dependencies - Modified Dockerfile to use `npm ci --omit=dev` for installing only production dependencies. - Added `dotenv` to the list of dependencies in package.json, removing it from devDependencies. --- Dockerfile | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1e4b8b32..2d08098b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,8 @@ RUN apk add --no-cache python3 make g++ # Copy package files COPY package*.json ./ -# Install dependencies -RUN npm ci --only=production +# Install production dependencies only +RUN npm ci --omit=dev # Copy application code COPY . . diff --git a/package.json b/package.json index 726b2329..d5fdc312 100644 --- a/package.json +++ b/package.json @@ -68,12 +68,12 @@ "uuid": "^9.0.0", "winston": "^3.11.0", "ws": "^8.12.0", - "xxhash-wasm": "^1.1.0" + "xxhash-wasm": "^1.1.0", + "dotenv": "^16.0.3" }, "devDependencies": { "@faker-js/faker": "^8.4.1", "ava": "^5.2.0", - "dotenv": "^16.0.3", "got": "^13.0.0", "sinon": "^17.0.1" }, From d5f05a7f8a362571bca9098e4639497f7f2c0e5d Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 4 Jan 2026 19:57:17 -0700 Subject: [PATCH 06/11] Fix health check in cortex --- Dockerfile | 6 +++--- docker-compose.prod.yml | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2d08098b..4ff2d271 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,9 @@ RUN apk del python3 make g++ # Expose GraphQL port EXPOSE 4000 -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD wget --no-verbose --tries=1 --spider http://localhost:4000/.well-known/apollo/server-health || exit 1 +# Health check - test GraphQL endpoint +HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ + CMD wget -qO- --post-data='{"query":"{ __typename }"}' --header='Content-Type: application/json' http://localhost:4000/graphql | grep -q "__typename" || exit 1 CMD ["npm", "start"] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 10c8140e..a708fe0e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -3,13 +3,16 @@ version: "3.8" services: # Traefik - Reverse proxy with automatic SSL traefik: - image: traefik:v3.0 + image: traefik:v3.6.2 container_name: traefik restart: unless-stopped command: - "--api.dashboard=true" + - "--api.insecure=true" + - "--log.level=DEBUG" - "--providers.docker=true" - "--providers.docker.exposedbydefault=false" + - "--providers.docker.network=cortex_web" - "--entrypoints.web.address=:80" - "--entrypoints.websecure.address=:443" - "--entrypoints.web.http.redirections.entrypoint.to=websecure" From 73eca7d72cf38d5d12cda8e8dbd2552eefbc78a0 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 4 Jan 2026 20:13:38 -0700 Subject: [PATCH 07/11] Update health check in Dockerfile to use /healthcheck endpoint, bypassing API key authentication --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ff2d271..14fa4154 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,9 @@ RUN apk del python3 make g++ # Expose GraphQL port EXPOSE 4000 -# Health check - test GraphQL endpoint +# Health check - uses /healthcheck which bypasses API key auth HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ - CMD wget -qO- --post-data='{"query":"{ __typename }"}' --header='Content-Type: application/json' http://localhost:4000/graphql | grep -q "__typename" || exit 1 + CMD wget -qO- http://localhost:4000/healthcheck || exit 1 CMD ["npm", "start"] From b336a30181fef1df35bb2928c956fabda068fbd0 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Mon, 5 Jan 2026 11:00:39 -0700 Subject: [PATCH 08/11] Enhance Docker Compose configuration for production deployment - Added port mappings for the file-handler and cortex services to expose them on a private network for Concierge access. - Updated deployment script to ensure the repository name is lowercase for Docker compatibility, improving the reliability of image pulls and environment variable settings. --- .github/workflows/deploy.yml | 10 +++++++--- docker-compose.prod.yml | 18 +++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b54f8b42..fda45bde 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -119,16 +119,19 @@ jobs: script: | cd /opt/cortex + # Repository name must be lowercase for Docker + REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + # Login to GitHub Container Registry echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin # Pull the new images - docker pull ghcr.io/${{ github.repository }}:latest - docker pull ghcr.io/${{ github.repository }}-file-handler:latest + docker pull ghcr.io/${REPO_NAME}:latest + docker pull ghcr.io/${REPO_NAME}-file-handler:latest # Update the stack export IMAGE_TAG=latest - export GITHUB_REPOSITORY_CORTEX=${{ github.repository }} + export GITHUB_REPOSITORY_CORTEX=${REPO_NAME} # Load environment variables set -a @@ -177,3 +180,4 @@ jobs: exit 1 fi + diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a708fe0e..96cc028d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -71,6 +71,9 @@ services: - REDIS_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 - STORAGE_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 - WHISPER_MEDIA_API_URL=http://file-handler:7071 + ports: + # Expose on private network for Concierge to access + - "10.0.0.2:4000:4000" depends_on: redis: condition: service_healthy @@ -85,7 +88,7 @@ services: - "traefik.http.services.cortex.loadbalancer.server.port=4000" - "traefik.docker.network=cortex_web" - # Cortex File Handler - File processing service + # Cortex File Handler - File processing service (internal + private network) file-handler: image: ghcr.io/${GITHUB_REPOSITORY_CORTEX}-file-handler:${IMAGE_TAG:-latest} container_name: cortex-file-handler @@ -97,21 +100,14 @@ services: - NODE_ENV=production - PORT=7071 - REDIS_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 + ports: + # Expose on private network for Concierge to access + - "10.0.0.2:7071:7071" depends_on: redis: condition: service_healthy networks: - - web - internal - labels: - - "traefik.enable=true" - - "traefik.http.routers.filehandler.rule=Host(`${DOMAIN}`) && PathPrefix(`/file-handler`)" - - "traefik.http.routers.filehandler.entrypoints=websecure" - - "traefik.http.routers.filehandler.tls.certresolver=letsencrypt" - - "traefik.http.services.filehandler.loadbalancer.server.port=7071" - - "traefik.http.middlewares.filehandler-strip.stripprefix.prefixes=/file-handler" - - "traefik.http.routers.filehandler.middlewares=filehandler-strip" - - "traefik.docker.network=cortex_web" networks: web: From 44f9388381476b6e0ee46c4c7167bbe15bded15e Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Mon, 5 Jan 2026 12:41:46 -0700 Subject: [PATCH 09/11] Update WHISPER_MEDIA_API_URL in production Docker Compose configuration - Changed the WHISPER_MEDIA_API_URL to include the /api/CortexFileHandler endpoint for improved API routing. - Removed unnecessary version declaration for cleaner configuration. --- docker-compose.prod.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 96cc028d..8bdb89b4 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: # Traefik - Reverse proxy with automatic SSL traefik: @@ -70,7 +68,7 @@ services: # Internal container networking - REDIS_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 - STORAGE_CONNECTION_STRING=redis://default:${REDIS_PASSWORD}@redis:6379 - - WHISPER_MEDIA_API_URL=http://file-handler:7071 + - WHISPER_MEDIA_API_URL=http://file-handler:7071/api/CortexFileHandler ports: # Expose on private network for Concierge to access - "10.0.0.2:4000:4000" From 05b6a9238a13d4b5f197cc22ad39538a2cbdaa63 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Mon, 5 Jan 2026 19:37:30 -0700 Subject: [PATCH 10/11] Update models and enhance translation tool functionality - Changed model identifiers in code_review.js and release_notes.js to reflect new versions. - Improved translation prompt in translate.js for clarity and added new parameters for enhanced processing. - Updated tool descriptions in sys_tool_browser_jina.js and sys_tool_browser.js for better user understanding and functionality adjustments. --- pathways/code_review.js | 2 +- pathways/release_notes.js | 3 ++- pathways/system/entity/tools/sys_tool_browser.js | 1 + pathways/system/entity/tools/sys_tool_browser_jina.js | 2 +- pathways/translate.js | 8 +++++--- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pathways/code_review.js b/pathways/code_review.js index e91813ee..cd6e92e9 100644 --- a/pathways/code_review.js +++ b/pathways/code_review.js @@ -9,7 +9,7 @@ export default { ] }) ], - model: 'oai-gpt4o', + model: 'gemini-pro-3-vision', tokenRatio: 0.75, enableDuplicateRequests: false, } diff --git a/pathways/release_notes.js b/pathways/release_notes.js index 79c892ed..b07b4808 100644 --- a/pathways/release_notes.js +++ b/pathways/release_notes.js @@ -9,7 +9,8 @@ export default { ] }) ], - model: 'oai-gpt4o', + model: 'gemini-flash-3-vision', + reasoningEffort: 'high', tokenRatio: 0.75, enableDuplicateRequests: false, } diff --git a/pathways/system/entity/tools/sys_tool_browser.js b/pathways/system/entity/tools/sys_tool_browser.js index 2465bf44..b6f669ee 100644 --- a/pathways/system/entity/tools/sys_tool_browser.js +++ b/pathways/system/entity/tools/sys_tool_browser.js @@ -9,6 +9,7 @@ export default { timeout: 300, toolDefinition: { type: "function", + enabled: false, icon: "🌍", function: { name: "FetchWebPageContent", diff --git a/pathways/system/entity/tools/sys_tool_browser_jina.js b/pathways/system/entity/tools/sys_tool_browser_jina.js index f7a8501e..410d2a22 100644 --- a/pathways/system/entity/tools/sys_tool_browser_jina.js +++ b/pathways/system/entity/tools/sys_tool_browser_jina.js @@ -12,7 +12,7 @@ export default { icon: "🌎", function: { name: "FetchWebPageContentJina", - description: "This tool allows you to fetch and extract the text content from any webpage using the Jina API. This is a great backup tool for web page content if you don't get a good enough response from your other browser tool or are blocked by a website.", + description: "This tool allows you to fetch and extract the text content from any webpage using the Jina reader API. Use this when you need to analyze or understand the content of a specific webpage.", parameters: { type: "object", properties: { diff --git a/pathways/translate.js b/pathways/translate.js index 9ebdd862..88b856f1 100644 --- a/pathways/translate.js +++ b/pathways/translate.js @@ -4,16 +4,18 @@ export default { prompt: [ new Prompt({ messages: [ - {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."}, + {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text to translate in any language, assistant will create a translation of that text in {{to}} (language string or ISO code). All text that the user posts is to be translated - assistant must not respond to the user in any way and should produce only the translation with no additional notes or commentary."}, {"role": "user", "content": "{{{text}}}"} ]}), ], inputParameters: { to: `Arabic`, tokenRatio: 0.2, + model: 'oai-gpt5-chat', }, - inputChunkSize: 500, - model: 'oai-gpt4o', + inputChunkSize: 1000, enableDuplicateRequests: false, + useParallelChunkProcessing: true, + enableCache: true, } \ No newline at end of file From 6ac204ff662fecb0aba5e7a4cb7738b18c3ebbc1 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Tue, 6 Jan 2026 13:47:00 -0700 Subject: [PATCH 11/11] Enhance Gemini15VisionPlugin to support additional URL sources - Updated the Gemini15VisionPlugin to check for a URL in multiple properties: gcs, image_url.url, and a new direct url property. - Improved error handling by ensuring the input is parsed correctly as JSON before processing. --- server/plugins/gemini15VisionPlugin.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/plugins/gemini15VisionPlugin.js b/server/plugins/gemini15VisionPlugin.js index e7edb29c..19272584 100644 --- a/server/plugins/gemini15VisionPlugin.js +++ b/server/plugins/gemini15VisionPlugin.js @@ -44,8 +44,9 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin { try { // First try to parse as JSON if it's a string const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart; - const {type, text, image_url, gcs} = part; - let fileUrl = gcs || image_url?.url; + const {type, text, image_url, gcs, url} = part; + // Check for URL in multiple places: gcs, image_url.url, or direct url property + let fileUrl = gcs || image_url?.url || url; if (typeof part === 'string') { return { text: inputPart };