Feauture/kernel implementation

.env.example

@@ -26,3 +26,11 @@ FLOWFILE_ADMIN_PASSWORD=changeme
 # Secret key for signing JWT tokens (min 32 characters)
 # Generate with: openssl rand -hex 32
 JWT_SECRET_KEY=your-secure-jwt-secret-key-change-in-production
+
+# ============================================
+# Internal Service Token
+# ============================================
+# Shared secret for kernel → Core API authentication.
+# Core passes this token to kernel containers automatically.
+# Generate with: openssl rand -hex 32
+FLOWFILE_INTERNAL_TOKEN=your-secure-internal-token-change-in-production

.github/workflows/test-docker-kernel-e2e.yml

@@ -0,0 +1,94 @@
+name: Docker Kernel E2E Tests
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'kernel_runtime/**'
+      - 'flowfile_core/flowfile_core/kernel/**'
+      - 'flowfile_core/Dockerfile'
+      - 'flowfile_worker/Dockerfile'
+      - 'docker-compose.yml'
+      - 'tests/integration/**'
+      - '.github/workflows/test-docker-kernel-e2e.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'kernel_runtime/**'
+      - 'flowfile_core/flowfile_core/kernel/**'
+      - 'flowfile_core/Dockerfile'
+      - 'flowfile_worker/Dockerfile'
+      - 'docker-compose.yml'
+      - 'tests/integration/**'
+      - '.github/workflows/test-docker-kernel-e2e.yml'
+  workflow_dispatch:
+
+jobs:
+  docker-kernel-e2e:
+    name: Docker Kernel E2E
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    env:
+      COMPOSE_PROJECT_NAME: flowfile-ci-${{ github.run_id }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            poetry-${{ runner.os }}-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+          poetry install --no-interaction --no-ansi
+
+      - name: Run Docker Kernel E2E tests
+        run: |
+          poetry run pytest tests/integration -m docker_integration -vv --tb=long --log-cli-level=INFO
+        timeout-minutes: 15
+
+      - name: Clean up Docker resources
+        if: always()
+        run: |
+          docker compose -p flowfile-ci-${{ github.run_id }} down -v --remove-orphans || true
+          docker ps -a --filter "name=flowfile-kernel" -q | xargs -r docker rm -f || true
+          docker system prune -f
+
+      - name: Show Docker logs on failure
+        if: failure()
+        run: |
+          echo "=== Docker containers ==="
+          docker ps -a
+          echo ""
+          echo "=== Compose service logs ==="
+          docker compose -p flowfile-ci-${{ github.run_id }} logs --tail=200 || true
+          echo ""
+          echo "=== Kernel container logs ==="
+          docker ps -a --filter "name=flowfile-kernel" --format "{{.Names}}" | while read name; do
+            echo "--- Logs for $name ---"
+            docker logs "$name" 2>&1 || true
+          done

.github/workflows/test-kernel-integration.yml

@@ -0,0 +1,91 @@
+name: Kernel Integration Tests
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'kernel_runtime/**'
+      - 'flowfile_core/flowfile_core/kernel/**'
+      - 'flowfile_core/flowfile_core/artifacts/**'
+      - 'flowfile_core/tests/flowfile/test_kernel*.py'
+      - 'flowfile_core/tests/flowfile/test_global_artifacts*.py'
+      - 'flowfile_core/tests/kernel_fixtures.py'
+      - '.github/workflows/test-kernel-integration.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'kernel_runtime/**'
+      - 'flowfile_core/flowfile_core/kernel/**'
+      - 'flowfile_core/flowfile_core/artifacts/**'
+      - 'flowfile_core/tests/flowfile/test_kernel*.py'
+      - 'flowfile_core/tests/flowfile/test_global_artifacts*.py'
+      - 'flowfile_core/tests/kernel_fixtures.py'
+      - '.github/workflows/test-kernel-integration.yml'
+  workflow_dispatch:
+
+jobs:
+  kernel-tests:
+    name: Kernel Integration Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: |
+          poetry install --no-interaction --no-ansi
+
+      - name: Build kernel Docker image
+        run: |
+          docker build -t flowfile-kernel -f kernel_runtime/Dockerfile kernel_runtime/
+          docker images | grep flowfile-kernel
+
+      - name: Run kernel integration tests
+        run: |
+          poetry run pytest flowfile_core/tests -m kernel -vv --tb=long --log-cli-level=INFO
+        timeout-minutes: 20
+        env:
+          # Set test mode to avoid conflicts
+          TEST_MODE: "1"
+          # Generate internal token for kernel <-> Core auth
+          FLOWFILE_INTERNAL_TOKEN: ${{ github.run_id }}-test-token
+
+      - name: Clean up Docker resources
+        if: always()
+        run: |
+          # Remove kernel containers
+          docker ps -a --filter "name=flowfile-kernel" -q | xargs -r docker rm -f
+          # Remove kernel image
+          docker images --filter "reference=flowfile-kernel" -q | xargs -r docker rmi -f || true
+          # Prune
+          docker system prune -f
+
+      - name: Show Docker logs on failure
+        if: failure()
+        run: |
+          echo "=== Docker containers ==="
+          docker ps -a
+          echo ""
+          echo "=== Kernel container logs ==="
+          docker ps -a --filter "name=flowfile-kernel" --format "{{.Names}}" | while read name; do
+            echo "--- Logs for $name ---"
+            docker logs "$name" 2>&1 || true
+          done

.github/workflows/test.yaml

@@ -25,6 +25,7 @@ jobs:
       backend_worker: ${{ steps.filter.outputs.backend_worker }}
       backend_frame: ${{ steps.filter.outputs.backend_frame }}
       backend_flowfile: ${{ steps.filter.outputs.backend_flowfile }}
+      kernel: ${{ steps.filter.outputs.kernel }}
       frontend: ${{ steps.filter.outputs.frontend }}
       docs: ${{ steps.filter.outputs.docs }}
       shared: ${{ steps.filter.outputs.shared }}
@@ -46,6 +47,11 @@ jobs:
               - 'flowfile_frame/**'
             backend_flowfile:
               - 'flowfile/**'
+            kernel:
+              - 'kernel_runtime/**'
+              - 'flowfile_core/flowfile_core/kernel/**'
+              - 'flowfile_core/tests/flowfile/test_kernel_integration.py'
+              - 'flowfile_core/tests/kernel_fixtures.py'
             frontend:
               - 'flowfile_frontend/**'
             docs:
@@ -145,7 +151,7 @@ jobs:
           needs.detect-changes.outputs.shared == 'true' ||
           needs.detect-changes.outputs.test_workflow == 'true' ||
           github.event.inputs.run_all_tests == 'true'
-        run: poetry run pytest flowfile_core/tests --disable-warnings $COV_ARGS
+        run: poetry run pytest flowfile_core/tests -m "not kernel" --disable-warnings $COV_ARGS
         env:
           COV_ARGS: ${{ (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12') && '--cov --cov-append --cov-report=' || '' }}
 
@@ -271,7 +277,7 @@ jobs:
           needs.detect-changes.outputs.test_workflow == 'true' ||
           github.event.inputs.run_all_tests == 'true'
         shell: pwsh
-        run: poetry run pytest flowfile_core/tests --disable-warnings
+        run: poetry run pytest flowfile_core/tests -m "not kernel" --disable-warnings
 
       - name: Run pytest for flowfile_worker
         if: |
@@ -299,6 +305,48 @@ jobs:
         shell: pwsh
         run: poetry run pytest flowfile/tests --disable-warnings
 
+  # Kernel integration tests - runs in parallel on a separate worker
+  kernel-tests:
+    needs: detect-changes
+    if: |
+      needs.detect-changes.outputs.kernel == 'true' ||
+      needs.detect-changes.outputs.backend_core == 'true' ||
+      needs.detect-changes.outputs.shared == 'true' ||
+      needs.detect-changes.outputs.test_workflow == 'true' ||
+      github.event.inputs.run_all_tests == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: 'pip'
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+          echo "$HOME/.poetry/bin" >> $GITHUB_PATH
+
+      - name: Install Dependencies
+        run: |
+          poetry install --no-interaction --no-ansi --with dev
+
+      - name: Build kernel Docker image
+        run: |
+          docker build -t flowfile-kernel -f kernel_runtime/Dockerfile kernel_runtime/
+
+      - name: Run kernel_runtime unit tests
+        run: |
+          pip install -e "kernel_runtime/[test]"
+          python -m pytest kernel_runtime/tests -v --disable-warnings
+
+      - name: Run kernel integration tests
+        run: |
+          poetry run pytest flowfile_core/tests -m kernel -v --disable-warnings
+
   # Frontend web build test - runs when frontend changes or test workflow changes
   test-web:
     needs: detect-changes
@@ -472,7 +520,7 @@ jobs:
 
   # Summary job - always runs to provide status
   test-summary:
-    needs: [detect-changes, backend-tests, backend-tests-windows, test-web, electron-tests-macos, electron-tests-windows, docs-test]
+    needs: [detect-changes, backend-tests, backend-tests-windows, kernel-tests, test-web, electron-tests-macos, electron-tests-windows, docs-test]
     if: always()
     runs-on: ubuntu-latest
     steps:
@@ -485,6 +533,7 @@ jobs:
           echo "  - Backend Worker: ${{ needs.detect-changes.outputs.backend_worker }}"
           echo "  - Backend Frame: ${{ needs.detect-changes.outputs.backend_frame }}"
           echo "  - Backend Flowfile: ${{ needs.detect-changes.outputs.backend_flowfile }}"
+          echo "  - Kernel: ${{ needs.detect-changes.outputs.kernel }}"
           echo "  - Frontend: ${{ needs.detect-changes.outputs.frontend }}"
           echo "  - Docs: ${{ needs.detect-changes.outputs.docs }}"
           echo "  - Shared/Dependencies: ${{ needs.detect-changes.outputs.shared }}"
@@ -493,6 +542,7 @@ jobs:
           echo "Job results:"
           echo "  - Backend Tests: ${{ needs.backend-tests.result }}"
           echo "  - Backend Tests (Windows): ${{ needs.backend-tests-windows.result }}"
+          echo "  - Kernel Tests: ${{ needs.kernel-tests.result }}"
           echo "  - Web Tests: ${{ needs.test-web.result }}"
           echo "  - Electron Tests (macOS): ${{ needs.electron-tests-macos.result }}"
           echo "  - Electron Tests (Windows): ${{ needs.electron-tests-windows.result }}"
@@ -501,6 +551,7 @@ jobs:
           # Fail if any non-skipped job failed
           if [[ "${{ needs.backend-tests.result }}" == "failure" ]] || \
              [[ "${{ needs.backend-tests-windows.result }}" == "failure" ]] || \
+             [[ "${{ needs.kernel-tests.result }}" == "failure" ]] || \
              [[ "${{ needs.test-web.result }}" == "failure" ]] || \
              [[ "${{ needs.electron-tests-macos.result }}" == "failure" ]] || \
              [[ "${{ needs.electron-tests-windows.result }}" == "failure" ]] || \

.gitignore

@@ -67,6 +67,9 @@ htmlcov/
 # Docker
 flowfile_data/
 
+# Egg info
+*.egg-info/
+
 # Secrets and keys - NEVER commit these
 master_key.txt
 *.key