feat: Comprehensive security upgrades and ML stack modernization

.github/workflows/pre-commit.yml

@@ -0,0 +1,65 @@
+name: Pre-commit Checks
+
+on:
+  push:
+    branches: [master, main, develop]
+  pull_request:
+    branches: [master, main, develop]
+
+jobs:
+  pre-commit:
+    name: Run Pre-commit Hooks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pre-commit
+
+      - name: Cache pre-commit environments
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: |
+            pre-commit-
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files --show-diff-on-failure
+
+      - name: Generate pre-commit summary
+        if: always()
+        run: |
+          echo "## Pre-commit Check Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Pre-commit hooks executed on all files." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Hooks Executed:" >> $GITHUB_STEP_SUMMARY
+          echo "- Code formatting (Ruff, Prettier)" >> $GITHUB_STEP_SUMMARY
+          echo "- Linting (Ruff, mypy, ESLint)" >> $GITHUB_STEP_SUMMARY
+          echo "- Security checks (Gitleaks, Bandit)" >> $GITHUB_STEP_SUMMARY
+          echo "- Shell script validation (shellcheck)" >> $GITHUB_STEP_SUMMARY
+          echo "- Dockerfile linting (Hadolint)" >> $GITHUB_STEP_SUMMARY

.github/workflows/security-scan.yml

@@ -0,0 +1,248 @@
+name: Security Scanning
+
+on:
+  push:
+    branches: [master, main, develop]
+    paths:
+      - 'backend/**'
+      - 'frontend/**'
+      - '**/Dockerfile*'
+      - '.github/workflows/security-scan.yml'
+  pull_request:
+    branches: [master, main, develop]
+    paths:
+      - 'backend/**'
+      - 'frontend/**'
+      - '**/Dockerfile*'
+      - '.github/workflows/security-scan.yml'
+  schedule:
+    # Run weekly security scans on Sundays at 00:00 UTC
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+    inputs:
+      component:
+        description: 'Component to scan (backend, frontend, or all)'
+        required: false
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - backend
+          - frontend
+
+jobs:
+  dockerfile-lint:
+    name: Lint Dockerfiles
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Hadolint on backend Dockerfile
+        uses: hadolint/hadolint-action@v3.1.0
+        with:
+          dockerfile: backend/Dockerfile.prod
+          config: .hadolint.yaml
+          failure-threshold: warning
+
+      - name: Run Hadolint on frontend Dockerfile
+        uses: hadolint/hadolint-action@v3.1.0
+        with:
+          dockerfile: frontend/Dockerfile.prod
+          config: .hadolint.yaml
+          failure-threshold: warning
+
+  build-and-scan-backend:
+    name: Build & Scan Backend
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build backend image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.prod
+          tags: opentranscribe-backend:${{ github.sha }}
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: opentranscribe-backend:${{ github.sha }}
+          format: 'sarif'
+          output: 'trivy-backend-results.sarif'
+          severity: 'CRITICAL,HIGH,MEDIUM'
+
+      - name: Upload Trivy results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-backend-results.sarif'
+          category: 'trivy-backend'
+
+      - name: Run Trivy vulnerability scanner (table output)
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: opentranscribe-backend:${{ github.sha }}
+          format: 'table'
+          severity: 'CRITICAL,HIGH,MEDIUM'
+
+      - name: Run Dockle
+        uses: erzz/dockle-action@v1
+        with:
+          image: opentranscribe-backend:${{ github.sha }}
+          exit-code: '0'
+          failure-threshold: warn
+
+      - name: Generate SBOM with Syft
+        uses: anchore/sbom-action@v0
+        with:
+          image: opentranscribe-backend:${{ github.sha }}
+          artifact-name: backend-sbom.spdx.json
+          output-file: backend-sbom.spdx.json
+
+      - name: Scan SBOM with Grype
+        uses: anchore/scan-action@v3
+        with:
+          sbom: backend-sbom.spdx.json
+          fail-build: false
+          severity-cutoff: medium
+
+      - name: Upload SBOM artifact
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: backend-sbom
+          path: backend-sbom.spdx.json
+          retention-days: 90
+
+  build-and-scan-frontend:
+    name: Build & Scan Frontend
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build frontend image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./frontend
+          file: ./frontend/Dockerfile.prod
+          tags: opentranscribe-frontend:${{ github.sha }}
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: opentranscribe-frontend:${{ github.sha }}
+          format: 'sarif'
+          output: 'trivy-frontend-results.sarif'
+          severity: 'CRITICAL,HIGH,MEDIUM'
+
+      - name: Upload Trivy results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-frontend-results.sarif'
+          category: 'trivy-frontend'
+
+      - name: Run Trivy vulnerability scanner (table output)
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: opentranscribe-frontend:${{ github.sha }}
+          format: 'table'
+          severity: 'CRITICAL,HIGH,MEDIUM'
+
+      - name: Run Dockle
+        uses: erzz/dockle-action@v1
+        with:
+          image: opentranscribe-frontend:${{ github.sha }}
+          exit-code: '0'
+          failure-threshold: warn
+
+      - name: Generate SBOM with Syft
+        uses: anchore/sbom-action@v0
+        with:
+          image: opentranscribe-frontend:${{ github.sha }}
+          artifact-name: frontend-sbom.spdx.json
+          output-file: frontend-sbom.spdx.json
+
+      - name: Scan SBOM with Grype
+        uses: anchore/scan-action@v3
+        with:
+          sbom: frontend-sbom.spdx.json
+          fail-build: false
+          severity-cutoff: medium
+
+      - name: Upload SBOM artifact
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: frontend-sbom
+          path: frontend-sbom.spdx.json
+          retention-days: 90
+
+  dependency-scan:
+    name: Scan Dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner on repository
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+          skip-dirs: 'node_modules,venv,.venv'
+
+      - name: Run Trivy config scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'config'
+          scan-ref: '.'
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+          skip-dirs: 'node_modules,venv,.venv'
+
+  summary:
+    name: Security Scan Summary
+    runs-on: ubuntu-latest
+    needs: [dockerfile-lint, build-and-scan-backend, build-and-scan-frontend, dependency-scan]
+    if: always()
+    steps:
+      - name: Check scan results
+        run: |
+          echo "## Security Scan Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "All security scans completed. Check individual job logs for details." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Completed Scans:" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Dockerfile Linting (Hadolint)" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Backend Image Scanning (Trivy, Dockle, Grype)" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Frontend Image Scanning (Trivy, Dockle, Grype)" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Dependency Scanning (Trivy)" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Reports Available:" >> $GITHUB_STEP_SUMMARY
+          echo "- SBOM artifacts uploaded for both backend and frontend" >> $GITHUB_STEP_SUMMARY
+          echo "- Vulnerability results uploaded to GitHub Security tab" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -1,6 +1,8 @@
 # OpenTranscribe .gitignore
 
 .github/DRAFT_ISSUES/
+offline-package-build/
+security-reports-*
 
 # Environment variables
 .env
@@ -238,3 +240,4 @@ frontend/static/ffmpeg/
 
 # Private project management files
 GITHUB_OPTIMIZATION_TASKS.md
+security-reports/

.hadolint.yaml

@@ -0,0 +1,33 @@
+# Hadolint configuration for OpenTranscribe Dockerfiles
+# https://github.com/hadolint/hadolint
+
+# Ignore specific rules
+ignored:
+  - DL3008  # Pin versions in apt-get install (Debian/Ubuntu)
+  - DL3009  # Delete the apt-get lists after installing (already handled)
+  - DL3015  # Additional packages by --no-install-recommends (sometimes we need them)
+  - DL3018  # Pin versions in apk add (Alpine - can break builds)
+  - DL3059  # Multiple consecutive RUN instructions (acceptable for layer caching)
+
+# Trust specific registries
+trustedRegistries:
+  - docker.io
+  - ghcr.io
+  - nvcr.io
+
+# Override default rules severity
+override:
+  error:
+    - DL3002  # Do not switch to root USER
+    - DL3020  # Use COPY instead of ADD for files and folders
+    - DL4006  # Set SHELL option -o pipefail before RUN with pipe
+  warning:
+    - DL3003  # Use WORKDIR to switch to a directory
+    - DL3007  # Using latest tag for image
+    - DL3042  # Avoid cache directory with pip
+  info:
+    - DL3013  # Pin versions in pip install
+    - DL3016  # Pin versions in npm install
+
+# Strict mode - warn on all other issues
+failure-threshold: warning