perf(i18n): parallelism + OpenAPI fix

.github/workflows/translate-and-deploy.yml

@@ -0,0 +1,363 @@
+name: Translate and Deploy Docs
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - '**.mdx'
+      - 'docs.json'
+      - 'openapi/**'
+      - 'images/**'
+  workflow_dispatch:
+    inputs:
+      force_full_translation:
+        description: 'Force full re-translation of all files'
+        required: false
+        default: 'false'
+        type: boolean
+
+env:
+  DEPLOY_BRANCH: prod
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+concurrency:
+  group: translate-deploy
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  # ============================================
+  # JOB 1: Setup and detect what needs translation
+  # ============================================
+  setup:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      changed_files: ${{ steps.changed.outputs.files }}
+      force_full: ${{ steps.changed.outputs.force_full }}
+
+    steps:
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          fetch-depth: 2
+
+      - name: Get changed files
+        id: changed
+        run: |
+          if [ "${{ github.event.inputs.force_full_translation }}" = "true" ]; then
+            echo "files=" >> $GITHUB_OUTPUT
+            echo "force_full=true" >> $GITHUB_OUTPUT
+          else
+            if git rev-parse HEAD~1 >/dev/null 2>&1; then
+              files=$(git diff --name-only HEAD~1 HEAD -- '*.mdx' | grep -v '^es/' | grep -v '^nl/' | grep -v '^fr/' | grep -v '^de/' | grep -v '^zh/' | grep -v '^it/' | grep -v '^ja/' | tr '\n' ',' | sed 's/,$//')
+              echo "files=$files" >> $GITHUB_OUTPUT
+            else
+              echo "files=" >> $GITHUB_OUTPUT
+            fi
+            echo "force_full=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Fetch existing translations from prod
+        run: |
+          git fetch origin prod:prod 2>/dev/null || true
+          if git rev-parse prod >/dev/null 2>&1; then
+            echo "Copying existing translations from prod branch..."
+            for lang in es nl fr de zh it ja; do
+              git checkout prod -- "$lang/" 2>/dev/null || echo "No existing $lang translations"
+            done
+          fi
+
+      - name: Upload workspace
+        uses: actions/upload-artifact@v4
+        with:
+          name: workspace-setup
+          path: |
+            **/*.mdx
+            **/*.json
+            scripts/
+            tsconfig.json
+            !node_modules/
+          retention-days: 1
+          if-no-files-found: warn
+
+  # ============================================
+  # JOB 2: Translate OpenAPI specs
+  # ============================================
+  translate-openapi:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: setup
+
+    steps:
+      - name: Download workspace
+        uses: actions/download-artifact@v4
+        with:
+          name: workspace-setup
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Translate OpenAPI specs
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          echo "::group::OpenAPI Translation"
+          npx ts-node scripts/translate-openapi.ts
+          echo "::endgroup::"
+
+      - name: Upload translated OpenAPI
+        uses: actions/upload-artifact@v4
+        with:
+          name: openapi-translations
+          path: |
+            es/openapi/
+            nl/openapi/
+            fr/openapi/
+            de/openapi/
+            zh/openapi/
+            it/openapi/
+            ja/openapi/
+          retention-days: 1
+          if-no-files-found: ignore
+
+  # ============================================
+  # JOB 3: Phase 1 - Catch up missing MDX translations
+  # ============================================
+  translate-catchup:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    needs: setup
+
+    steps:
+      - name: Download workspace
+        uses: actions/download-artifact@v4
+        with:
+          name: workspace-setup
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Validate OpenAI API key
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          set -e
+          if [ -z "$OPENAI_API_KEY" ]; then
+            echo "::error::OPENAI_API_KEY secret is not set"
+            exit 1
+          fi
+          echo "✓ OpenAI API key configured"
+
+      - name: Phase 1 - Translate missing pages
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          echo "::group::Translation Progress"
+          npx ts-node scripts/translate-docs.ts --phase=catchup
+          echo "::endgroup::"
+
+      - name: Upload translations
+        uses: actions/upload-artifact@v4
+        with:
+          name: translations-catchup
+          path: |
+            es/
+            nl/
+            fr/
+            de/
+            zh/
+            it/
+            ja/
+          retention-days: 1
+          if-no-files-found: ignore
+
+  # ============================================
+  # JOB 4: Phase 2 - Translate changed files
+  # ============================================
+  translate-incremental:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: [setup, translate-catchup]
+    if: needs.setup.outputs.changed_files != '' && needs.setup.outputs.force_full != 'true'
+
+    steps:
+      - name: Download workspace
+        uses: actions/download-artifact@v4
+        with:
+          name: workspace-setup
+
+      - name: Download catchup translations
+        uses: actions/download-artifact@v4
+        with:
+          name: translations-catchup
+          path: .
+        continue-on-error: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Phase 2 - Translate changed files
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          echo "Changed files: ${{ needs.setup.outputs.changed_files }}"
+          echo "::group::Translation Progress"
+          npx ts-node scripts/translate-docs.ts --phase=incremental --files="${{ needs.setup.outputs.changed_files }}"
+          echo "::endgroup::"
+
+      - name: Upload translations
+        uses: actions/upload-artifact@v4
+        with:
+          name: translations-incremental
+          path: |
+            es/
+            nl/
+            fr/
+            de/
+            zh/
+            it/
+            ja/
+          retention-days: 1
+          if-no-files-found: ignore
+
+  # ============================================
+  # JOB 5: Deploy to prod branch
+  # ============================================
+  deploy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [setup, translate-openapi, translate-catchup, translate-incremental]
+    if: always() && needs.translate-catchup.result == 'success' && needs.translate-openapi.result == 'success'
+
+    steps:
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          fetch-depth: 1
+
+      - name: Download OpenAPI translations
+        uses: actions/download-artifact@v4
+        with:
+          name: openapi-translations
+          path: .
+        continue-on-error: true
+
+      - name: Download catchup translations
+        uses: actions/download-artifact@v4
+        with:
+          name: translations-catchup
+          path: .
+        continue-on-error: true
+
+      - name: Download incremental translations
+        if: needs.translate-incremental.result == 'success'
+        uses: actions/download-artifact@v4
+        with:
+          name: translations-incremental
+          path: .
+        continue-on-error: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Update docs.json with languages
+        run: npx ts-node scripts/update-docs-config.ts
+
+      - name: Verify translations
+        run: |
+          echo "::group::Translation Counts"
+          echo "=== MDX Files ==="
+          total_mdx=0
+          for lang in es nl fr de zh it ja; do
+            count=$(find "$lang" -name "*.mdx" 2>/dev/null | wc -l || echo "0")
+            echo "$lang: $count MDX files"
+            total_mdx=$((total_mdx + count))
+          done
+          echo "Total MDX: $total_mdx files"
+          echo ""
+          echo "=== OpenAPI Files ==="
+          total_openapi=0
+          for lang in es nl fr de zh it ja; do
+            count=$(find "$lang/openapi" -name "*.json" 2>/dev/null | wc -l || echo "0")
+            echo "$lang: $count OpenAPI files"
+            total_openapi=$((total_openapi + count))
+          done
+          echo "Total OpenAPI: $total_openapi files"
+          echo "::endgroup::"
+
+          if [ "$total_mdx" -eq 0 ]; then
+            echo "::warning::No MDX translations found"
+          fi
+
+      - name: Deploy to prod branch
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          git add -A
+
+          if git diff --staged --quiet; then
+            echo "No changes to deploy"
+            exit 0
+          fi
+
+          git commit -m "chore: auto-translated docs from ${{ github.sha }}"
+          git push -f origin HEAD:${{ env.DEPLOY_BRANCH }}
+
+          echo "✓ Deployed to ${{ env.DEPLOY_BRANCH }} branch"
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "## 🌍 Translation Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
+          echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Setup | ${{ needs.setup.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| OpenAPI Translation | ${{ needs.translate-openapi.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| MDX Catch-up (Phase 1) | ${{ needs.translate-catchup.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| MDX Incremental (Phase 2) | ${{ needs.translate-incremental.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Languages:** Spanish, Dutch, French, German, Chinese, Italian, Japanese" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Translation Counts" >> $GITHUB_STEP_SUMMARY
+          echo "| Language | MDX | OpenAPI |" >> $GITHUB_STEP_SUMMARY
+          echo "|----------|-----|---------|" >> $GITHUB_STEP_SUMMARY
+          for lang in es nl fr de zh it ja; do
+            mdx_count=$(find "$lang" -name "*.mdx" 2>/dev/null | wc -l || echo "0")
+            openapi_count=$(find "$lang/openapi" -name "*.json" 2>/dev/null | wc -l || echo "0")
+            echo "| $lang | $mdx_count | $openapi_count |" >> $GITHUB_STEP_SUMMARY
+          done

.gitignore

@@ -5,3 +5,4 @@ node_modules/
 .next/
 out/
 build/
+dist/

openapi/crawls.json

@@ -107,7 +107,7 @@
                         "type": "string",
                         "description": "Parser name to run on each page and produce structured `json` output (e.g. `\"@olostep/extract-emails\"`). Automatically adds `json` to `formats` when set.",
                         "example": "@olostep/extract-emails"
-                      },
+                      }
                     }
                   }
                 },