diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 4c67d82a..c52b3c86 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -63,6 +63,11 @@ jobs: with: node-version: '20' + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Setup caching for build artifacts uses: actions/cache@v4 id: cache-npm @@ -174,6 +179,13 @@ jobs: echo "✅ Configuration patches applied" + - name: Patch docs source + run: | + cd "${{ env.TEMP_DIR }}/llama-stack/docs" + echo "🔧 Patching docs source (sidebar validation, blog warnings, MDX compat)..." + "${{ github.workspace }}/patch-docs-source.sh" --repo-dir "${{ github.workspace }}" + echo "✅ Docs source patched" + - name: Import existing versioning artifacts run: | echo "đŸ“Ĩ Importing existing versioning artifacts from repository..." @@ -362,34 +374,20 @@ jobs: echo "đŸ—ī¸ Building latest version (no version snapshot needed)" fi - - name: Fix versioned docs relative imports + - name: Inline raw-loader imports in versioned docs run: | cd "${{ env.TEMP_DIR }}/llama-stack/docs" - echo "🔧 Fixing relative imports in versioned docs..." + echo "🔧 Inlining raw-loader imports in versioned docs..." if [ -d "versioned_docs" ]; then - # Fix relative paths (../../../file.md -> absolute path) - find versioned_docs -name "*.mdx" -type f \ - -exec grep -l "!!raw-loader!\.\." {} \; | \ - while read file; do - echo "Fixing relative raw-loader imports in $file" - # Pattern: !!raw-loader! followed by any number of ../ then a filename - # Replace with absolute path to repo root + filename - perl -i -pe "s|!!raw-loader!(\.\.\/)+([^'\"]*)|!!raw-loader!${{ env.TEMP_DIR }}/llama-stack/\$2|g" "$file" - done - - # Fix hardcoded old temp paths (/tmp/tmp.XXXXXX/llama-stack/file.md -> current temp path) - find versioned_docs -name "*.mdx" -type f \ - -exec grep -l "!!raw-loader!/tmp/tmp\." {} \; | \ - while read file; do - echo "Fixing hardcoded temp path imports in $file" - # Pattern: !!raw-loader!/tmp/tmp.XXXXXX/llama-stack/filename - # Replace with current temp directory path - perl -i -pe "s|!!raw-loader!/tmp/tmp\.[^/]+/llama-stack/([^'\"]*)|!!raw-loader!${{ env.TEMP_DIR }}/llama-stack/\$1|g" "$file" - done - - echo "✅ Versioned docs import paths corrected" + for version_dir in versioned_docs/version-*; do + if [ -d "$version_dir" ]; then + echo " Processing $version_dir..." + python3 "${{ github.workspace }}/inline-raw-loader.py" "$version_dir" "${{ env.TEMP_DIR }}/llama-stack" + fi + done + echo "✅ Versioned docs raw-loader imports inlined" else echo "â„šī¸ No versioned docs found" fi diff --git a/build-archived-version.sh b/build-archived-version.sh index 68ec7bed..f3ecc47e 100755 --- a/build-archived-version.sh +++ b/build-archived-version.sh @@ -63,14 +63,14 @@ if [ -f "static/deprecated-llama-stack-spec.yaml" ]; then npm run gen-api-docs deprecated 2>&1 | grep -E "^Successfully" || true fi -# Step 4: Inline raw-loader imports +# Step 4: Patch docs source (sidebar validation, blog warnings, MDX compat) +echo "--- Patching docs source ---" +"$REPO_DIR/patch-docs-source.sh" --repo-dir "$REPO_DIR" + +# Step 5: Inline raw-loader imports echo "--- Inlining raw-loader imports ---" python3 "$REPO_DIR/inline-raw-loader.py" docs "$TEMP_DIR/llama-stack" -# Step 5: Fix MDX compatibility issues -echo "--- Fixing MDX compatibility ---" -python3 "$REPO_DIR/fix-mdx-compat.py" docs - # Step 6: Patch config for standalone archived build echo "--- Patching config for baseUrl: /$VERSION/ ---" export VERSION @@ -102,12 +102,6 @@ config = config.replace( `themeConfig: {${bannerEntry}` ); -// Fix blog include pattern to support both .md and .mdx -config = config.replace( - /include:\s*\['?\*\.md'?\]/g, - "include: ['*.{md,mdx}']" -); - fs.writeFileSync('docusaurus.config.ts', config); console.log('Config patched'); CONFIGEOF diff --git a/build-latest.sh b/build-latest.sh new file mode 100755 index 00000000..8b4a5855 --- /dev/null +++ b/build-latest.sh @@ -0,0 +1,194 @@ +#!/bin/bash +set -euo pipefail + +# build-latest.sh - Build the latest (main branch) Docusaurus docs locally +# +# Usage: ./build-latest.sh [--llama-stack-dir ] [--output-dir ] +# +# Examples: +# ./build-latest.sh +# ./build-latest.sh --llama-stack-dir /tmp/llama-stack +# ./build-latest.sh --output-dir /tmp/docs-output +# +# Output: docs/ directory (or specified output dir) containing the full static site + +REPO_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Parse optional arguments +LLAMA_STACK_DIR="" +OUTPUT_DIR="$REPO_DIR/docs" +while [[ $# -gt 0 ]]; do + case $1 in + --llama-stack-dir) LLAMA_STACK_DIR="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# Setup temp directory for the build +TEMP_DIR=$(mktemp -d) +BUILD_DIR="$TEMP_DIR/llama-stack/docs" +trap 'rm -rf "$TEMP_DIR"' EXIT + +echo "=== Building latest docs ===" + +# Step 1: Get llama-stack at main branch +if [ -n "$LLAMA_STACK_DIR" ] && [ -d "$LLAMA_STACK_DIR" ]; then + echo "--- Cloning from local repo ---" + git clone --local --no-checkout "$LLAMA_STACK_DIR" "$TEMP_DIR/llama-stack" + cd "$TEMP_DIR/llama-stack" + git checkout main +else + echo "--- Cloning from GitHub ---" + git clone --depth 1 --branch main https://github.com/llamastack/llama-stack.git "$TEMP_DIR/llama-stack" +fi + +cd "$BUILD_DIR" + +# Step 2: Install dependencies +echo "--- Installing dependencies ---" +npm ci 2>&1 | tail -5 + +# Step 3: Patch docs source for clean build +echo "--- Patching docs source ---" +"$REPO_DIR/patch-docs-source.sh" --repo-dir "$REPO_DIR" + +# Step 4: Generate API docs +echo "--- Generating API docs ---" + +if [ -f "static/llama-stack-spec.yaml" ]; then + npm run gen-api-docs stable 2>&1 | grep -E "^Successfully" || true +fi + +if [ -f "static/experimental-llama-stack-spec.yaml" ]; then + npm run gen-api-docs experimental 2>&1 | grep -E "^Successfully" || true +fi + +if [ -f "static/deprecated-llama-stack-spec.yaml" ]; then + npm run gen-api-docs deprecated 2>&1 | grep -E "^Successfully" || true +fi + +# Step 5: Inline raw-loader imports +echo "--- Inlining raw-loader imports ---" +python3 "$REPO_DIR/inline-raw-loader.py" docs "$TEMP_DIR/llama-stack" + +# Step 6: Set up versioning configuration +echo "--- Setting up versioning ---" +cp "$REPO_DIR/versionsArchived.json" ./ + +# Load existing versions.json or create empty +if [ -f "$REPO_DIR/docs/versions.json" ]; then + cp "$REPO_DIR/docs/versions.json" ./ + echo "Loaded existing versions.json" +else + echo "[]" > versions.json + echo "Created empty versions.json" +fi + +# Copy existing versioned_docs and versioned_sidebars if they exist +if [ -d "$REPO_DIR/versioned_docs" ]; then + cp -r "$REPO_DIR/versioned_docs" ./ + echo "Imported existing versioned_docs" +fi + +if [ -d "$REPO_DIR/versioned_sidebars" ]; then + cp -r "$REPO_DIR/versioned_sidebars" ./ + echo "Imported existing versioned_sidebars" +fi + +# Inline raw-loader imports in versioned docs +if [ -d "versioned_docs" ]; then + for version_dir in versioned_docs/version-*; do + if [ -d "$version_dir" ]; then + python3 "$REPO_DIR/inline-raw-loader.py" "$version_dir" "$TEMP_DIR/llama-stack" + fi + done +fi + +# Patch Docusaurus config for versioning +node << 'EOF' +const fs = require('fs'); + +let config = fs.readFileSync('docusaurus.config.ts', 'utf8'); + +// Add versioning imports after OpenAPI import +const versioningImports = ` +// Import fs for versioning configuration +const fs = require('fs'); + +// Versioning configuration for llamastack.github.io +const versionsArchived = (() => { + try { + return JSON.parse(fs.readFileSync('./versionsArchived.json', 'utf8')); + } catch (e) { + console.warn('Could not load versionsArchived.json:', e); + return {}; + } +})(); + +const archivedVersionsDropdownItems = Object.entries(versionsArchived).map( + ([versionName, versionUrl]) => ({ + label: versionName, + href: versionUrl, + }) +); +`; + +config = config.replace( + /import type \* as OpenApiPlugin from "docusaurus-plugin-openapi-docs";/, + `import type * as OpenApiPlugin from "docusaurus-plugin-openapi-docs"; + +${versioningImports}` +); + +// Add version dropdown to navbar (replace GitHub item) +const versionDropdown = ` { + href: 'https://github.com/llamastack/llama-stack', + label: 'GitHub', + position: 'right', +}, +{ + type: 'docsVersionDropdown', + position: 'right', + dropdownItemsAfter: archivedVersionsDropdownItems.length > 0 ? [ + { + type: 'html', + value: '', + }, + { + type: 'html', + className: 'dropdown-archived-versions', + value: 'Previous versions', + }, + ...archivedVersionsDropdownItems, + ] : [], +},`; + +config = config.replace( + /\s*{\s*href:\s*'https:\/\/github\.com\/llamastack\/llama-stack',\s*label:\s*'GitHub',\s*position:\s*'right',\s*},/, + versionDropdown +); + +fs.writeFileSync('docusaurus.config.ts', config); +console.log('Versioning config patched'); +EOF + +# Step 7: Patch out duplicate version badges from OpenAPI MDX files +echo "--- Patching duplicate version badges ---" +find . -name "llama-stack-specification.info.mdx" -type f -exec sed -i '' '/$/d' {} \; 2>/dev/null || \ +find . -name "llama-stack-specification.info.mdx" -type f -exec sed -i '/$/d' {} \; + +# Step 8: Build +echo "--- Building ---" +NODE_OPTIONS="--max-old-space-size=8192" npm run build 2>&1 | tail -50 + +# Step 9: Copy build output +echo "--- Copying to $OUTPUT_DIR ---" +rm -rf "$OUTPUT_DIR" +mkdir -p "$OUTPUT_DIR" +cp -r build/* "$OUTPUT_DIR/" +touch "$OUTPUT_DIR/.nojekyll" + +echo "=== Done building latest docs ===" +echo "Output: $OUTPUT_DIR" +du -sh "$OUTPUT_DIR" diff --git a/patch-docs-source.sh b/patch-docs-source.sh new file mode 100755 index 00000000..a2160295 --- /dev/null +++ b/patch-docs-source.sh @@ -0,0 +1,162 @@ +#!/bin/bash +set -euo pipefail + +# patch-docs-source.sh - Patch llama-stack docs source for a clean Docusaurus build +# +# Usage: ./patch-docs-source.sh [--repo-dir ] +# +# Must be run from inside the llama-stack docs/ directory (where docusaurus.config.ts lives). +# The --repo-dir flag points to the llamastack.github.io repo root (for finding helper scripts). +# If omitted, it defaults to the directory containing this script. +# +# What it does: +# 1. Validates sidebar entries and removes references to non-existent doc files +# 2. Suppresses blog truncation warnings (onUntruncatedBlogPosts: 'warn') +# 3. Fixes blog include pattern to support .mdx files +# 4. Fixes MDX compatibility issues in doc files + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$SCRIPT_DIR" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --repo-dir) REPO_DIR="$2"; shift 2 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# Verify we're in a Docusaurus docs directory +if [ ! -f "docusaurus.config.ts" ]; then + echo "ERROR: docusaurus.config.ts not found in $(pwd)" + echo "This script must be run from inside the llama-stack docs/ directory." + exit 1 +fi + +echo "=== Patching docs source ===" + +# Step 1: Validate sidebar entries and remove references to non-existent files +echo "--- Validating sidebar entries ---" +node << 'SIDEBAR_EOF' +const fs = require('fs'); +const path = require('path'); + +const sidebarsFile = 'sidebars.ts'; +if (!fs.existsSync(sidebarsFile)) { + console.log('No sidebars.ts found, skipping sidebar validation'); + process.exit(0); +} + +let content = fs.readFileSync(sidebarsFile, 'utf8'); +const docsDir = 'docs'; + +// Find all string doc IDs in the sidebars file +// Matches patterns like: 'some/doc/id' or "some/doc/id" in array contexts +const docIdPattern = /['"]([a-zA-Z0-9_\-\/]+)['"]/g; +let match; +let removedCount = 0; + +// Collect all doc IDs referenced in sidebars +const referencedIds = []; +while ((match = docIdPattern.exec(content)) !== null) { + const id = match[1]; + // Skip things that are clearly not doc IDs (config keys, URLs, etc.) + if (id.includes('http') || id.includes('.') || id.length < 3) continue; + if (['category', 'link', 'doc', 'autogenerated', 'right', 'left'].includes(id)) continue; + referencedIds.push(id); +} + +// Check each referenced ID for existence +for (const id of referencedIds) { + const mdPath = path.join(docsDir, id + '.md'); + const mdxPath = path.join(docsDir, id + '.mdx'); + const indexMdPath = path.join(docsDir, id, 'index.md'); + const indexMdxPath = path.join(docsDir, id, 'index.mdx'); + + if (fs.existsSync(mdPath) || fs.existsSync(mdxPath) || + fs.existsSync(indexMdPath) || fs.existsSync(indexMdxPath)) { + continue; + } + + // Doc file doesn't exist - remove this entry from sidebars + // Remove the line containing this ID (handles trailing commas) + const escapedId = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const linePattern = new RegExp(`\\s*['"]${escapedId}['"],?\\s*\\n`, 'g'); + const newContent = content.replace(linePattern, '\n'); + if (newContent !== content) { + console.log(` Removed missing sidebar entry: ${id}`); + content = newContent; + removedCount++; + } +} + +if (removedCount > 0) { + fs.writeFileSync(sidebarsFile, content); + console.log(`Removed ${removedCount} invalid sidebar entries`); +} else { + console.log('All sidebar entries are valid'); +} +SIDEBAR_EOF + +# Step 2: Suppress blog truncation warnings and fix blog include pattern +echo "--- Patching docusaurus.config.ts ---" +node << 'CONFIG_EOF' +const fs = require('fs'); + +let config = fs.readFileSync('docusaurus.config.ts', 'utf8'); +let changed = false; + +// Add onUntruncatedBlogPosts: 'warn' to blog plugin config if not already present +if (!config.includes('onUntruncatedBlogPosts')) { + // Insert into the blog config section - look for blogSidebarCount or blog: { + // Try to find the blog plugin configuration + const blogConfigPatterns = [ + // Pattern: blog: { ... blogSidebarCount: ... + /(blog:\s*\{)/, + // Pattern: preset blog options + /(blogSidebarCount:\s*[^,}]+)/, + ]; + + for (const pattern of blogConfigPatterns) { + const match = config.match(pattern); + if (match) { + if (pattern === blogConfigPatterns[0]) { + config = config.replace(pattern, `$1\n onUntruncatedBlogPosts: 'warn',`); + } else { + config = config.replace(pattern, `$1,\n onUntruncatedBlogPosts: 'warn'`); + } + changed = true; + console.log(' Added onUntruncatedBlogPosts: warn'); + break; + } + } + + if (!changed) { + // Fallback: look for any presets blog section + console.log(' WARNING: Could not find blog config section to patch onUntruncatedBlogPosts'); + } +} + +// Fix blog include pattern to support both .md and .mdx +if (config.match(/include:\s*\['?\*\.md'?\]/)) { + config = config.replace( + /include:\s*\['?\*\.md'?\]/g, + "include: ['*.{md,mdx}']" + ); + changed = true; + console.log(' Fixed blog include pattern for .mdx support'); +} + +if (changed) { + fs.writeFileSync('docusaurus.config.ts', config); + console.log('Config patched'); +} else { + console.log('Config already up to date'); +} +CONFIG_EOF + +# Step 3: Fix MDX compatibility issues +echo "--- Fixing MDX compatibility ---" +python3 "$REPO_DIR/fix-mdx-compat.py" docs + +echo "=== Done patching docs source ==="