ai-testing.yml

name: AI Testing Assistant CI

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main ]
  workflow_dispatch: # Allow manual trigger

jobs:
  test-without-ai:
    name: Standard Tests (No AI)
    runs-on: ubuntu-latest
    timeout-minutes: 30  # Prevent infinite runs
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up JDK 11
      uses: actions/setup-java@v4
      with:
        java-version: '11'
        distribution: 'temurin'
    
    - name: Cache Maven dependencies
      uses: actions/cache@v4
      with:
        path: ~/.m2
        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
        restore-keys: ${{ runner.os }}-m2
    
    - name: Set up Chrome
      uses: browser-actions/setup-chrome@v1
      with:
        chrome-version: stable
    
    - name: Setup ChromeDriver
      run: |
        # Use the new Chrome for Testing API
        CHROME_VERSION=$(google-chrome --version | cut -d ' ' -f3)
        echo "Chrome version: $CHROME_VERSION"
        
        # Get matching ChromeDriver version from Chrome for Testing API
        CHROMEDRIVER_URL=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json" | \
          jq -r ".versions[] | select(.version == \"$CHROME_VERSION\") | .downloads.chromedriver[] | select(.platform == \"linux64\") | .url" | head -1)
        
        if [ -z "$CHROMEDRIVER_URL" ]; then
          # Fallback: get latest stable version
          echo "Exact version not found, using latest stable"
          CHROMEDRIVER_URL=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/latest-versions-per-milestone-with-downloads.json" | \
            jq -r '.milestones | to_entries | sort_by(.key | tonumber) | last | .value.downloads.chromedriver[] | select(.platform == "linux64") | .url')
        fi
        
        if [ -n "$CHROMEDRIVER_URL" ]; then
          echo "Downloading ChromeDriver from: $CHROMEDRIVER_URL"
          wget -O chromedriver.zip "$CHROMEDRIVER_URL"
          unzip chromedriver.zip
          
          # Find the chromedriver binary (it might be in a subdirectory)
          CHROMEDRIVER_BIN=$(find . -name chromedriver -type f | head -1)
          sudo mv "$CHROMEDRIVER_BIN" /usr/local/bin/
          sudo chmod +x /usr/local/bin/chromedriver
        else
          echo "Could not find ChromeDriver download URL, using system package"
          sudo apt-get update && sudo apt-get install -y chromium-chromedriver
          sudo ln -sf /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver
        fi
        
        # Verify installation
        chromedriver --version
    
    - name: Run standard tests (excluding AI tests)
      timeout-minutes: 25
      run: |
        # Run only specific framework unit tests, exclude AI tests and lesson examples
        mvn clean test \
          -Dtest="org.k11techlab.framework_unittests.configurationManagerTests,org.k11techlab.framework_unittests.ScreenshotUtilTest" \
          -Dwebdriver.chrome.driver=/usr/local/bin/chromedriver \
          -Dheadless=true \
          -Djava.awt.headless=true \
          -Dselenium.suppress.cdp.warnings=true \
          -DfailIfNoTests=false

  test-with-ai:
    name: AI Tests with Ollama
    runs-on: ubuntu-latest
    timeout-minutes: 45  # Allow more time for AI/LLM operations
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up JDK 11
      uses: actions/setup-java@v4
      with:
        java-version: '11'
        distribution: 'temurin'
    
    - name: Cache Maven dependencies
      uses: actions/cache@v4
      with:
        path: ~/.m2
        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
        restore-keys: ${{ runner.os }}-m2
    
    - name: Set up Chrome
      uses: browser-actions/setup-chrome@v1
      with:
        chrome-version: stable
    
    - name: Setup ChromeDriver
      run: |
        # Use the new Chrome for Testing API
        CHROME_VERSION=$(google-chrome --version | cut -d ' ' -f3)
        echo "Chrome version: $CHROME_VERSION"
        
        # Get matching ChromeDriver version from Chrome for Testing API
        CHROMEDRIVER_URL=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json" | \
          jq -r ".versions[] | select(.version == \"$CHROME_VERSION\") | .downloads.chromedriver[] | select(.platform == \"linux64\") | .url" | head -1)
        
        if [ -z "$CHROMEDRIVER_URL" ]; then
          # Fallback: get latest stable version
          echo "Exact version not found, using latest stable"
          CHROMEDRIVER_URL=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/latest-versions-per-milestone-with-downloads.json" | \
            jq -r '.milestones | to_entries | sort_by(.key | tonumber) | last | .value.downloads.chromedriver[] | select(.platform == "linux64") | .url')
        fi
        
        if [ -n "$CHROMEDRIVER_URL" ]; then
          echo "Downloading ChromeDriver from: $CHROMEDRIVER_URL"
          wget -O chromedriver.zip "$CHROMEDRIVER_URL"
          unzip chromedriver.zip
          
          # Find the chromedriver binary (it might be in a subdirectory)
          CHROMEDRIVER_BIN=$(find . -name chromedriver -type f | head -1)
          sudo mv "$CHROMEDRIVER_BIN" /usr/local/bin/
          sudo chmod +x /usr/local/bin/chromedriver
        else
          echo "Could not find ChromeDriver download URL, using system package"
          sudo apt-get update && sudo apt-get install -y chromium-chromedriver
          sudo ln -sf /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver
        fi
        
        # Verify installation
        chromedriver --version
        google-chrome --version
    
    - name: Install Ollama
      run: |
        curl -fsSL https://ollama.ai/install.sh | sh
        
    - name: Start Ollama service
      run: |
        # Start Ollama in background
        nohup ollama serve > ollama.log 2>&1 &
        echo "Ollama PID: $!"
        
        # Wait for Ollama to be ready (GitHub Actions compatible)
        for i in {1..30}; do
          if curl -f http://localhost:11434/api/tags >/dev/null 2>&1; then
            echo "✅ Ollama is ready!"
            break
          fi
          echo "⏳ Waiting for Ollama... ($i/30)"
          sleep 2
        done
        
        # Verify Ollama is actually running
        if ! curl -f http://localhost:11434/api/tags >/dev/null 2>&1; then
          echo "❌ Ollama failed to start"
          cat ollama.log
          exit 1
        fi
      
    - name: Pull AI model
      run: |
        # Pull a smaller, faster model for CI with enhanced retry logic
        echo "📥 Pulling tinyllama model..."
        
        # Try pulling tinyllama with retries
        for attempt in {1..3}; do
          echo "Attempt $attempt to pull tinyllama..."
          if timeout 300 ollama pull tinyllama; then
            echo "✅ Successfully pulled tinyllama on attempt $attempt"
            break
          else
            echo "❌ Failed to pull tinyllama on attempt $attempt"
            if [ $attempt -eq 3 ]; then
              echo "⚠️ All attempts failed, trying alternative model"
              timeout 300 ollama pull phi3-mini || {
                echo "❌ Alternative model pull also failed"
                ollama list
              }
            fi
            sleep 10
          fi
        done
        
        echo "📋 Available models after pull:"
        ollama list
        
        # Verify the model is actually usable
        echo "🧪 Testing model functionality:"
        if ollama list | grep -q "tinyllama"; then
          ollama run tinyllama "Say OK" || echo "⚠️ tinyllama test failed"
        elif ollama list | grep -q "phi3-mini"; then
          ollama run phi3-mini "Say OK" || echo "⚠️ phi3-mini test failed"
        else
          echo "❌ No suitable models available"
        fi
      
    - name: Verify Ollama setup
      run: |
        curl http://localhost:11434/api/tags
        ollama list
    
    - name: Create AI test configuration
      run: |
        # Create test resources directory if it doesn't exist
        mkdir -p src/test/resources
        
        # Create AI test configuration
        echo "ai.model=tinyllama" > src/test/resources/ai-test.properties
        echo "ai.timeout=120000" >> src/test/resources/ai-test.properties
        echo "ai.enabled=true" >> src/test/resources/ai-test.properties
        echo "ollama.host=http://localhost:11434" >> src/test/resources/ai-test.properties
        
        echo "📁 Created AI test configuration:"
        cat src/test/resources/ai-test.properties
        
        # Test Ollama with a direct call to verify it's really working
        echo "🧪 Testing Ollama directly before running Java tests:"
        curl -X POST http://localhost:11434/api/generate \
          -H "Content-Type: application/json" \
          -d '{"model": "tinyllama", "prompt": "Say HELLO", "stream": false}' | \
          jq -r '.response' || echo "Direct Ollama test failed"
    
    - name: Run AI tests
      run: |
        # Final verification before running tests
        echo "🔍 Pre-test verification:"
        curl -f http://localhost:11434/api/tags || exit 1
        chromedriver --version || exit 1
        
        echo "🧪 Running comprehensive AI tests..."
        
        # Run AI tests with comprehensive system properties and detailed logging
        mvn test -Dtest="FullAIDemo,SimpleAIDemo,QuickAITest" \
          -Dwebdriver.chrome.driver=/usr/local/bin/chromedriver \
          -Dheadless=true \
          -Dai.model=tinyllama \
          -Dollama.host=http://localhost:11434 \
          -Djava.awt.headless=true \
          -Dtest.parallel=false \
          -Dselenium.suppress.cdp.warnings=true \
          -Dsurefire.useSystemClassLoader=false \
          -X | tee ai-test-output.log
        
        echo "📊 Test execution completed. Checking results..."
        
        # Display key sections of the log
        echo "==== AI TEST OUTPUT SUMMARY ===="
        grep -E "(🤖|✅|❌|🔍|📝|🐛|⚠️)" ai-test-output.log || echo "No AI emojis found in output"
        
        echo "==== SUREFIRE REPORTS ===="
        find target/surefire-reports -name "*.xml" -exec basename {} \; 2>/dev/null || echo "No surefire reports found"
        
        echo "==== TEST RESULTS ===="
        cat target/surefire-reports/TEST-*.xml 2>/dev/null | grep -o 'tests="[0-9]*" errors="[0-9]*" failures="[0-9]*"' || echo "No test results found"
      env:
        OLLAMA_HOST: http://localhost:11434
        DISPLAY: :99
    
    - name: Upload AI test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: ai-test-results
        path: |
          target/surefire-reports/
          target/site/
    
    - name: Display detailed AI test results
      if: always()
      run: |
        echo "📊 DETAILED AI TEST ANALYSIS:"
        echo "============================"
        
        # Show test counts and results
        if [ -d "target/surefire-reports" ]; then
          echo "📁 Surefire reports directory exists"
          ls -la target/surefire-reports/
          
          # Parse and display test results
          for report in target/surefire-reports/TEST-*.xml; do
            if [ -f "$report" ]; then
              echo "📋 Processing report: $(basename $report)"
              
              # Extract test statistics
              tests=$(grep -o 'tests="[0-9]*"' "$report" | cut -d'"' -f2)
              errors=$(grep -o 'errors="[0-9]*"' "$report" | cut -d'"' -f2)
              failures=$(grep -o 'failures="[0-9]*"' "$report" | cut -d'"' -f2)
              
              echo "   Tests: $tests, Errors: $errors, Failures: $failures"
              
              # Show any failure messages
              if [ "$failures" != "0" ] || [ "$errors" != "0" ]; then
                echo "   ❌ Failure details:"
                grep -A5 -B2 '<failure\|<error' "$report" || echo "   No detailed failure info found"
              fi
            fi
          done
        else
          echo "❌ No surefire reports directory found"
        fi
        
        # Show console output
        if [ -f "ai-test-output.log" ]; then
          echo "📄 Console output analysis:"
          echo "AI client initializations: $(grep -c 'Ollama AI Status' ai-test-output.log || echo '0')"
          echo "Real AI responses: $(grep -c 'REAL AI' ai-test-output.log || echo '0')"
          echo "Test completions: $(grep -c 'working!' ai-test-output.log || echo '0')"
          
          # Show last few lines of important output
          echo "🔍 Last AI-related outputs:"
          grep -E "(🤖|✅|❌)" ai-test-output.log | tail -10 || echo "No AI emoji outputs found"
        fi
    
    - name: Generate AI test report
      if: always()
      run: |
        echo "## 🤖 AI Test Results" >> $GITHUB_STEP_SUMMARY
        echo "AI tests completed with Ollama + tinyllama model" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        
        # Check for test results
        if [ -f target/surefire-reports/TEST-*.xml ]; then
          echo "✅ AI test reports generated" >> $GITHUB_STEP_SUMMARY
          
          # Count tests
          TOTAL_TESTS=$(grep -o 'tests="[0-9]*"' target/surefire-reports/TEST-*.xml | grep -o '[0-9]*' | awk '{sum+=$1} END {print sum}')
          FAILED_TESTS=$(grep -o 'failures="[0-9]*"' target/surefire-reports/TEST-*.xml | grep -o '[0-9]*' | awk '{sum+=$1} END {print sum}')
          
          echo "📊 **Test Summary:**" >> $GITHUB_STEP_SUMMARY
          echo "- Total Tests: ${TOTAL_TESTS:-0}" >> $GITHUB_STEP_SUMMARY
          echo "- Failed Tests: ${FAILED_TESTS:-0}" >> $GITHUB_STEP_SUMMARY
        else
          echo "❌ No test reports found" >> $GITHUB_STEP_SUMMARY
        fi
        
        # Show Ollama status
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "🔧 **Environment:**" >> $GITHUB_STEP_SUMMARY
        echo "- Ollama Status: $(curl -f http://localhost:11434/api/tags >/dev/null 2>&1 && echo 'Running' || echo 'Stopped')" >> $GITHUB_STEP_SUMMARY
        echo "- Available Models: $(ollama list | tail -n +2 | wc -l)" >> $GITHUB_STEP_SUMMARY

  test-ai-fallback:
    name: AI Fallback Tests (No Ollama)
    runs-on: ubuntu-latest
    timeout-minutes: 20  # Shorter timeout for fallback tests
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up JDK 11
      uses: actions/setup-java@v4
      with:
        java-version: '11'
        distribution: 'temurin'
    
    - name: Cache Maven dependencies
      uses: actions/cache@v4
      with:
        path: ~/.m2
        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
        restore-keys: ${{ runner.os }}-m2
    
    - name: Set up Chrome
      uses: browser-actions/setup-chrome@v1
      with:
        chrome-version: stable
    
    - name: Setup ChromeDriver
      run: |
        # Use the new Chrome for Testing API
        CHROME_VERSION=$(google-chrome --version | cut -d ' ' -f3)
        echo "Chrome version: $CHROME_VERSION"
        
        # Get matching ChromeDriver version from Chrome for Testing API
        CHROMEDRIVER_URL=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json" | \
          jq -r ".versions[] | select(.version == \"$CHROME_VERSION\") | .downloads.chromedriver[] | select(.platform == \"linux64\") | .url" | head -1)
        
        if [ -z "$CHROMEDRIVER_URL" ]; then
          # Fallback: get latest stable version
          echo "Exact version not found, using latest stable"
          CHROMEDRIVER_URL=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/latest-versions-per-milestone-with-downloads.json" | \
            jq -r '.milestones | to_entries | sort_by(.key | tonumber) | last | .value.downloads.chromedriver[] | select(.platform == "linux64") | .url')
        fi
        
        if [ -n "$CHROMEDRIVER_URL" ]; then
          echo "Downloading ChromeDriver from: $CHROMEDRIVER_URL"
          wget -O chromedriver.zip "$CHROMEDRIVER_URL"
          unzip chromedriver.zip
          
          # Find the chromedriver binary (it might be in a subdirectory)
          CHROMEDRIVER_BIN=$(find . -name chromedriver -type f | head -1)
          sudo mv "$CHROMEDRIVER_BIN" /usr/local/bin/
          sudo chmod +x /usr/local/bin/chromedriver
        else
          echo "Could not find ChromeDriver download URL, using system package"
          sudo apt-get update && sudo apt-get install -y chromium-chromedriver
          sudo ln -sf /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver
        fi
        
        # Verify installation
        chromedriver --version
    
    - name: Run AI fallback tests (SimpleAIClient only)
      run: |
        mvn test -Dtest="org.k11techlab.framework_unittests.aiTests.SimpleAIDemo" \
          -Dwebdriver.chrome.driver=/usr/local/bin/chromedriver \
          -Dheadless=true
    
    - name: Generate fallback test report
      if: always()
      run: |
        echo "## 🔧 AI Fallback Test Results" >> $GITHUB_STEP_SUMMARY
        echo "AI fallback tests completed using SimpleAIClient" >> $GITHUB_STEP_SUMMARY
        echo "These tests verify AI framework works without Ollama" >> $GITHUB_STEP_SUMMARY