diff --git a/samples/llm-usage/.devproxy/azure-ai-prices.json b/samples/llm-usage/.devproxy/azure-ai-prices.json new file mode 100644 index 0000000..af1d2a4 --- /dev/null +++ b/samples/llm-usage/.devproxy/azure-ai-prices.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/openaitelemetryplugin.pricesfile.schema.json", + "prices": { + "gpt-4.1-2025-04-14": { + "input": 0.97, + "output": 3.87 + }, + "llama3.2": { + "input": 0.97, + "output": 3.87 + } + } +} \ No newline at end of file diff --git a/samples/llm-usage/.devproxy/devproxyrc.json b/samples/llm-usage/.devproxy/devproxyrc.json new file mode 100644 index 0000000..4f2eb64 --- /dev/null +++ b/samples/llm-usage/.devproxy/devproxyrc.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/rc.schema.json", + "plugins": [ + { + "name": "LatencyPlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll", + "configSection": "apiLatencyPlugin", + "urlsToWatch": [ + "http://api.ecs.eu/*" + ] + }, + { + "name": "OpenAITelemetryPlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll", + "configSection": "openAITelemetryPlugin" + }, + { + "name": "CrudApiPlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll", + "configSection": "feedbackApi", + "urlsToWatch": [ + "http://api.ecs.eu/feedback" + ] + }, + { + "name": "MarkdownReporter", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll" + } + ], + "urlsToWatch": [ + "https://models.github.ai/inference/chat/completions*" + ], + "feedbackApi": { + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/crudapiplugin.schema.json", + "apiFile": "feedback-api.json" + }, + "apiLatencyPlugin": { + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/latencyplugin.schema.json", + "minMs": 200, + "maxMs": 500 + }, + "openAITelemetryPlugin": { + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/openaitelemetryplugin.schema.json", + "currency": "EUR", + "includeCosts": true, + "pricesFile": "azure-ai-prices.json" + }, + "logLevel": "trace", + "newVersionNotification": "stable", + "showSkipMessages": true +} \ No newline at end of file diff --git a/samples/llm-usage/.devproxy/feedback-api.json b/samples/llm-usage/.devproxy/feedback-api.json new file mode 100644 index 0000000..076331b --- /dev/null +++ b/samples/llm-usage/.devproxy/feedback-api.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/crudapiplugin.apifile.schema.json", + "actions": [ + { + "action": "getAll" + }, + { + "action": "create" + } + ], + "baseUrl": "http://api.ecs.eu/feedback", + "dataFile": "feedback-data.json" +} \ No newline at end of file diff --git a/samples/llm-usage/.devproxy/feedback-data.json b/samples/llm-usage/.devproxy/feedback-data.json new file mode 100644 index 0000000..36b0a13 --- /dev/null +++ b/samples/llm-usage/.devproxy/feedback-data.json @@ -0,0 +1,47 @@ +[ + { + "id": 1, + "feedback": "The presentation slides were very well designed and informative. Would be great to have them shared with all attendees afterward.", + "date": "2025-05-14T15:30:00" + }, + { + "id": 2, + "feedback": "The speaker was excellent but the Q&A session was cut short. Please allocate more time for questions in future events.", + "date": "2025-05-14T16:45:00" + }, + { + "id": 3, + "feedback": "Try to speak more slowly and enunciate more clearly. Some attendees in the back had trouble hearing you.", + "date": "2025-05-14T14:20:00" + }, + { + "id": 4, + "feedback": "The coffee during the break was amazing!", + "date": "2025-05-14T10:15:00" + }, + { + "id": 5, + "feedback": "Include more real-world examples in your next presentation to better illustrate the concepts.", + "date": "2025-05-15T09:30:00" + }, + { + "id": 6, + "feedback": "The room was too cold, please adjust the temperature for tomorrow's sessions.", + "date": "2025-05-15T11:10:00" + }, + { + "id": 7, + "feedback": "I really liked the speaker's tie.", + "date": "2025-05-15T13:45:00" + }, + { + "id": 8, + "feedback": "Your slides had too much text. Consider using more visuals and less text for better audience engagement.", + "date": "2025-05-15T10:20:00" + }, + { + "id": 9, + "feedback": "The session was okay.", + "date": "2025-05-15T13:47:00" + } +] \ No newline at end of file diff --git a/samples/llm-usage/.devproxy/simulate-ai.json b/samples/llm-usage/.devproxy/simulate-ai.json new file mode 100644 index 0000000..6e8d940 --- /dev/null +++ b/samples/llm-usage/.devproxy/simulate-ai.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/rc.schema.json", + "plugins": [ + { + "name": "LatencyPlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll", + "configSection": "apiLatencyPlugin", + "urlsToWatch": [ + "http://api.ecs.eu/*" + ] + }, + { + "name": "OpenAITelemetryPlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll", + "configSection": "openAITelemetryPlugin" + }, + { + "name": "CrudApiPlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll", + "configSection": "feedbackApi", + "urlsToWatch": [ + "http://api.ecs.eu/feedback" + ] + }, + { + "name": "OpenAIMockResponsePlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll" + }, + { + "name": "MarkdownReporter", + "enabled": true, + "pluginPath": "~appFolder/plugins/DevProxy.Plugins.dll" + } + ], + "urlsToWatch": [ + "https://models.github.ai/inference/chat/completions*" + ], + "feedbackApi": { + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/crudapiplugin.schema.json", + "apiFile": "feedback-api.json" + }, + "apiLatencyPlugin": { + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/latencyplugin.schema.json", + "minMs": 200, + "maxMs": 500 + }, + "openAITelemetryPlugin": { + "$schema": "https://raw.githubusercontent.com/dotnet/dev-proxy/main/schemas/v1.0.0/openaitelemetryplugin.schema.json", + "currency": "EUR", + "includeCosts": true, + "pricesFile": "azure-ai-prices.json" + }, + "languageModel": { + "enabled": true, + "model": "llama3.2" + }, + "logLevel": "trace", + "newVersionNotification": "stable", + "showSkipMessages": true +} \ No newline at end of file diff --git a/samples/llm-usage/.github/copilot-instructions.md b/samples/llm-usage/.github/copilot-instructions.md new file mode 100644 index 0000000..6c229ef --- /dev/null +++ b/samples/llm-usage/.github/copilot-instructions.md @@ -0,0 +1,85 @@ +# Dev Proxy LLM Usage Test Project + +## Architecture Overview + +This is a **Dev Proxy test application** that demonstrates LLM usage monitoring and cost tracking. The app fetches feedback from a mock API, uses OpenAI's GPT-4.1 to categorize feedback, and displays results with visual "pills" - all while Dev Proxy intercepts and analyzes the LLM calls. + +**Key Components:** +- **Frontend**: Vanilla JavaScript SPA with OpenAI SDK integration (`js/app.js`) +- **Mock API**: Dev Proxy serves feedback data from `.devproxy/feedback-data.json` +- **LLM Integration**: GitHub Models API (models.github.ai) for GPT-4.1 categorization +- **Testing**: Playwright E2E tests that verify the full AI pipeline +- **Monitoring**: Dev Proxy tracks costs, latency, and usage patterns + +## Critical Dev Proxy Integration + +**Dev Proxy Configuration** (`.devproxy/devproxyrc.json`): +- `CrudApiPlugin`: Serves mock feedback API from `feedback-data.json` +- `OpenAITelemetryPlugin`: Tracks LLM costs and usage (EUR pricing in `azure-ai-prices.json`) +- `LatencyPlugin`: Simulates 200-500ms API delays +- `MarkdownReporter`: Generates usage reports + +**Essential Commands:** +```bash +# Start dev server (port 8007) +npm start + +# Run Playwright tests with Dev Proxy +npm test + +# Install Edge browser for testing +npm run install:msedge +``` + +## LLM Integration Pattern + +**Authentication**: Uses GitHub token via `js/env.js` (replaced by CI workflow) +**API Configuration**: +```javascript +const llmUrl = "https://models.github.ai/inference"; +const model = "openai/gpt-4.1"; +``` + +**Retry Logic**: 3 attempts with category validation (`for-organizers`, `for-speakers`, `useless`) +**UI Pattern**: Pills are hidden by default, revealed via `.pills-visible` class after analysis + +## Testing Strategy + +**E2E Test Flow** (`tests/feedback-analysis.spec.js`): +1. Load app and wait for feedback items +2. Click "Analyze Feedback" button +3. Wait for "Analysis Complete" (120s timeout for LLM calls) +4. Verify all pills have valid categories (not all "Unknown") + +**Key Test Configuration**: +- Uses Microsoft Edge browser only +- Network request/response logging enabled +- Slow test timeout for LLM processing +- Base URL: `http://127.0.0.1:8007` + +## CI/CD Integration + +**GitHub Actions Workflow** (`.github/workflows/test.yml`): +- Replaces `env.js` with actual `GITHUB_TOKEN` +- Sets up Dev Proxy with auto-recording +- Installs Chromium certificates for proxy +- Uploads Dev Proxy reports and logs as artifacts + +**Key Workflow Features**: +- Uses `dev-proxy-tools/actions` for setup +- Enables job summary reporting +- Captures complete proxy logs for debugging + +## Development Guidelines + +**Environment Setup**: Always use Dev Proxy - the app expects intercepted APIs +**API Mocking**: Modify `.devproxy/feedback-data.json` to change test data +**Styling**: Uses CSS custom properties with dark mode support +**Error Handling**: LLM failures gracefully degrade to "unknown" category + +**File Structure**: +- `js/app.js`: Core application logic +- `js/env.js`: API key configuration (replaced in CI) +- `.devproxy/`: Complete Dev Proxy configuration +- `tests/`: Playwright E2E tests +- `css/styles.css`: Modern CSS with pill styling diff --git a/samples/llm-usage/.github/workflows/test.yml b/samples/llm-usage/.github/workflows/test.yml new file mode 100644 index 0000000..e9868ea --- /dev/null +++ b/samples/llm-usage/.github/workflows/test.yml @@ -0,0 +1,67 @@ +name: Test Dev Proxy LLM Usage with Playwright + +on: + workflow_dispatch: + +permissions: + models: read + +jobs: + test-llm-usage: + name: Test Dev Proxy LLM Usage with Playwright + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Update env.js + run: | + echo "Updating env.js..." + echo "export const apiKey = '${{ secrets.GITHUB_TOKEN }}';" > ./js/env.js + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install dependencies + run: npm ci + + - name: Install Edge browser + run: npm run install:msedge + + - name: Setup Dev Proxy + uses: dev-proxy-tools/actions/setup@v1 + with: + auto-record: true + version: v1.0.0-beta.8 + report-job-summary: $GITHUB_STEP_SUMMARY + + - name: Install the Dev Proxy certificate for Chromium + uses: dev-proxy-tools/actions/chromium-cert@v1 + + - name: Run Playwright tests + run: npm test + + - name: Stop recording + uses: dev-proxy-tools/actions/record-stop@v1 + + - name: Upload Dev Proxy reports + uses: actions/upload-artifact@v4 + with: + name: Reports + path: ./*Reporter* + + - name: Show logs + if: always() + run: | + echo "Dev Proxy logs:" + cat devproxy.log + + - name: Upload Dev Proxy logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: devproxy.log + path: devproxy.log diff --git a/samples/llm-usage/.gitignore b/samples/llm-usage/.gitignore new file mode 100644 index 0000000..59313b8 --- /dev/null +++ b/samples/llm-usage/.gitignore @@ -0,0 +1,13 @@ +node_modules +*.log +.DS_Store +*Reporter* + +# Playwright +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ + +# act +.artifacts \ No newline at end of file diff --git a/samples/llm-usage/.vscode/extensions.json b/samples/llm-usage/.vscode/extensions.json new file mode 100644 index 0000000..8293909 --- /dev/null +++ b/samples/llm-usage/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "garrytrinder.dev-proxy-toolkit", + "ms-playwright.playwright", + "github.vscode-github-actions", + "SanjulaGanepola.github-local-actions" + ] +} \ No newline at end of file diff --git a/samples/llm-usage/.vscode/launch.json b/samples/llm-usage/.vscode/launch.json new file mode 100644 index 0000000..efa9770 --- /dev/null +++ b/samples/llm-usage/.vscode/launch.json @@ -0,0 +1,79 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "๐Ÿงช Run tests, local AI & API", + "type": "node", + "request": "launch", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "test" + ], + "console": "integratedTerminal", + "preLaunchTask": "devproxy-start-simulate-ai", + "postDebugTask": "devproxy-stop", + "presentation": { + "group": "Playwright Tests", + "order": 1 + } + }, + { + "name": "๐Ÿงช Run tests, cloud AI & local API", + "type": "node", + "request": "launch", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "test" + ], + "console": "integratedTerminal", + "preLaunchTask": "devproxy-start", + "postDebugTask": "devproxy-stop", + "presentation": { + "group": "Playwright Tests", + "order": 2 + } + }, + { + "name": "๐ŸŒ Run web app, local AI & API", + "type": "node", + "request": "launch", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "start" + ], + "console": "integratedTerminal", + "serverReadyAction": { + "pattern": "Available on:", + "uriFormat": "http://127.0.0.1:8007", + "action": "openExternally" + }, + "preLaunchTask": "devproxy-start-simulate-ai", + "postDebugTask": "devproxy-stop", + "presentation": { + "group": "Web App Launch", + "order": 1 + } + }, + { + "name": "๐ŸŒ Run web app, cloud AI & local API", + "type": "node", + "request": "launch", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "start" + ], + "console": "integratedTerminal", + "serverReadyAction": { + "pattern": "Available on:", + "uriFormat": "http://127.0.0.1:8007", + "action": "openExternally" + }, + "preLaunchTask": "devproxy-start", + "postDebugTask": "devproxy-stop", + "presentation": { + "group": "Web App Launch", + "order": 2 + } + } + ] +} \ No newline at end of file diff --git a/samples/llm-usage/.vscode/tasks.json b/samples/llm-usage/.vscode/tasks.json new file mode 100644 index 0000000..4f0b818 --- /dev/null +++ b/samples/llm-usage/.vscode/tasks.json @@ -0,0 +1,43 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "npm install", + "type": "shell", + "command": "npm", + "args": [ + "install" + ], + "group": "build", + }, + { + "label": "devproxy-start", + "type": "devproxy", + "command": "start", + "isBackground": true, + "problemMatcher": "$devproxy-watch", + "dependsOn": "npm install", + "args": [ + "--record" + ], + }, + { + "label": "devproxy-start-simulate-ai", + "type": "devproxy", + "command": "start", + "isBackground": true, + "problemMatcher": "$devproxy-watch", + "args": [ + "-c", + ".devproxy/simulate-ai.json", + "--record" + ], + "dependsOn": "npm install" + }, + { + "label": "devproxy-stop", + "type": "devproxy", + "command": "stop" + } + ] +} \ No newline at end of file diff --git a/samples/llm-usage/README.md b/samples/llm-usage/README.md new file mode 100644 index 0000000..3159f28 --- /dev/null +++ b/samples/llm-usage/README.md @@ -0,0 +1,84 @@ +# Track language model usage and costs + +## Summary + +Sample web app that demonstrates how to use Dev Proxy to monitor and track LLM usage and costs. + +![GitHub Actions summary for 'Test Dev Proxy LLM Usage with Playwright #38' by garrytrinder. Status: Success. Duration: 3m 40s. Includes LLM usage report for gpt-4-1-2025-04-14 showing token usage and cost in development environment.](/assets/llm-usage-github.png) + +The sample showcases: + +- **LLM Cost Tracking**: Monitor token usage and costs for OpenAI API calls +- **Functional API Integration**: Serve test data using CrudApiPlugin +- **Latency Simulation**: Add realistic delays to API responses +- **VS Code Integration**: Use Dev Proxy Toolkit for local development and testing +- **End-to-End Testing**: Playwright tests that verify the complete AI pipeline +- **CI/CD Integration**: GitHub Actions workflow with automated testing + +## Compatibility + +![Dev Proxy v1.0.0-beta.8](https://img.shields.io/badge/devproxy-v1.0.0-green.svg) + +## Contributors + +* [Garry Trinder](https://github.com/garrytrinder) + +## Version history + +Version|Date|Comments +-------|----|-------- +1.0|July 28, 2025|Initial release + +## Minimal path to awesome + +1. Clone this repository (or [download this solution as a .ZIP file](https://pnp.github.io/download-partial/?url=https://github.com/pnp/proxy-samples/tree/main/samples/llm-usage) then unzip it) +1. Open the repository in Visual Studio Code +1. Install the [Dev Proxy Toolkit](https://marketplace.visualstudio.com/items?itemName=garrytrinder.dev-proxy-toolkit) extension + +### Run locally (with local AI) + +> [!NOTE] +> +> For Dev Proxy to simulate AI responses, you will need to run a local model on your machine which is accessible via an OpenAI API compatible endpoint, .e.g. Ollama. By default, this sample uses the Llama 3.2 model, change the `languageModel.model` property in the `.devproxy/simulate-ai.json` file to use a different model. Ensure that the model is running before starting Dev Proxy. + +1. Start debug session in Visual Studio Code by pressing F5 +1. Wait for the process to complete +1. Open the markdown file that is created in the root of the project to view the LLM usage report for the run. + +### Run locally (with cloud AI) + +1. Generate a fine-grained personal access token with `models:read` permission granted. +1. Update the `apiKey` variable value in `js/env.js` with your token. +1. Open **Run and Debug** panel and select **๐Ÿงช Run tests, cloud AI & local API** debug configuration +1. Start the debug session by pressing F5 +1. Wait for the process to complete +1. Open the markdown file that is created in the root of the project to view the LLM usage report for the run. + +### Run in GitHub Actions + +> [!NOTE] +> +> Enable GitHub Actions in your repository settings before running the workflow. + +1. Push the **main** branch to your GitHub repository +1. Open a browser and navigate to your repository +1. Open the **Actions** tab in your repository +1. Trigger the **Test Dev Proxy LLM Usage with Playwright** workflow manually +1. Wait for the workflow to complete +1. View the usage report in the job summary + +## Help + +We do not support samples, but this community is always willing to help, and we want to improve these samples. We use GitHub to track issues, which makes it easy for community members to volunteer their time and help resolve issues. + +You can try looking at [issues related to this sample](https://github.com/pnp/proxy-samples/issues?q=label%3A%22sample%3A%20llm-usage%22) to see if anybody else is having the same issues. + +If you encounter any issues using this sample, [create a new issue](https://github.com/pnp/proxy-samples/issues/new). + +Finally, if you have an idea for improvement, [make a suggestion](https://github.com/pnp/proxy-samples/issues/new). + +## Disclaimer + +**THIS CODE IS PROVIDED *AS IS* WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING ANY IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR NON-INFRINGEMENT.** + +![](https://m365-visitor-stats.azurewebsites.net/SamplesGallery/proxy-samples-llm-usage) diff --git a/samples/llm-usage/assets/llm-usage-github.png b/samples/llm-usage/assets/llm-usage-github.png new file mode 100644 index 0000000..d575dc1 Binary files /dev/null and b/samples/llm-usage/assets/llm-usage-github.png differ diff --git a/samples/llm-usage/assets/sample.json b/samples/llm-usage/assets/sample.json new file mode 100644 index 0000000..4ed38e1 --- /dev/null +++ b/samples/llm-usage/assets/sample.json @@ -0,0 +1,78 @@ +[ + { + "name": "pnp-devproxy-llm-usage", + "source": "pnp", + "title": "Track language model usage and costs", + "shortDescription": "This sample web application demonstrates how to use Dev Proxy to monitor and track LLM usage and costs.", + "url": "https://github.com/pnp/proxy-samples/tree/main/samples/llm-usage", + "downloadUrl": "https://pnp.github.io/download-partial/?url=https://github.com/pnp/proxy-samples/tree/main/samples/llm-usage", + "longDescription": [ + "This sample web application demonstrates how to use Dev Proxy to monitor and track LLM usage and costs." + ], + "creationDateTime": "2025-07-28", + "updateDateTime": "2025-07-28", + "products": [ + "Dev Proxy" + ], + "metadata": [ + { + "key": "SAMPLE ID", + "value": "llm-usage" + }, + { + "key": "PRESET", + "value": "No" + }, + { + "key": "MOCKS", + "value": "No" + }, + { + "key": "PROXY VERSION", + "value": "v1.0.0" + } + ], + "thumbnails": [ + { + "type": "image", + "order": 100, + "url": "https://github.com/pnp/proxy-samples/raw/main/samples/llm-usage/assets/llm-usage-github.png", + "alt": "GitHub Actions summary for 'Test Dev Proxy LLM Usage with Playwright #38' by garrytrinder. Status: Success. Duration: 3m 40s. Includes LLM usage report for gpt-4-1-2025-04-14 showing token usage and cost in development environment." + } + ], + "authors": [ + { + "gitHubAccount": "garrytrinder", + "pictureUrl": "https://github.com/garrytrinder.png", + "name": "Garry Trinder" + } + ], + "references": [ + { + "name": "Get started with Dev Proxy", + "description": "The tutorial will introduce you to Dev Proxy and show you how to use its features.", + "url": "https://learn.microsoft.com/microsoft-cloud/dev/dev-proxy/get-started" + }, + { + "name": "Simulate OpenAI API", + "description": "Learn how to simulate OpenAI API calls using Dev Proxy.", + "url": "https://learn.microsoft.com/microsoft-cloud/dev/dev-proxy/technical-reference/openaitelemetryplugin" + }, + { + "name": "Use Dev Proxy to track language model usage and costs with GitHub Actions", + "description": "Learn how to use Dev Proxy to track language model usage and costs in GitHub Actions workflows.", + "url": "https://learn.microsoft.com/microsoft-cloud/dev/dev-proxy/how-to/use-dev-proxy-with-github-actions-language-model-usage-costs" + }, + { + "name": "Use Dev Proxy with GitHub Actions", + "description": "Learn how to use Dev Proxy in GitHub Actions worfklows.", + "url": "https://learn.microsoft.com/microsoft-cloud/dev/dev-proxy/how-to/use-dev-proxy-with-github-actions" + }, + { + "name": "Use Dev Proxy with Visual Studio Code debug configurations", + "description": "Learn how to use Dev Proxy with Visual Studio Code debug configurations.", + "url": "https://learn.microsoft.com/microsoft-cloud/dev/dev-proxy/how-to/use-dev-proxy-with-vs-code-debug-configurations" + } + ] + } +] \ No newline at end of file diff --git a/samples/llm-usage/css/styles.css b/samples/llm-usage/css/styles.css new file mode 100644 index 0000000..0af3f61 --- /dev/null +++ b/samples/llm-usage/css/styles.css @@ -0,0 +1,205 @@ +:root { + --background-primary: #f8f8f8; + --background-secondary: #ffffff; + --text-primary: #111111; + --text-secondary: #6e6e6e; + --text-tertiary: #8a8a8a; + --border-color: #e5e5e5; + --accent-blue: #5e6ad2; + --accent-green: #2cb67d; + --accent-orange: #f2994a; + --accent-red: #e56565; + --pill-organizer-bg: #5e6ad2; + --pill-attendee-bg: #2cb67d; + --pill-speaker-bg: #f2994a; + /* Using orange for speakers */ + --pill-useless-bg: #e56565; + --shadow: 0 1px 3px rgba(0, 0, 0, 0.1); +} + +/* Dark mode colors */ +@media (prefers-color-scheme: dark) { + :root { + --background-primary: #151515; + --background-secondary: #1f1f1f; + --text-primary: #ffffff; + --text-secondary: #a9a9a9; + --text-tertiary: #888888; + --border-color: #2a2a2a; + --shadow: 0 1px 3px rgba(0, 0, 0, 0.3); + } +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif; + background-color: var(--background-primary); + color: var(--text-primary); + line-height: 1.5; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.app { + max-width: 800px; + margin: 0 auto; + padding: 2rem 1rem; + display: flex; + flex-direction: column; + min-height: 100vh; +} + +header { + margin-bottom: 2rem; +} + +header h1 { + font-weight: 700; + font-size: 1.5rem; + letter-spacing: -0.02em; +} + +main { + flex-grow: 1; +} + +.evaluations-container { + background-color: var(--background-secondary); + border-radius: 8px; + border: 1px solid var(--border-color); + box-shadow: var(--shadow); + overflow: hidden; +} + +.evaluations-list { + display: flex; + flex-direction: column; +} + +.evaluation-item { + padding: 16px 20px; + border-bottom: 1px solid var(--border-color); + transition: background-color 0.1s ease; +} + +.evaluation-item:last-child { + border-bottom: none; +} + +.evaluation-item:hover { + background-color: rgba(0, 0, 0, 0.02); +} + +.evaluation-content { + margin-bottom: 12px; + color: var(--text-primary); + font-size: 0.9375rem; +} + +.evaluation-meta { + display: flex; + justify-content: space-between; + align-items: center; +} + +.evaluation-pill { + display: inline-block; + padding: 4px 10px; + border-radius: 16px; + font-size: 0.75rem; + font-weight: 500; + color: white; + /* Hide pills by default */ + display: none; +} + +.pills-visible .evaluation-pill { + display: inline-block; +} + +.pill-organizer { + background-color: var(--pill-organizer-bg); +} + +.pill-attendee { + background-color: var(--pill-attendee-bg); +} + +.pill-speaker { + background-color: var(--pill-speaker-bg); +} + +.pill-useless { + background-color: var(--pill-useless-bg); +} + +.evaluation-date { + color: var(--text-tertiary); + font-size: 0.75rem; +} + +footer { + margin-top: 2rem; + text-align: center; + color: var(--text-secondary); + font-size: 0.75rem; +} + +/* Analysis button styles */ +.analysis-container { + display: flex; + justify-content: center; + margin-bottom: 1rem; +} + +.analyze-button { + background-color: var(--accent-blue); + color: white; + border: none; + border-radius: 6px; + padding: 8px 16px; + font-family: 'Inter', sans-serif; + font-weight: 500; + font-size: 0.875rem; + cursor: pointer; + transition: background-color 0.2s ease; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); +} + +.analyze-button:hover { + background-color: rgba(94, 106, 210, 0.9); +} + +.analyze-button:disabled { + background-color: #a0a0a0; + cursor: not-allowed; +} + +.loader { + display: inline-block; + width: 16px; + height: 16px; + margin-left: 8px; + border: 2px solid rgba(255, 255, 255, 0.3); + border-radius: 50%; + border-top-color: white; + animation: spin 1s linear infinite; + vertical-align: middle; +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} + +@media (max-width: 640px) { + .app { + padding: 1rem; + } +} \ No newline at end of file diff --git a/samples/llm-usage/index.html b/samples/llm-usage/index.html new file mode 100644 index 0000000..4e139e9 --- /dev/null +++ b/samples/llm-usage/index.html @@ -0,0 +1,37 @@ + + + + + + + Session Evaluations + + + + + + + +
+
+

Session Evaluations

+
+
+
+ +
+
+
+ +
+
+
+ +
+ + + + + \ No newline at end of file diff --git a/samples/llm-usage/js/app.js b/samples/llm-usage/js/app.js new file mode 100644 index 0000000..a8aa05e --- /dev/null +++ b/samples/llm-usage/js/app.js @@ -0,0 +1,253 @@ +/** + * Session Evaluations Web App + * + * This script fetches and displays session evaluations with classification + * labels (pills) in a modern UI. + */ + +const llmUrl = "https://models.github.ai/inference"; +const model = "openai/gpt-4.1"; + +// Import OpenAI from CDN +import OpenAI from "https://cdn.jsdelivr.net/npm/openai@4.98.0/+esm"; +import { apiKey } from "./env.js"; // Import API key from env.js + +const openai = new OpenAI({ + baseURL: llmUrl, + apiKey, + dangerouslyAllowBrowser: true, +}); + +/** + * Format a date string into a human-readable format + * @param {string} dateString - ISO date string + * @return {string} Formatted date string + */ +function formatDate(dateString) { + const date = new Date(dateString); + return new Intl.DateTimeFormat("en-US", { + month: "short", + day: "numeric", + hour: "numeric", + minute: "numeric", + }).format(date); +} + +/** + * Get the display name and CSS class for a category + * @param {string} category - The category identifier + * @return {Object} Object containing display name and CSS class + */ +function getCategoryInfo(category) { + switch (category) { + case "for-organizers": + return { display: "For Organizers", cssClass: "pill-organizer" }; + case "for-speakers": + return { display: "For Speakers", cssClass: "pill-speaker" }; + case "useless": + return { display: "Useless", cssClass: "pill-useless" }; + default: + return { display: "Unknown", cssClass: "" }; + } +} + +/** + * Creates HTML for an evaluation item + * @param {Object} evaluation - The evaluation data + * @return {string} HTML string for the evaluation + */ +function createEvaluationHTML(evaluation) { + const categoryInfo = getCategoryInfo(evaluation.category); + + return ` +
+

${evaluation.feedback}

+
+ ${ + categoryInfo.display + } + ${formatDate(evaluation.date)} +
+
+ `; +} + +/** + * Fetches evaluations from the API + * @return {Promise} Promise resolving to array of evaluations + */ +async function fetchEvaluations() { + const response = await fetch("http://api.ecs.eu/feedback"); + return await response.json(); +} + +/** + * Renders the evaluations to the DOM + * @param {Array} evaluations - Array of evaluation objects + */ +function renderEvaluations(evaluations) { + const container = document.getElementById("evaluations-list"); + + // Sort evaluations by date (newest first) + const sortedEvaluations = [...evaluations].sort( + (a, b) => new Date(b.date) - new Date(a.date) + ); + + // Generate HTML for all evaluations + const evaluationsHTML = sortedEvaluations + .map((evaluation) => createEvaluationHTML(evaluation)) + .join(""); + + // Update the DOM + container.innerHTML = evaluationsHTML; +} + +/** + * Retrieves the category of feedback using AI + * @param {string} feedbackText - The feedback text to categorize + * @return {Promise} Promise resolving to the category + */ +async function getEvaluationCategory(feedbackText) { + const validCategories = ["for-organizers", "for-speakers", "useless"]; + const maxRetries = 3; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + // Call OpenAI with model + const response = await openai.chat.completions.create({ + model, + messages: [ + { + role: "system", + content: `Classify the following piece of feedback into one of the following categories: ${validCategories.join( + ", " + )}. Respond with the category name only. The feedback is:`, + }, + { + role: "user", + content: feedbackText, + }, + ], + }); + + // Extract the category from the response + const category = response.choices[0].message.content.trim().toLowerCase(); + + // Check if it's a valid category + if (validCategories.includes(category)) { + return category; + } + + console.log( + `Attempt ${attempt}: Invalid category response "${category}". Retrying...` + ); + } catch (error) { + // If we're on the last attempt, rethrow the error + if (attempt === maxRetries) { + throw error; + } + console.log( + `Attempt ${attempt} failed with error: ${error.message}. Retrying...` + ); + } + } + + // If we've exhausted all retries, return a default category + console.warn("Failed to get a valid category after maximum retries"); + return "unknown"; // Default fallback category after all retries fail +} + +/** + * Analyzes feedback content using AI to categorize it + * Shows a loading state and then reveals the pills after analysis is complete + * @param {Array} evaluations - Array of evaluation objects + */ +async function analyzeFeedback(evaluations) { + const button = document.getElementById("analyze-button"); + const evaluationsContainer = document.querySelector(".evaluations-container"); + + // Disable button and show loading state + button.disabled = true; + button.innerHTML = 'Analyzing '; + + // Show pills container immediately + evaluationsContainer.classList.add("pills-visible"); + + // Create a copy of evaluations to work with + const evaluationsCopy = [...evaluations]; + + // Counter to track progress + let processedCount = 0; + const totalCount = evaluations.length; + + try { + // Process each evaluation with OpenAI API + const processingPromises = evaluations.map(async (evaluation, index) => { + try { + // Get category for this evaluation + evaluation.category = await getEvaluationCategory(evaluation.feedback); + + // Update the counter + processedCount++; + + // Update the button text to show progress + button.innerHTML = `Analyzing ${processedCount}/${totalCount} `; + + // Re-render all evaluations with currently available categories + renderEvaluations(evaluationsCopy); + + return evaluation; + } catch (error) { + console.error(`Error processing evaluation #${index}:`, error); + // Set default category for failed analyses + evaluation.category = "unknown"; + return evaluation; + } + }); + + // Wait for all processing to complete + await Promise.all(processingPromises); + + // Final render (should be the same as the last incremental render) + renderEvaluations(evaluationsCopy); + + // Log the analyzed evaluations for reference + console.log("Analyzed evaluations:", evaluationsCopy); + + // Update button state + button.innerHTML = "Analysis Complete"; + // Keep button disabled as analysis is done + } catch (error) { + console.error("Error analyzing feedback:", error); + button.innerHTML = "Analysis Failed"; + setTimeout(() => { + button.disabled = false; + button.innerHTML = "Retry Analysis"; + }, 2000); + } +} + +/** + * Initializes the application + */ +async function initApp() { + try { + // Show loading state (could add a spinner here) + const evaluations = await fetchEvaluations(); + renderEvaluations(evaluations); + + // Add event listener to analyze button + const analyzeButton = document.getElementById("analyze-button"); + analyzeButton.addEventListener("click", () => analyzeFeedback(evaluations)); + } catch (error) { + console.error("Error fetching evaluations:", error); + document.getElementById("evaluations-list").innerHTML = ` +
+

Failed to load evaluations. Please try again later.

+
+ `; + } +} + +// Initialize the app when the DOM is fully loaded +document.addEventListener("DOMContentLoaded", initApp); diff --git a/samples/llm-usage/js/env.js b/samples/llm-usage/js/env.js new file mode 100644 index 0000000..29bf9bc --- /dev/null +++ b/samples/llm-usage/js/env.js @@ -0,0 +1 @@ +export const apiKey = 'steve'; \ No newline at end of file diff --git a/samples/llm-usage/package-lock.json b/samples/llm-usage/package-lock.json new file mode 100644 index 0000000..223ad08 --- /dev/null +++ b/samples/llm-usage/package-lock.json @@ -0,0 +1,661 @@ +{ + "name": "ecs-demo", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ecs-demo", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "http-server": "^14.1.1" + }, + "devDependencies": { + "@playwright/test": "^1.54.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.54.0.tgz", + "integrity": "sha512-6Mnd5daQmLivaLu5kxUg6FxPtXY4sXsS5SUwKjWNy4ISe4pKraNHoFxcsaTFiNUULbjy0Vlb5HT86QuM0Jy1pQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.54.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/corser": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "integrity": "sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-server": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", + "license": "MIT", + "dependencies": { + "basic-auth": "^2.0.1", + "chalk": "^4.1.2", + "corser": "^2.0.1", + "he": "^1.2.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy": "^1.18.1", + "mime": "^1.6.0", + "minimist": "^1.2.6", + "opener": "^1.5.1", + "portfinder": "^1.0.28", + "secure-compare": "3.0.1", + "union": "~0.5.0", + "url-join": "^4.0.1" + }, + "bin": { + "http-server": "bin/http-server" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "license": "(WTFPL OR MIT)", + "bin": { + "opener": "bin/opener-bin.js" + } + }, + "node_modules/playwright": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.54.0.tgz", + "integrity": "sha512-y9yzHmXRwEUOpghM7XGcA38GjWuTOUMaTIcm/5rHcYVjh5MSp9qQMRRMc/+p1cx+csoPnX4wkxAF61v5VKirxg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.54.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.54.0.tgz", + "integrity": "sha512-uiWpWaJh3R3etpJ0QrpligEMl62Dk1iSAB6NUXylvmQz+e3eipXHDHvOvydDAssb5Oqo0E818qdn0L9GcJSTyA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/portfinder": { + "version": "1.0.37", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.37.tgz", + "integrity": "sha512-yuGIEjDAYnnOex9ddMnKZEMFE0CcGo6zbfzDklkmT1m5z734ss6JMzN9rNB3+RR7iS+F10D4/BVIaXOyh8PQKw==", + "license": "MIT", + "dependencies": { + "async": "^3.2.6", + "debug": "^4.3.6" + }, + "engines": { + "node": ">= 10.12" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/secure-compare": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==", + "license": "MIT" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/union": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "dependencies": { + "qs": "^6.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "license": "MIT" + }, + "node_modules/whatwg-encoding": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=12" + } + } + } +} diff --git a/samples/llm-usage/package.json b/samples/llm-usage/package.json new file mode 100644 index 0000000..527924e --- /dev/null +++ b/samples/llm-usage/package.json @@ -0,0 +1,20 @@ +{ + "name": "ecs-demo", + "version": "1.0.0", + "description": "", + "main": "server.js", + "scripts": { + "start": "http-server -c-1 -p 8007", + "test": "playwright test", + "install:msedge": "npx playwright install msedge" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.54.0" + }, + "dependencies": { + "http-server": "^14.1.1" + } +} diff --git a/samples/llm-usage/playwright.config.js b/samples/llm-usage/playwright.config.js new file mode 100644 index 0000000..61f216e --- /dev/null +++ b/samples/llm-usage/playwright.config.js @@ -0,0 +1,45 @@ +// @ts-check +import { defineConfig, devices } from '@playwright/test'; + +/** + * @see https://playwright.dev/docs/test-configuration + */ +export default defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + headless: true, + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: 'http://127.0.0.1:8007', + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + /* Take screenshot on failure */ + screenshot: 'only-on-failure' + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'Microsoft Edge', + use: { ...devices['Desktop Edge'], channel: 'msedge' }, + } + ], + + /* Run your local dev server before starting the tests */ + webServer: { + command: 'npm start', + port: 8007, + reuseExistingServer: !process.env.CI, + timeout: 120 * 1000 + }, +}); diff --git a/samples/llm-usage/tests/feedback-analysis.spec.js b/samples/llm-usage/tests/feedback-analysis.spec.js new file mode 100644 index 0000000..b5157d0 --- /dev/null +++ b/samples/llm-usage/tests/feedback-analysis.spec.js @@ -0,0 +1,49 @@ +// playwright/test-feedback-analysis.spec.js +// Test: Feedback analysis UI end-to-end + +import { test, expect } from '@playwright/test'; + +test.describe('Session Evaluations Analysis', () => { + test('loads feedback, analyzes, and displays pills', async ({ page, baseURL }) => { + + // log network requests for debugging + page.on('request', request => { + console.log(`Request: ${request.method()} ${request.url()}`); + }); + page.on('response', async response => { + console.log(`Response: ${response.status()} ${response.url()}`); + }); + + // increase timeout to handle retries + test.slow(); + + // 1. Load the webpage using its baseUrl + await page.goto(baseURL); + + // 2. Wait for feedback to load (wait for at least one evaluation item) + await page.waitForSelector('.evaluation-item'); + + // 3. When feedback has loaded, press the Analyze button + await page.waitForSelector('#analyze-button'); + const analyzeButton = page.locator('#analyze-button'); + await analyzeButton.click(); + + // 4. Wait for button text to eventually become "Analysis Complete" + await expect(analyzeButton).toHaveText(/Analysis Complete/, { timeout: 120000 }); + + // 5. Verify that all feedbacks contain an evaluation pill that contains a value + const evaluationItems = await page.$$('.evaluation-item'); + const pillTexts = []; + + for (const item of evaluationItems) { + const pill = await item.$('.evaluation-pill'); + const pillText = await pill?.innerText(); + expect(pillText && pillText.trim().length).toBeGreaterThan(0); + pillTexts.push(pillText?.trim()); + } + + // Fail if all pills contain "Unknown" + const allUnknown = pillTexts.every(text => text === 'Unknown'); + expect(allUnknown).toBe(false); + }); +});