diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index f2796e77..815d897c 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -22,7 +22,7 @@ # # Smoke Copilot # -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"645e02150c10debfc8b183a29f0bd9d21f049ed186f1390711815e35c5c8f981","compiler_version":"v0.65.5","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"cf6434aefd0a2238761794df41a93c37ffab5fac9dc1e9dbd4aaa08b804ee04a","compiler_version":"v0.65.5","strict":true,"agent_id":"copilot"} name: "Smoke Copilot" "on": @@ -172,14 +172,14 @@ jobs: run: | bash ${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh { - cat << 'GH_AW_PROMPT_4844a2c00237214e_EOF' + cat << 'GH_AW_PROMPT_281d3298617bfa61_EOF' - GH_AW_PROMPT_4844a2c00237214e_EOF + GH_AW_PROMPT_281d3298617bfa61_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_4844a2c00237214e_EOF' + cat << 'GH_AW_PROMPT_281d3298617bfa61_EOF' Tools: add_comment, add_labels, missing_tool, missing_data, noop @@ -211,12 +211,12 @@ jobs: {{/if}} - GH_AW_PROMPT_4844a2c00237214e_EOF + GH_AW_PROMPT_281d3298617bfa61_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_4844a2c00237214e_EOF' + cat << 'GH_AW_PROMPT_281d3298617bfa61_EOF' {{#runtime-import .github/workflows/smoke-copilot.md}} - GH_AW_PROMPT_4844a2c00237214e_EOF + GH_AW_PROMPT_281d3298617bfa61_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -413,12 +413,12 @@ jobs: mkdir -p ${RUNNER_TEMP}/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_fc363219b5fe465c_EOF' + cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_471f689b1ef28e93_EOF' {"add_comment":{"hide_older_comments":true,"max":1},"add_labels":{"allowed":["smoke-copilot"]},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"}} - GH_AW_SAFE_OUTPUTS_CONFIG_fc363219b5fe465c_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_471f689b1ef28e93_EOF - name: Write Safe Outputs Tools run: | - cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/tools_meta.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_META_40e0ce5b281f6039_EOF' + cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/tools_meta.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_META_442e987a70006619_EOF' { "description_suffixes": { "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added.", @@ -427,8 +427,8 @@ jobs: "repo_params": {}, "dynamic_tools": [] } - GH_AW_SAFE_OUTPUTS_TOOLS_META_40e0ce5b281f6039_EOF - cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_5a9c65ba3145686b_EOF' + GH_AW_SAFE_OUTPUTS_TOOLS_META_442e987a70006619_EOF + cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_eaf0db7e83085f6e_EOF' { "add_comment": { "defaultMax": 1, @@ -525,7 +525,7 @@ jobs: } } } - GH_AW_SAFE_OUTPUTS_VALIDATION_5a9c65ba3145686b_EOF + GH_AW_SAFE_OUTPUTS_VALIDATION_eaf0db7e83085f6e_EOF node ${RUNNER_TEMP}/gh-aw/actions/generate_safe_outputs_tools.cjs - name: Generate Safe Outputs MCP Server Config id: safe-outputs-config @@ -593,7 +593,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.11' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_09a986e954b2b810_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh + cat << GH_AW_MCP_CONFIG_bd8c8a3403a06a63_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh { "mcpServers": { "github": { @@ -603,7 +603,7 @@ jobs: "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", - "GITHUB_TOOLSETS": "repos,pull_requests" + "GITHUB_TOOLSETS": "pull_requests" }, "guard-policies": { "allow-only": { @@ -634,7 +634,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_09a986e954b2b810_EOF + GH_AW_MCP_CONFIG_bd8c8a3403a06a63_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -652,7 +652,7 @@ jobs: touch /tmp/gh-aw/agent-step-summary.md # shellcheck disable=SC1003 sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --excluded-tools=browser_close,browser_resize,browser_console_messages,browser_handle_dialog,browser_evaluate,browser_file_upload,browser_fill_form,browser_press_key,browser_type,browser_navigate,browser_navigate_back,browser_network_requests,browser_run_code,browser_take_screenshot,browser_snapshot,browser_click,browser_drag,browser_hover,browser_select_option,browser_tabs,browser_wait_for --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} @@ -1063,7 +1063,7 @@ jobs: touch /tmp/gh-aw/agent-step-summary.md # shellcheck disable=SC1003 sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,telemetry.enterprise.githubcopilot.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --excluded-tools=browser_close,browser_resize,browser_console_messages,browser_handle_dialog,browser_evaluate,browser_file_upload,browser_fill_form,browser_press_key,browser_type,browser_navigate,browser_navigate_back,browser_network_requests,browser_run_code,browser_take_screenshot,browser_snapshot,browser_click,browser_drag,browser_hover,browser_select_option,browser_tabs,browser_wait_for --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} diff --git a/.github/workflows/smoke-copilot.md b/.github/workflows/smoke-copilot.md index 4da4269d..3b9363d7 100644 --- a/.github/workflows/smoke-copilot.md +++ b/.github/workflows/smoke-copilot.md @@ -22,7 +22,7 @@ tools: bash: - "*" github: - toolsets: [repos, pull_requests] + toolsets: [pull_requests] safe-outputs: add-comment: hide-older-comments: true @@ -100,7 +100,7 @@ post-steps: The following tests were already executed in a deterministic pre-agent step. Your job is to verify the results and produce the summary comment. ### 1. GitHub MCP Testing -The last 2 merged pull requests have been fetched. Verify MCP connectivity by calling `github-list_pull_requests` for ${{ github.repository }} (limit 1, state merged) and confirm data is returned. +Verify MCP connectivity by calling `github-list_pull_requests` for ${{ github.repository }} (limit 1, state merged). Confirm the result matches the pre-fetched data below. ### 2. GitHub.com Connectivity Pre-step result: HTTP ${{ steps.smoke-data.outputs.SMOKE_HTTP_CODE }} from github.com. @@ -111,9 +111,6 @@ Pre-step wrote and read back: "${{ steps.smoke-data.outputs.SMOKE_FILE_CONTENT } File path: ${{ steps.smoke-data.outputs.SMOKE_FILE_PATH }} Verify by running `cat` on the file path using bash to confirm it exists. -### 4. Bash Tool Testing -Run a simple bash command (e.g., `echo "bash works"`) to verify the bash tool is functional. - ## Pre-Fetched PR Data ``` diff --git a/scripts/ci/postprocess-smoke-workflows.ts b/scripts/ci/postprocess-smoke-workflows.ts index 0381224b..d9ab4c05 100644 --- a/scripts/ci/postprocess-smoke-workflows.ts +++ b/scripts/ci/postprocess-smoke-workflows.ts @@ -244,6 +244,37 @@ for (const workflowPath of workflowPaths) { console.log(` Replaced ${imageTagMatches.length} --image-tag/--skip-pull with --build-local`); } + // Exclude unused Playwright/browser tools from Copilot CLI for smoke-copilot. + // The Copilot CLI includes 21 built-in browser_* tools when --allow-all-tools is set. + // These tools are never used in smoke-copilot but add ~10,500 tokens/turn of dead weight. + // We inject --excluded-tools after --allow-all-tools to suppress them. + const isCopilotSmoke = workflowPath.includes('smoke-copilot.lock.yml'); + if (isCopilotSmoke) { + const excludedToolsFlag = + '--excluded-tools=browser_close,browser_resize,browser_console_messages,' + + 'browser_handle_dialog,browser_evaluate,browser_file_upload,browser_fill_form,' + + 'browser_press_key,browser_type,browser_navigate,browser_navigate_back,' + + 'browser_network_requests,browser_run_code,browser_take_screenshot,' + + 'browser_snapshot,browser_click,browser_drag,browser_hover,' + + 'browser_select_option,browser_tabs,browser_wait_for'; + // First, strip any existing --excluded-tools flag to make this idempotent + const existingExcludedRegex = / --excluded-tools=[^\s'"]*/g; + const existingMatches = content.match(existingExcludedRegex); + if (existingMatches) { + content = content.replace(existingExcludedRegex, ''); + console.log(` Removed ${existingMatches.length} existing --excluded-tools flag(s)`); + } + const allowAllToolsCount = (content.match(/--allow-all-tools/g) || []).length; + if (allowAllToolsCount > 0) { + content = content.replace( + /--allow-all-tools/g, + `--allow-all-tools ${excludedToolsFlag}` + ); + modified = true; + console.log(` Injected --excluded-tools (21 browser tools) in ${allowAllToolsCount} location(s)`); + } + } + // Remove unused "Setup Scripts" step from update_cache_memory jobs. // The step downloads a private action but is never used in these jobs, // causing 401 Unauthorized failures when permissions: {} is set.