From 43d592f2e72a07d7cc15d64d25bf1df164cb4a42 Mon Sep 17 00:00:00 2001 From: Landon Cox Date: Thu, 2 Apr 2026 13:00:25 -0700 Subject: [PATCH 1/2] perf: exclude browser tools and reduce turns in smoke-copilot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements recommendations from #1624: 1. Exclude 21 unused Playwright/browser tools via --excluded-tools in postprocess-smoke-workflows.ts (saves ~10,500 tokens/turn) 2. Remove redundant MCP verification call — pre-step already proves MCP connectivity, no need for agent to call list_pull_requests 3. Remove redundant bash echo test — bash functionality is already proven by the file write/read test in section 3 4. Drop repos toolset — only pull_requests tools are used, removing 4 unused tool schemas (~2,400 tokens/turn) Expected impact: 5→3 LLM turns, ~25-30% token reduction per run. Closes #1624 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/smoke-copilot.lock.yml | 37 +++++++++++------------ .github/workflows/smoke-copilot.md | 7 ++--- scripts/ci/postprocess-smoke-workflows.ts | 24 +++++++++++++++ 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index f2796e77..7a49ea8b 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -22,7 +22,7 @@ # # Smoke Copilot # -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"645e02150c10debfc8b183a29f0bd9d21f049ed186f1390711815e35c5c8f981","compiler_version":"v0.65.5","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"cf6434aefd0a2238761794df41a93c37ffab5fac9dc1e9dbd4aaa08b804ee04a","compiler_version":"v0.65.5","strict":true,"agent_id":"copilot"} name: "Smoke Copilot" "on": @@ -172,14 +172,14 @@ jobs: run: | bash ${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh { - cat << 'GH_AW_PROMPT_4844a2c00237214e_EOF' + cat << 'GH_AW_PROMPT_281d3298617bfa61_EOF' - GH_AW_PROMPT_4844a2c00237214e_EOF + GH_AW_PROMPT_281d3298617bfa61_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_4844a2c00237214e_EOF' + cat << 'GH_AW_PROMPT_281d3298617bfa61_EOF' Tools: add_comment, add_labels, missing_tool, missing_data, noop @@ -211,18 +211,17 @@ jobs: {{/if}} - GH_AW_PROMPT_4844a2c00237214e_EOF + GH_AW_PROMPT_281d3298617bfa61_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_4844a2c00237214e_EOF' + cat << 'GH_AW_PROMPT_281d3298617bfa61_EOF' {{#runtime-import .github/workflows/smoke-copilot.md}} - GH_AW_PROMPT_4844a2c00237214e_EOF + GH_AW_PROMPT_281d3298617bfa61_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_EXPR_EC16C26C: ${{ steps.smoke-data.outputs.SMOKE_FILE_CONTENT }} GH_AW_EXPR_2805DAC9: ${{ steps.smoke-data.outputs.SMOKE_FILE_PATH }} GH_AW_EXPR_7EA93000: ${{ steps.smoke-data.outputs.SMOKE_HTTP_CODE }} @@ -413,12 +412,12 @@ jobs: mkdir -p ${RUNNER_TEMP}/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_fc363219b5fe465c_EOF' + cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_471f689b1ef28e93_EOF' {"add_comment":{"hide_older_comments":true,"max":1},"add_labels":{"allowed":["smoke-copilot"]},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"}} - GH_AW_SAFE_OUTPUTS_CONFIG_fc363219b5fe465c_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_471f689b1ef28e93_EOF - name: Write Safe Outputs Tools run: | - cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/tools_meta.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_META_40e0ce5b281f6039_EOF' + cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/tools_meta.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_META_442e987a70006619_EOF' { "description_suffixes": { "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added.", @@ -427,8 +426,8 @@ jobs: "repo_params": {}, "dynamic_tools": [] } - GH_AW_SAFE_OUTPUTS_TOOLS_META_40e0ce5b281f6039_EOF - cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_5a9c65ba3145686b_EOF' + GH_AW_SAFE_OUTPUTS_TOOLS_META_442e987a70006619_EOF + cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_eaf0db7e83085f6e_EOF' { "add_comment": { "defaultMax": 1, @@ -525,7 +524,7 @@ jobs: } } } - GH_AW_SAFE_OUTPUTS_VALIDATION_5a9c65ba3145686b_EOF + GH_AW_SAFE_OUTPUTS_VALIDATION_eaf0db7e83085f6e_EOF node ${RUNNER_TEMP}/gh-aw/actions/generate_safe_outputs_tools.cjs - name: Generate Safe Outputs MCP Server Config id: safe-outputs-config @@ -593,7 +592,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.11' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_09a986e954b2b810_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh + cat << GH_AW_MCP_CONFIG_bd8c8a3403a06a63_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh { "mcpServers": { "github": { @@ -603,7 +602,7 @@ jobs: "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", - "GITHUB_TOOLSETS": "repos,pull_requests" + "GITHUB_TOOLSETS": "pull_requests" }, "guard-policies": { "allow-only": { @@ -634,7 +633,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_09a986e954b2b810_EOF + GH_AW_MCP_CONFIG_bd8c8a3403a06a63_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -652,7 +651,7 @@ jobs: touch /tmp/gh-aw/agent-step-summary.md # shellcheck disable=SC1003 sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --excluded-tools=browser_close,browser_resize,browser_console_messages,browser_handle_dialog,browser_evaluate,browser_file_upload,browser_fill_form,browser_press_key,browser_type,browser_navigate,browser_navigate_back,browser_network_requests,browser_run_code,browser_take_screenshot,browser_snapshot,browser_click,browser_drag,browser_hover,browser_select_option,browser_tabs,browser_wait_for --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} @@ -1063,7 +1062,7 @@ jobs: touch /tmp/gh-aw/agent-step-summary.md # shellcheck disable=SC1003 sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,telemetry.enterprise.githubcopilot.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --excluded-tools=browser_close,browser_resize,browser_console_messages,browser_handle_dialog,browser_evaluate,browser_file_upload,browser_fill_form,browser_press_key,browser_type,browser_navigate,browser_navigate_back,browser_network_requests,browser_run_code,browser_take_screenshot,browser_snapshot,browser_click,browser_drag,browser_hover,browser_select_option,browser_tabs,browser_wait_for --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} diff --git a/.github/workflows/smoke-copilot.md b/.github/workflows/smoke-copilot.md index 4da4269d..51432950 100644 --- a/.github/workflows/smoke-copilot.md +++ b/.github/workflows/smoke-copilot.md @@ -22,7 +22,7 @@ tools: bash: - "*" github: - toolsets: [repos, pull_requests] + toolsets: [pull_requests] safe-outputs: add-comment: hide-older-comments: true @@ -100,7 +100,7 @@ post-steps: The following tests were already executed in a deterministic pre-agent step. Your job is to verify the results and produce the summary comment. ### 1. GitHub MCP Testing -The last 2 merged pull requests have been fetched. Verify MCP connectivity by calling `github-list_pull_requests` for ${{ github.repository }} (limit 1, state merged) and confirm data is returned. +Pre-step result: MCP connectivity confirmed — 2 merged PRs were pre-fetched successfully (see data below). No additional MCP call is required. ### 2. GitHub.com Connectivity Pre-step result: HTTP ${{ steps.smoke-data.outputs.SMOKE_HTTP_CODE }} from github.com. @@ -111,9 +111,6 @@ Pre-step wrote and read back: "${{ steps.smoke-data.outputs.SMOKE_FILE_CONTENT } File path: ${{ steps.smoke-data.outputs.SMOKE_FILE_PATH }} Verify by running `cat` on the file path using bash to confirm it exists. -### 4. Bash Tool Testing -Run a simple bash command (e.g., `echo "bash works"`) to verify the bash tool is functional. - ## Pre-Fetched PR Data ``` diff --git a/scripts/ci/postprocess-smoke-workflows.ts b/scripts/ci/postprocess-smoke-workflows.ts index 0381224b..00e7da00 100644 --- a/scripts/ci/postprocess-smoke-workflows.ts +++ b/scripts/ci/postprocess-smoke-workflows.ts @@ -244,6 +244,30 @@ for (const workflowPath of workflowPaths) { console.log(` Replaced ${imageTagMatches.length} --image-tag/--skip-pull with --build-local`); } + // Exclude unused Playwright/browser tools from Copilot CLI for smoke-copilot. + // The Copilot CLI includes 21 built-in browser_* tools when --allow-all-tools is set. + // These tools are never used in smoke-copilot but add ~10,500 tokens/turn of dead weight. + // We inject --excluded-tools after --allow-all-tools to suppress them. + const isCopilotSmoke = workflowPath.includes('smoke-copilot.lock.yml'); + if (isCopilotSmoke) { + const excludedToolsFlag = + '--excluded-tools=browser_close,browser_resize,browser_console_messages,' + + 'browser_handle_dialog,browser_evaluate,browser_file_upload,browser_fill_form,' + + 'browser_press_key,browser_type,browser_navigate,browser_navigate_back,' + + 'browser_network_requests,browser_run_code,browser_take_screenshot,' + + 'browser_snapshot,browser_click,browser_drag,browser_hover,' + + 'browser_select_option,browser_tabs,browser_wait_for'; + const allowAllToolsCount = (content.match(/--allow-all-tools/g) || []).length; + if (allowAllToolsCount > 0 && !content.includes('--excluded-tools')) { + content = content.replace( + /--allow-all-tools/g, + `--allow-all-tools ${excludedToolsFlag}` + ); + modified = true; + console.log(` Injected --excluded-tools (21 browser tools) in ${allowAllToolsCount} location(s)`); + } + } + // Remove unused "Setup Scripts" step from update_cache_memory jobs. // The step downloads a private action but is never used in these jobs, // causing 401 Unauthorized failures when permissions: {} is set. From 95f2f55214bc60e0ac90f9776dc04d207aec42d4 Mon Sep 17 00:00:00 2001 From: Landon Cox Date: Thu, 2 Apr 2026 13:09:07 -0700 Subject: [PATCH 2/2] fix: address PR review feedback - Restore MCP call in section 1 so the workflow actually tests GitHub MCP connectivity (not just CLI/API) - Make --excluded-tools injection idempotent: strip any existing flag before re-injecting so updates to the excluded list are always applied correctly on re-runs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/smoke-copilot.lock.yml | 1 + .github/workflows/smoke-copilot.md | 2 +- scripts/ci/postprocess-smoke-workflows.ts | 9 ++++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 7a49ea8b..815d897c 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -222,6 +222,7 @@ jobs: uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_EXPR_EC16C26C: ${{ steps.smoke-data.outputs.SMOKE_FILE_CONTENT }} GH_AW_EXPR_2805DAC9: ${{ steps.smoke-data.outputs.SMOKE_FILE_PATH }} GH_AW_EXPR_7EA93000: ${{ steps.smoke-data.outputs.SMOKE_HTTP_CODE }} diff --git a/.github/workflows/smoke-copilot.md b/.github/workflows/smoke-copilot.md index 51432950..3b9363d7 100644 --- a/.github/workflows/smoke-copilot.md +++ b/.github/workflows/smoke-copilot.md @@ -100,7 +100,7 @@ post-steps: The following tests were already executed in a deterministic pre-agent step. Your job is to verify the results and produce the summary comment. ### 1. GitHub MCP Testing -Pre-step result: MCP connectivity confirmed — 2 merged PRs were pre-fetched successfully (see data below). No additional MCP call is required. +Verify MCP connectivity by calling `github-list_pull_requests` for ${{ github.repository }} (limit 1, state merged). Confirm the result matches the pre-fetched data below. ### 2. GitHub.com Connectivity Pre-step result: HTTP ${{ steps.smoke-data.outputs.SMOKE_HTTP_CODE }} from github.com. diff --git a/scripts/ci/postprocess-smoke-workflows.ts b/scripts/ci/postprocess-smoke-workflows.ts index 00e7da00..d9ab4c05 100644 --- a/scripts/ci/postprocess-smoke-workflows.ts +++ b/scripts/ci/postprocess-smoke-workflows.ts @@ -257,8 +257,15 @@ for (const workflowPath of workflowPaths) { 'browser_network_requests,browser_run_code,browser_take_screenshot,' + 'browser_snapshot,browser_click,browser_drag,browser_hover,' + 'browser_select_option,browser_tabs,browser_wait_for'; + // First, strip any existing --excluded-tools flag to make this idempotent + const existingExcludedRegex = / --excluded-tools=[^\s'"]*/g; + const existingMatches = content.match(existingExcludedRegex); + if (existingMatches) { + content = content.replace(existingExcludedRegex, ''); + console.log(` Removed ${existingMatches.length} existing --excluded-tools flag(s)`); + } const allowAllToolsCount = (content.match(/--allow-all-tools/g) || []).length; - if (allowAllToolsCount > 0 && !content.includes('--excluded-tools')) { + if (allowAllToolsCount > 0) { content = content.replace( /--allow-all-tools/g, `--allow-all-tools ${excludedToolsFlag}`