From e55e475b0f446f7c70812121c6a60892b96b5d30 Mon Sep 17 00:00:00 2001 From: MantisClone Date: Fri, 21 Mar 2025 17:21:10 -0400 Subject: [PATCH 1/2] fix: token name --- .github/workflows/integration-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 2962fbf..60b8e45 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -55,7 +55,7 @@ jobs: This is a test of the automated comment system. This comment was triggered by the integration test workflow. secrets: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} # For manual testing - simulate PR events manual-test: @@ -67,7 +67,7 @@ jobs: id: create-issue uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.GH_PAT }} script: | const commentType = '${{ github.event.inputs.test_comment_type }}'; const workflowRef = '${{ github.event.inputs.workflow_ref }}'; @@ -92,7 +92,7 @@ jobs: if: github.event.inputs.test_comment_type == 'first_pr' uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.GH_PAT }} script: | const issueNumber = JSON.parse('${{ steps.create-issue.outputs.result }}').issue_number; @@ -121,7 +121,7 @@ jobs: if: github.event.inputs.test_comment_type == 'ready_for_review' uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.GH_PAT }} script: | const issueNumber = JSON.parse('${{ steps.create-issue.outputs.result }}').issue_number; @@ -149,7 +149,7 @@ jobs: if: github.event.inputs.test_comment_type == 'merged' uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.GH_PAT }} script: | const issueNumber = JSON.parse('${{ steps.create-issue.outputs.result }}').issue_number; From 5a5ea2606f61d79867da0530860c94478855a004 Mon Sep 17 00:00:00 2001 From: MantisClone Date: Fri, 21 Mar 2025 23:32:40 -0400 Subject: [PATCH 2/2] fix: Remove broken manually triggered tests --- .github/workflows/integration-test.yml | 146 +------------------------ README.md | 83 +++++++------- 2 files changed, 48 insertions(+), 181 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 60b8e45..9623e68 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1,36 +1,16 @@ name: Integration Test for PR Automated Comments on: - # Trigger on pull requests to test actual PR comment behavior pull_request: types: [opened, ready_for_review, closed] - # Manual trigger for testing different branches and scenarios - workflow_dispatch: - inputs: - workflow_ref: - description: 'Branch or tag of the auto-comments repo to test (e.g., main, feature-branch, v1)' - required: true - default: 'main' - type: string - test_comment_type: - description: 'Which comment type to test' - required: true - type: choice - options: - - first_pr - - ready_for_review - - merged - jobs: - # For real PRs - call the actual workflow - real-pr-comments: - name: Real PR Comments Test + test-pr-comments: + name: Test PR Comments if: github.event_name == 'pull_request' uses: RequestNetwork/auto-comments/.github/workflows/pr-auto-comments.yml@main with: org_name: "RequestNetwork" - additional_internal_users: "integration-test-bot" first_pr_comment: | # [TEST] Welcome to your first contribution! @@ -56,125 +36,3 @@ jobs: This is a test of the automated comment system. This comment was triggered by the integration test workflow. secrets: token: ${{ secrets.GH_PAT }} - - # For manual testing - simulate PR events - manual-test: - name: Manual Test - if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - name: Create test issue - id: create-issue - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GH_PAT }} - script: | - const commentType = '${{ github.event.inputs.test_comment_type }}'; - const workflowRef = '${{ github.event.inputs.workflow_ref }}'; - - // Create a test issue to track this test run - const issue = await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: `Integration Test: ${commentType} comment from ${workflowRef}`, - body: `This is an automatically created issue to test the PR automated comments workflow. - - Testing branch/tag: \`${workflowRef}\` - Testing comment type: \`${commentType}\` - - The test will post the response comment below.` - }); - - console.log(`Created test issue #${issue.data.number}`); - return { issue_number: issue.data.number }; - - - name: Test First PR Comment - if: github.event.inputs.test_comment_type == 'first_pr' - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GH_PAT }} - script: | - const issueNumber = JSON.parse('${{ steps.create-issue.outputs.result }}').issue_number; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body: `Calling workflow for First PR comment test with ref: ${{ github.event.inputs.workflow_ref }}` - }); - - // Call the workflow - await github.rest.actions.createWorkflowDispatch({ - owner: 'RequestNetwork', - repo: 'auto-comments', - workflow_id: 'pr-auto-comments.yml', - ref: '${{ github.event.inputs.workflow_ref }}', - inputs: { - org_name: 'RequestNetwork', - event_type: 'opened', - is_first_pr: 'true', - issue_number: issueNumber.toString() - } - }); - - - name: Test Ready for Review Comment - if: github.event.inputs.test_comment_type == 'ready_for_review' - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GH_PAT }} - script: | - const issueNumber = JSON.parse('${{ steps.create-issue.outputs.result }}').issue_number; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body: `Calling workflow for Ready for Review comment test with ref: ${{ github.event.inputs.workflow_ref }}` - }); - - // Call the workflow - await github.rest.actions.createWorkflowDispatch({ - owner: 'RequestNetwork', - repo: 'auto-comments', - workflow_id: 'pr-auto-comments.yml', - ref: '${{ github.event.inputs.workflow_ref }}', - inputs: { - org_name: 'RequestNetwork', - event_type: 'ready_for_review', - issue_number: issueNumber.toString() - } - }); - - - name: Test Merged PR Comment - if: github.event.inputs.test_comment_type == 'merged' - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GH_PAT }} - script: | - const issueNumber = JSON.parse('${{ steps.create-issue.outputs.result }}').issue_number; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body: `Calling workflow for Merged PR comment test with ref: ${{ github.event.inputs.workflow_ref }}` - }); - - // Call the workflow - await github.rest.actions.createWorkflowDispatch({ - owner: 'RequestNetwork', - repo: 'auto-comments', - workflow_id: 'pr-auto-comments.yml', - ref: '${{ github.event.inputs.workflow_ref }}', - inputs: { - org_name: 'RequestNetwork', - event_type: 'merged', - issue_number: issueNumber.toString() - } - }); - - - name: Wait for workflow to complete - run: | - echo "Waiting for the workflow to complete..." - sleep 20 - echo "Test completed. Please check the issue #${{ fromJson(steps.create-issue.outputs.result).issue_number }} for results." diff --git a/README.md b/README.md index 4c53671..3784732 100644 --- a/README.md +++ b/README.md @@ -6,59 +6,68 @@ This repository serves as an integration testing environment for the [auto-comme The `auto-comments` repository contains a reusable workflow that automatically posts comments on Pull Requests from external contributors. This test repository helps verify that the workflow functions correctly in real-world scenarios before changes are merged into the main workflow. -## Test Workflows +## Proper Testing Methodology -### 1. Automatic PR Testing +The core functionality of the auto-comments workflow can only be properly tested with real user accounts, as it relies on checking organization membership status. For comprehensive testing, you should: -The repository includes an integration test workflow that runs automatically on Pull Requests: +### Create a Testing Matrix -- It's triggered by real PR events (opened, ready for review, closed/merged) -- It calls the reusable workflow from the main repository -- It uses test-specific comment messages that clearly indicate they're from a test +| Test Case | User Type | PR Action | Expected Result | +|-----------|-----------|-----------|-----------------| +| 1 | Internal (org member) | Open PR | No comments | +| 2 | External (non-member) | Open first PR | First PR comment appears | +| 3 | External (non-member) | Open second PR | No first PR comment | +| 4 | External (non-member) | Mark PR ready for review | Ready for review comment appears | +| 5 | External (non-member) | Merge PR | Merged PR comment appears | -This allows testing the workflow by simply creating PRs in this repository. +### Test with a Real External Account -### 2. Manual Testing +1. Create or use a GitHub account that is **not** a member of the RequestNetwork organization +2. Have this account create a PR to the auto-comments-test repository +3. Check that the first PR comment appears +4. Have the same account create a second PR +5. Verify that the first PR comment doesn't appear this time +6. Mark the PR as ready for review (after creating as draft) +7. Verify the ready for review comment appears +8. Merge the PR +9. Verify the merged PR comment appears -For testing specific branches or features, you can manually trigger the integration test: +### Verify Variable Substitution -1. Go to the Actions tab -2. Select the "Integration Test for PR Automated Comments" workflow -3. Click "Run workflow" -4. Choose: - - Which branch/tag of the auto-comments repo to test - - Which comment type to test (first PR, ready for review, merged) -5. The workflow will: - - Create a test issue - - Call the workflow with the specified parameters - - Show results in the issue comments +In all comments, verify that: +- `{{username}}` correctly shows the PR author's username +- `{{repository}}` correctly shows "auto-comments-test" +- `{{org}}` correctly shows "RequestNetwork" -## Contributing +## How This Repository Works -When making changes to the `auto-comments` workflow: +This repository includes a simplified integration-test.yml workflow that: -1. Create a feature branch in the main repository -2. Test your changes using this repository -3. Once verified, merge your changes to the main branch +1. Is triggered by actual pull request events (opened, ready for review, closed) +2. Calls the reusable workflow from the auto-comments repository +3. Provides test-specific comment messages clearly marked with "[TEST]" -## Testing Strategies +By using real pull requests, we can accurately test the workflow's core functionality: +- Organization membership detection +- First PR detection +- Event-specific comments +- Variable substitution -### Testing a Feature Branch +## Best Practices for Testing Changes -To test changes before merging to main: +1. **Local Validation**: First validate workflow syntax locally +2. **Real User Testing**: Have a non-org member create a PR to verify the full functionality +3. **Review Results**: Check that comments appear correctly and variables are substituted properly -1. Push your changes to a feature branch in the auto-comments repository -2. Run the manual test workflow in this repository -3. Select your feature branch in the "workflow_ref" input -4. Verify the workflow behaves as expected +Remember: The most crucial aspects (first-PR detection, org membership detection) can only be properly verified with actual GitHub accounts that are not members of your organization. -### Testing Edge Cases +## Creating a Test Account -You can test specific scenarios by: - -1. Creating PRs with different characteristics in this repository -2. Transitioning PRs through different states (draft, ready, merged) -3. Using different authors (org members vs. external contributors) +For thorough testing, consider: +1. Creating a dedicated testing GitHub account +2. Keeping it outside your organization +3. Using it exclusively for testing external contributor experiences +4. Documenting test results with screenshots ## License