diff --git a/.github/actions/weekly-report/README.md b/.github/actions/weekly-report/README.md index bf5f92f..a8e4062 100644 --- a/.github/actions/weekly-report/README.md +++ b/.github/actions/weekly-report/README.md @@ -14,6 +14,8 @@ This action generates a report containing: - **Smart repository filtering**: Uses GitHub Search API to identify repositories with recent activity (commits in the last 7 days) before checking for issues and PRs - **Fallback mechanism**: If no repositories are found with recent commits, falls back to checking all organization repositories to ensure complete coverage - **Activity-based reporting**: Only includes repositories with actual activity in the generated report +- **Rate limit handling**: Automatically retries on rate limit errors with exponential backoff, and provides clear warnings when data is incomplete +- **Configurable delays**: Optional delays between API calls to reduce rate limit pressure ## Usage @@ -25,6 +27,7 @@ This action generates a report containing: organization: 'QuantEcon' output-format: 'markdown' exclude-repos: 'lecture-python.notebooks,auto-updated-repo' + api-delay: '1' # Add 1 second delay between API calls to avoid rate limits ``` ## Inputs @@ -35,6 +38,7 @@ This action generates a report containing: | `organization` | GitHub organization name | No | `QuantEcon` | | `output-format` | Output format (`markdown` or `json`) | No | `markdown` | | `exclude-repos` | Comma-separated list of repository names to exclude from the report | No | `''` | +| `api-delay` | Delay in seconds between API calls to avoid rate limits (0 = no delay) | No | `0` | ## Outputs @@ -59,6 +63,16 @@ See the [weekly report workflow](../../workflows/weekly-report.yml) for a comple The generated markdown report includes: - A summary table showing activity by repository - Total counts across all repositories +- Data completeness warnings if API calls failed due to rate limits or other errors - Report metadata (generation date, period covered) -Only repositories with activity in the reporting period are included in the detailed table. \ No newline at end of file +Only repositories with activity in the reporting period are included in the detailed table. + +## Rate Limiting + +GitHub's API has rate limits (5000 requests/hour for authenticated requests). For large organizations: + +- **Monitor warnings**: The report will include warnings when rate limits are hit +- **Add delays**: Use the `api-delay` parameter to add delays between requests (e.g., `api-delay: '1'` for 1 second delays) +- **Run during off-peak**: Schedule reports during off-peak hours to avoid conflicts with other API usage +- **Incomplete data**: When rate limited, the report will show `0` for affected repositories and include a warning \ No newline at end of file diff --git a/.github/actions/weekly-report/action.yml b/.github/actions/weekly-report/action.yml index cbe3e2b..361a4f1 100644 --- a/.github/actions/weekly-report/action.yml +++ b/.github/actions/weekly-report/action.yml @@ -18,6 +18,10 @@ inputs: description: 'Comma-separated list of repository names to exclude from the report' required: false default: '' + api-delay: + description: 'Delay in seconds between API calls to avoid rate limits (0 = no delay)' + required: false + default: '0' outputs: report-content: @@ -35,4 +39,5 @@ runs: INPUT_GITHUB_TOKEN: ${{ inputs.github-token }} INPUT_ORGANIZATION: ${{ inputs.organization }} INPUT_OUTPUT_FORMAT: ${{ inputs.output-format }} - INPUT_EXCLUDE_REPOS: ${{ inputs.exclude-repos }} \ No newline at end of file + INPUT_EXCLUDE_REPOS: ${{ inputs.exclude-repos }} + INPUT_API_DELAY: ${{ inputs.api-delay }} \ No newline at end of file diff --git a/.github/actions/weekly-report/generate-report.sh b/.github/actions/weekly-report/generate-report.sh index b76ba4a..8cd7bb4 100755 --- a/.github/actions/weekly-report/generate-report.sh +++ b/.github/actions/weekly-report/generate-report.sh @@ -1,11 +1,17 @@ #!/bin/bash set -e +echo "DEBUG: Starting weekly report generation" +echo "DEBUG: Environment check - GITHUB_OUTPUT: ${GITHUB_OUTPUT:-NOT_SET}" + # Get inputs GITHUB_TOKEN="${INPUT_GITHUB_TOKEN}" ORGANIZATION="${INPUT_ORGANIZATION:-QuantEcon}" OUTPUT_FORMAT="${INPUT_OUTPUT_FORMAT:-markdown}" EXCLUDE_REPOS="${INPUT_EXCLUDE_REPOS:-}" +API_DELAY="${INPUT_API_DELAY:-0}" # Optional delay between API calls in seconds + +echo "DEBUG: Inputs - ORG: $ORGANIZATION, FORMAT: $OUTPUT_FORMAT, EXCLUDE: $EXCLUDE_REPOS" # Date calculations for last week WEEK_AGO=$(date -d "7 days ago" -u +"%Y-%m-%dT%H:%M:%SZ") @@ -14,13 +20,72 @@ NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") echo "Generating weekly report for ${ORGANIZATION} organization" echo "Period: ${WEEK_AGO} to ${NOW}" -# Function to make GitHub API calls +# Function to make GitHub API calls with rate limit handling api_call() { local endpoint="$1" local page="${2:-1}" - curl -s -H "Authorization: token ${GITHUB_TOKEN}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com${endpoint}?page=${page}&per_page=100" + local max_retries=3 + local retry_count=0 + local delay="${API_DELAY:-0}" + + # Add delay between requests if specified + if [ "$delay" -gt 0 ]; then + sleep "$delay" + fi + + while [ $retry_count -lt $max_retries ]; do + # Construct URL with proper query parameter handling + local url="https://api.github.com${endpoint}" + if [[ "$endpoint" == *"?"* ]]; then + url="${url}&page=${page}&per_page=100" + else + url="${url}?page=${page}&per_page=100" + fi + + local response=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github.v3+json" \ + "$url") + + local http_code=$(echo "$response" | tail -n1) + local body=$(echo "$response" | head -n -1) + + case "$http_code" in + 200) + echo "$body" + return 0 + ;; + 403) + # Check if it's a rate limit error + if echo "$body" | jq -e '.message' 2>/dev/null | grep -q "rate limit"; then + retry_count=$((retry_count + 1)) + if [ $retry_count -lt $max_retries ]; then + local wait_time=$((retry_count * retry_count * 60)) # Exponential backoff: 1min, 4min, 9min + echo "Rate limit exceeded for $endpoint. Waiting ${wait_time}s before retry $retry_count/$max_retries..." >&2 + sleep "$wait_time" + continue + else + echo "Rate limit exceeded for $endpoint after $max_retries retries. Data will be incomplete." >&2 + echo '{"error": "rate_limit_exceeded", "message": "API rate limit exceeded"}' + return 1 + fi + else + echo "Access forbidden for $endpoint: $body" >&2 + echo '{"error": "forbidden", "message": "Access forbidden"}' + return 1 + fi + ;; + 404) + echo "Repository not found: $endpoint" >&2 + echo '{"error": "not_found", "message": "Repository not found"}' + return 1 + ;; + *) + echo "API call failed for $endpoint with status $http_code: $body" >&2 + echo '{"error": "api_error", "message": "API call failed"}' + return 1 + ;; + esac + done } # Get repositories with recent activity using GitHub Search API @@ -83,80 +148,179 @@ if [ -n "$EXCLUDE_REPOS" ]; then fi # Initialize report variables +total_current_issues=0 total_opened_issues=0 total_closed_issues=0 total_merged_prs=0 +failed_repos=0 +rate_limited_repos=0 report_content="" # Start building the report if [ "$OUTPUT_FORMAT" = "markdown" ]; then - report_content="# QuantEcon Weekly Report\n\n" - report_content+="**Report Period:** $(date -d "$WEEK_AGO" '+%B %d, %Y') - $(date -d "$NOW" '+%B %d, %Y')\n\n" - report_content+="## Summary\n\n" - report_content+="| Repository | Opened Issues | Closed Issues | Merged PRs |\n" - report_content+="|------------|---------------|---------------|------------|\n" + report_content="# QuantEcon Weekly Report + +**Report Period:** $(date -d "$WEEK_AGO" '+%B %d, %Y') - $(date -d "$NOW" '+%B %d, %Y') + +## Summary + +| Repository | Total Current Issues | Opened Issues | Closed Issues | Merged PRs | +|------------|---------------------|---------------|---------------|------------|" + echo "DEBUG: Initial report content set, length: ${#report_content}" fi # Process each repository +repo_count=0 while IFS= read -r repo; do [ -z "$repo" ] && continue + repo_count=$((repo_count + 1)) echo "Processing repository: $repo" + # Count total current open issues + current_issues_response=$(api_call "/repos/${ORGANIZATION}/${repo}/issues?state=open") + if [ $? -eq 0 ]; then + current_issues=$(echo "$current_issues_response" | jq 'if type == "array" then [.[] | select(.pull_request == null)] | length else 0 end') + else + current_issues=0 + if echo "$current_issues_response" | jq -e '.error' 2>/dev/null | grep -q "rate_limit"; then + rate_limited_repos=$((rate_limited_repos + 1)) + else + failed_repos=$((failed_repos + 1)) + fi + fi + # Count opened issues in the last week - opened_issues=$(api_call "/repos/${ORGANIZATION}/${repo}/issues" | \ - jq --arg since "$WEEK_AGO" '[.[] | select(.created_at >= $since and .pull_request == null)] | length') + opened_response=$(api_call "/repos/${ORGANIZATION}/${repo}/issues") + if [ $? -eq 0 ]; then + opened_issues=$(echo "$opened_response" | jq --arg since "$WEEK_AGO" 'if type == "array" then [.[] | select(.created_at >= $since and .pull_request == null)] | length else 0 end') + else + opened_issues=0 + if echo "$opened_response" | jq -e '.error' 2>/dev/null | grep -q "rate_limit"; then + rate_limited_repos=$((rate_limited_repos + 1)) + else + failed_repos=$((failed_repos + 1)) + fi + fi # Count closed issues in the last week - closed_issues=$(api_call "/repos/${ORGANIZATION}/${repo}/issues?state=closed" | \ - jq --arg since "$WEEK_AGO" '[.[] | select(.closed_at >= $since and .pull_request == null)] | length') + closed_response=$(api_call "/repos/${ORGANIZATION}/${repo}/issues?state=closed") + if [ $? -eq 0 ]; then + closed_issues=$(echo "$closed_response" | jq --arg since "$WEEK_AGO" 'if type == "array" then [.[] | select(.closed_at != null and .closed_at >= $since and .pull_request == null)] | length else 0 end') + else + closed_issues=0 + if echo "$closed_response" | jq -e '.error' 2>/dev/null | grep -q "rate_limit"; then + rate_limited_repos=$((rate_limited_repos + 1)) + else + failed_repos=$((failed_repos + 1)) + fi + fi # Count merged PRs in the last week - merged_prs=$(api_call "/repos/${ORGANIZATION}/${repo}/pulls?state=closed" | \ - jq --arg since "$WEEK_AGO" '[.[] | select(.merged_at != null and .merged_at >= $since)] | length') + prs_response=$(api_call "/repos/${ORGANIZATION}/${repo}/pulls?state=closed") + if [ $? -eq 0 ]; then + merged_prs=$(echo "$prs_response" | jq --arg since "$WEEK_AGO" 'if type == "array" then [.[] | select(.merged_at != null and .merged_at >= $since)] | length else 0 end') + else + merged_prs=0 + if echo "$prs_response" | jq -e '.error' 2>/dev/null | grep -q "rate_limit"; then + rate_limited_repos=$((rate_limited_repos + 1)) + else + failed_repos=$((failed_repos + 1)) + fi + fi # Handle null/empty values + current_issues=${current_issues:-0} opened_issues=${opened_issues:-0} closed_issues=${closed_issues:-0} merged_prs=${merged_prs:-0} # Add to totals + total_current_issues=$((total_current_issues + current_issues)) total_opened_issues=$((total_opened_issues + opened_issues)) total_closed_issues=$((total_closed_issues + closed_issues)) total_merged_prs=$((total_merged_prs + merged_prs)) - # Add to report if there's activity - if [ $((opened_issues + closed_issues + merged_prs)) -gt 0 ]; then + # Add to report if there's activity or current open issues + if [ $((current_issues + opened_issues + closed_issues + merged_prs)) -gt 0 ]; then if [ "$OUTPUT_FORMAT" = "markdown" ]; then - report_content+="| $repo | $opened_issues | $closed_issues | $merged_prs |\n" + report_content="${report_content} +| $repo | $current_issues | $opened_issues | $closed_issues | $merged_prs |" fi fi done <<< "$repo_names" +echo "DEBUG: Processed $repo_count repositories" +echo "DEBUG: Final report content length: ${#report_content}" + # Add summary to report if [ "$OUTPUT_FORMAT" = "markdown" ]; then - report_content+="|**Total**|**$total_opened_issues**|**$total_closed_issues**|**$total_merged_prs**|\n\n" - report_content+="## Details\n\n" - report_content+="- **Total Repositories Checked:** $(echo "$repo_names" | wc -l)\n" - report_content+="- **Total Issues Opened:** $total_opened_issues\n" - report_content+="- **Total Issues Closed:** $total_closed_issues\n" - report_content+="- **Total PRs Merged:** $total_merged_prs\n\n" - report_content+="*Report generated on $(date) by QuantEcon Weekly Report Action*\n" + report_content="${report_content} +|**Total**|**$total_current_issues**|**$total_opened_issues**|**$total_closed_issues**|**$total_merged_prs**| + +## Details + +- **Total Repositories Checked:** $(echo "$repo_names" | wc -l) +- **Total Current Open Issues:** $total_current_issues +- **Total Issues Opened:** $total_opened_issues +- **Total Issues Closed:** $total_closed_issues +- **Total PRs Merged:** $total_merged_prs" + + # Add warnings about incomplete data if any API calls failed + if [ $rate_limited_repos -gt 0 ] || [ $failed_repos -gt 0 ]; then + report_content="${report_content} + +### ⚠️ Data Completeness Warnings +" + if [ $rate_limited_repos -gt 0 ]; then + report_content="${report_content} +- **Rate Limited:** $rate_limited_repos API calls hit rate limits. Data may be incomplete." + fi + if [ $failed_repos -gt 0 ]; then + report_content="${report_content} +- **Failed Requests:** $failed_repos API calls failed. Data may be incomplete." + fi + report_content="${report_content} + +*Consider adding API delays or running during off-peak hours to avoid rate limits.*" + fi + + report_content="${report_content} + +*Report generated on $(date) by QuantEcon Weekly Report Action*" fi # Create summary -summary="Week Summary: $total_opened_issues issues opened, $total_closed_issues issues closed, $total_merged_prs PRs merged" +summary="Week Summary: $total_current_issues current open issues, $total_opened_issues issues opened, $total_closed_issues issues closed, $total_merged_prs PRs merged" # Save report to file -echo -e "$report_content" > weekly-report.md +echo "$report_content" > weekly-report.md -# Set outputs -echo "report-content<> $GITHUB_OUTPUT -echo -e "$report_content" >> $GITHUB_OUTPUT -echo "EOF" >> $GITHUB_OUTPUT +# Debug: Check if GITHUB_OUTPUT is set and accessible +echo "DEBUG: GITHUB_OUTPUT environment variable: ${GITHUB_OUTPUT:-NOT_SET}" +echo "DEBUG: Report content length: ${#report_content}" +echo "DEBUG: Summary: $summary" -echo "report-summary=$summary" >> $GITHUB_OUTPUT +# Set outputs +if [ -n "$GITHUB_OUTPUT" ]; then + echo "DEBUG: Writing to GITHUB_OUTPUT file" + echo "DEBUG: Content preview (first 100 chars): ${report_content:0:100}" + echo "DEBUG: Summary preview: $summary" + + # Use a unique delimiter to avoid conflicts with content + delimiter="QUANTECON_REPORT_END_$(date +%s)" + echo "report-content<<${delimiter}" >> "$GITHUB_OUTPUT" + echo "$report_content" >> "$GITHUB_OUTPUT" + echo "${delimiter}" >> "$GITHUB_OUTPUT" + + echo "report-summary=$summary" >> "$GITHUB_OUTPUT" + + echo "DEBUG: Outputs written to GITHUB_OUTPUT" + echo "DEBUG: GITHUB_OUTPUT file size: $(wc -c < "$GITHUB_OUTPUT")" +else + echo "ERROR: GITHUB_OUTPUT environment variable not set!" +fi echo "Weekly report generated successfully!" echo "Summary: $summary" \ No newline at end of file diff --git a/.github/workflows/weekly-report.yml b/.github/workflows/weekly-report.yml index 3e2fc15..335276c 100644 --- a/.github/workflows/weekly-report.yml +++ b/.github/workflows/weekly-report.yml @@ -33,8 +33,17 @@ jobs: uses: actions/github-script@v7 with: script: | - const reportContent = `${{ steps.report.outputs.report-content }}`; - const summary = `${{ steps.report.outputs.report-summary }}`; + const fs = require('fs'); + + // Read the report content directly from the generated file + let reportContent = ''; + try { + reportContent = fs.readFileSync('weekly-report.md', 'utf8'); + console.log(`Successfully read report file, length: ${reportContent.length}`); + } catch (error) { + console.error('Failed to read weekly-report.md:', error); + reportContent = 'Error: Unable to load weekly report content.'; + } // Create issue title with current date const now = new Date(); @@ -55,7 +64,7 @@ jobs: }); console.log(`Created issue: ${title}`); - console.log(`Summary: ${summary}`); + console.log(`Report content length: ${reportContent.length}`); - name: Upload report as artifact uses: actions/upload-artifact@v4