From 3d1755af96a3225032a1c22b0644147ff41cb171 Mon Sep 17 00:00:00 2001 From: Jameson Grieve <37882431+JamesonRGrieve@users.noreply.github.com> Date: Wed, 4 Jun 2025 11:24:59 -0700 Subject: [PATCH 01/44] Update test-pytest.yml --- .github/workflows/test-pytest.yml | 735 ++++-------------------------- 1 file changed, 97 insertions(+), 638 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index 0a9bad5..33c5bd4 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -64,7 +64,7 @@ jobs: lint: uses: ./.github/workflows/test-lint-py.yml permissions: - contents: write # Lint job might push changes + contents: write test-source-branch: needs: lint @@ -104,78 +104,46 @@ jobs: - name: Check for test collection errors id: check-collection run: | - echo "Running pytest collection check..." - - # Redirect ALL output to files, including subprocess stderr - python -m pytest --collect-only -v > collection_output.txt 2>&1 || true - - # Save to debug file for artifact - echo "=== COLLECTION CHECK OUTPUT ===" >> debug_collection.log - cat collection_output.txt >> debug_collection.log - - # Set default values HAS_COLLECTION_ERRORS="false" NO_TESTS_FOUND="false" ERROR_TYPE="none" ERROR_DETAILS="none" - if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt; then - echo "::error::Test discovery errors detected in PR branch - Python modules could not be imported correctly" - - # Attempt to identify specific error type - if grep -q "ImportError" collection_output.txt; then - ERROR_TYPE="ImportError" - elif grep -q "ModuleNotFoundError" collection_output.txt; then - ERROR_TYPE="ModuleNotFoundError" - elif grep -q "SyntaxError" collection_output.txt; then - ERROR_TYPE="SyntaxError" - elif grep -q "ERROR collecting" collection_output.txt; then - ERROR_TYPE="CollectionError" - elif grep -q "Interrupted:" collection_output.txt; then - ERROR_TYPE="Interrupted" - else - ERROR_TYPE="UnknownError" - fi - - echo "PR branch discovery error type: $ERROR_TYPE" - - ERROR_FILE=$(grep -o "ERROR collecting.*\.py" collection_output.txt | grep -o "[a-zA-Z0-9_/]*\.py" || echo "Unknown file") - - if [[ "$ERROR_FILE" != "Unknown file" ]]; then - echo "Error in file $ERROR_FILE" - grep -A 15 "$ERROR_FILE" collection_output.txt > error_details.txt - # Truncate to 200 chars for GitHub output, full details go to artifact - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g' | cut -c1-200) - else - grep -A 15 "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt | head -20 > error_details.txt - # Truncate to 200 chars for GitHub output, full details go to artifact - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g' | cut -c1-200) + python -m pytest --collect-only --quiet --tb=no >/dev/null 2>collection_check.txt + COLLECTION_EXIT_CODE=$? + + if [ $COLLECTION_EXIT_CODE -ne 0 ]; then + if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting" collection_check.txt; then + echo "::error::Test discovery errors detected in PR branch" + HAS_COLLECTION_ERRORS="true" + + if grep -q "ImportError" collection_check.txt; then + ERROR_TYPE="ImportError" + elif grep -q "ModuleNotFoundError" collection_check.txt; then + ERROR_TYPE="ModuleNotFoundError" + elif grep -q "SyntaxError" collection_check.txt; then + ERROR_TYPE="SyntaxError" + else + ERROR_TYPE="CollectionError" + fi + + ERROR_DETAILS=$(head -1 collection_check.txt | tr '\n' ' ' | cut -c1-200) fi - - echo "::error::PR branch discovery error details (truncated): ${ERROR_DETAILS}..." - HAS_COLLECTION_ERRORS="true" else - echo "No discovery errors detected in PR branch" - - TEST_COUNT=$(grep -o "collected [0-9]* item" collection_output.txt | grep -o "[0-9]*" || echo "0") - + TEST_COUNT=$(python -m pytest --collect-only --quiet 2>/dev/null | grep -o "collected [0-9]* item" | grep -o "[0-9]*" || echo "0") if [[ "$TEST_COUNT" == "0" ]]; then echo "::warning::No tests were found in the PR branch" NO_TESTS_FOUND="true" ERROR_TYPE="NoTestsFound" - ERROR_DETAILS="No test files discovered matching pytest pattern" - else - echo "Found $TEST_COUNT tests in PR branch" + ERROR_DETAILS="No test files discovered" fi fi - # Set all the outputs with size limits echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT - - # For backward compatibility + if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then echo "has_errors=true" >> $GITHUB_OUTPUT else @@ -185,55 +153,39 @@ jobs: - name: Run tests on PR Branch if: steps.check-collection.outputs.has_collection_errors != 'true' env: - PYTHONUNBUFFERED: "1" - PYTHONIOENCODING: "utf-8" + PYTHONUNBUFFERED: "0" PYTHONWARNINGS: "ignore" - MIGRATION_MANAGER_LOG_LEVEL: "ERROR" - MIGRATION_MANAGER_QUIET: "true" - SQLALCHEMY_WARN_20: "false" - SQLALCHEMY_SILENCE_UBER_WARNING: "1" PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" PYTEST_CURRENT_TEST: "" + COLUMNS: "80" run: | - echo "Running tests on PR branch..." - - # Run pytest with aggressive output suppression - redirect everything to log file - exec 3>&1 4>&2 - exec 1>/dev/null 2>/dev/null - - python -m pytest --json-report --json-report-file=pr_results.json --tb=no -q --disable-warnings --log-level=CRITICAL --log-cli-level=CRITICAL 2>&1 | tee test_run_output.log >/dev/null || true - - # Restore output for final messages - exec 1>&3 2>&4 - - # Save output to debug file for artifact - echo "=== PYTEST RUN OUTPUT ===" >> debug_test_run.log - if [ -f test_run_output.log ]; then - cat test_run_output.log >> debug_test_run.log - fi + python -m pytest \ + --json-report \ + --json-report-file=pr_results.json \ + --tb=no \ + --no-header \ + --no-summary \ + -q \ + --disable-warnings \ + --log-level=CRITICAL \ + --log-cli-level=CRITICAL \ + --capture=sys \ + >/dev/null 2>&1 || true if [ -f pr_results.json ]; then - echo "Test results file created successfully" + echo "Test results generated successfully" else - echo "::error::Failed to create test results file for PR branch" + echo "::error::Failed to create test results file" fi - name: Extract test results and create artifacts id: extract-results run: | - echo "PR_BRANCH=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV - echo "Processing test results for PR branch: $PR_BRANCH" - - # Create debug file for detailed output - exec 3>&1 4>&2 - exec 1> >(tee -a debug_extract_results.log) 2>&1 - python -c " import json import sys import os - # Default values in case file doesn't exist or is invalid pr_total = 0 pr_passed = 0 pr_percentage = 0 @@ -248,42 +200,34 @@ jobs: with open('pr_results.json') as f: pr_results = json.load(f) - # Check for collection errors by looking at exitcode or error patterns if pr_results.get('exitcode', 0) > 1: - pr_total = 0 # Explicitly set to 0 - no tests run when collection fails + pr_total = 0 pr_passed = 0 elif 'summary' in pr_results and isinstance(pr_results['summary'], dict): - # Normal case - extract data from summary summary = pr_results['summary'] pr_total = summary.get('total', 0) pr_passed = summary.get('passed', 0) - # Extract all tests by outcome and collect all test nodeids with reasons if 'tests' in pr_results: for test in pr_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') if nodeid: - all_tests.append(nodeid) # Track all tests regardless of outcome + all_tests.append(nodeid) if outcome in ['failed', 'error']: failing_tests.append(nodeid) elif outcome == 'skipped': skipped_tests.append(nodeid) - # Extract skip reason skip_reason = 'No reason provided' if 'longrepr' in test and test['longrepr']: - # longrepr can be a string or list, handle both longrepr = test['longrepr'] if isinstance(longrepr, list) and longrepr: skip_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' elif isinstance(longrepr, str): skip_reason = longrepr - elif 'call' in test and test['call'] and 'longrepr' in test['call']: - skip_reason = str(test['call']['longrepr']) skipped_tests_with_reasons[nodeid] = skip_reason.strip() elif outcome == 'xfailed': xfailed_tests.append(nodeid) - # Extract xfail reason xfail_reason = 'No reason provided' if 'longrepr' in test and test['longrepr']: longrepr = test['longrepr'] @@ -291,17 +235,13 @@ jobs: xfail_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' elif isinstance(longrepr, str): xfail_reason = longrepr - elif 'call' in test and test['call'] and 'longrepr' in test['call']: - xfail_reason = str(test['call']['longrepr']) xfailed_tests_with_reasons[nodeid] = xfail_reason.strip() - # Calculate percentage safely pr_percentage = (pr_passed / pr_total * 100) if pr_total > 0 else 0 except Exception as e: - pass # Use defaults on any error + pass - # Set scalar outputs only (no large arrays) with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'total={pr_total}\\n') f.write(f'passed={pr_passed}\\n') @@ -310,7 +250,6 @@ jobs: f.write(f'skipped_count={len(skipped_tests)}\\n') f.write(f'xfailed_count={len(xfailed_tests)}\\n') - # Save test lists to artifact files instead of job outputs test_data = { 'failing_tests': failing_tests, 'skipped_tests': skipped_tests, @@ -324,11 +263,6 @@ jobs: json.dump(test_data, f, indent=2) " - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "PR branch test results processed" - - name: Upload PR branch artifacts if: always() uses: actions/upload-artifact@v4 @@ -336,11 +270,8 @@ jobs: name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | pr_test_data.json - debug_*.log - test_run_output.log - collection_output.txt - error_details.txt pr_results.json + collection_check.txt retention-days: 3 if-no-files-found: ignore @@ -381,78 +312,46 @@ jobs: - name: Check for test collection errors id: check-collection run: | - echo "Running pytest collection check..." - - # Redirect ALL output to files, including subprocess stderr - python -m pytest --collect-only -v > collection_output.txt 2>&1 || true - - # Save to debug file for artifact - echo "=== COLLECTION CHECK OUTPUT ===" >> debug_target_collection.log - cat collection_output.txt >> debug_target_collection.log - - # Set default values HAS_COLLECTION_ERRORS="false" NO_TESTS_FOUND="false" ERROR_TYPE="none" ERROR_DETAILS="none" - if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt; then - echo "::warning::Test discovery errors detected in target branch - Python modules could not be imported correctly" - - # Attempt to identify specific error type - if grep -q "ImportError" collection_output.txt; then - ERROR_TYPE="ImportError" - elif grep -q "ModuleNotFoundError" collection_output.txt; then - ERROR_TYPE="ModuleNotFoundError" - elif grep -q "SyntaxError" collection_output.txt; then - ERROR_TYPE="SyntaxError" - elif grep -q "ERROR collecting" collection_output.txt; then - ERROR_TYPE="CollectionError" - elif grep -q "Interrupted:" collection_output.txt; then - ERROR_TYPE="Interrupted" - else - ERROR_TYPE="UnknownError" - fi - - echo "Target branch discovery error type: $ERROR_TYPE" - - ERROR_FILE=$(grep -o "ERROR collecting.*\.py" collection_output.txt | grep -o "[a-zA-Z0-9_/]*\.py" || echo "Unknown file") - - if [[ "$ERROR_FILE" != "Unknown file" ]]; then - echo "Error in file $ERROR_FILE" - grep -A 15 "$ERROR_FILE" collection_output.txt > error_details.txt - # Truncate to 200 chars for GitHub output, full details go to artifact - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g' | cut -c1-200) - else - grep -A 15 "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt | head -20 > error_details.txt - # Truncate to 200 chars for GitHub output, full details go to artifact - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g' | cut -c1-200) + python -m pytest --collect-only --quiet --tb=no >/dev/null 2>collection_check.txt + COLLECTION_EXIT_CODE=$? + + if [ $COLLECTION_EXIT_CODE -ne 0 ]; then + if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting" collection_check.txt; then + echo "::warning::Test discovery errors detected in target branch" + HAS_COLLECTION_ERRORS="true" + + if grep -q "ImportError" collection_check.txt; then + ERROR_TYPE="ImportError" + elif grep -q "ModuleNotFoundError" collection_check.txt; then + ERROR_TYPE="ModuleNotFoundError" + elif grep -q "SyntaxError" collection_check.txt; then + ERROR_TYPE="SyntaxError" + else + ERROR_TYPE="CollectionError" + fi + + ERROR_DETAILS=$(head -1 collection_check.txt | tr '\n' ' ' | cut -c1-200) fi - - echo "::warning::Target branch discovery error details (truncated): ${ERROR_DETAILS}..." - HAS_COLLECTION_ERRORS="true" else - echo "No discovery errors detected in target branch" - - TEST_COUNT=$(grep -o "collected [0-9]* item" collection_output.txt | grep -o "[0-9]*" || echo "0") - + TEST_COUNT=$(python -m pytest --collect-only --quiet 2>/dev/null | grep -o "collected [0-9]* item" | grep -o "[0-9]*" || echo "0") if [[ "$TEST_COUNT" == "0" ]]; then echo "::warning::No tests were found in the target branch" NO_TESTS_FOUND="true" ERROR_TYPE="NoTestsFound" - ERROR_DETAILS="No test files discovered matching pytest pattern" - else - echo "Found $TEST_COUNT tests in target branch" + ERROR_DETAILS="No test files discovered" fi fi - # Set all the outputs with size limits echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT - - # For backward compatibility + if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then echo "has_errors=true" >> $GITHUB_OUTPUT else @@ -462,56 +361,40 @@ jobs: - name: Run tests on target branch if: steps.check-collection.outputs.has_collection_errors != 'true' env: - PYTHONUNBUFFERED: "1" - PYTHONIOENCODING: "utf-8" + PYTHONUNBUFFERED: "0" PYTHONWARNINGS: "ignore" - MIGRATION_MANAGER_LOG_LEVEL: "ERROR" - MIGRATION_MANAGER_QUIET: "true" - SQLALCHEMY_WARN_20: "false" - SQLALCHEMY_SILENCE_UBER_WARNING: "1" PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" PYTEST_CURRENT_TEST: "" + COLUMNS: "80" run: | - echo "Running tests on target branch..." - - # Run pytest with aggressive output suppression - redirect everything to log file - exec 3>&1 4>&2 - exec 1>/dev/null 2>/dev/null - - python -m pytest --json-report --json-report-file=target_results.json --tb=no -q --disable-warnings --log-level=CRITICAL --log-cli-level=CRITICAL 2>&1 | tee target_test_run_output.log >/dev/null || true - - # Restore output for final messages - exec 1>&3 2>&4 - - # Save output to debug file for artifact - echo "=== PYTEST RUN OUTPUT ===" >> debug_target_test_run.log - if [ -f target_test_run_output.log ]; then - cat target_test_run_output.log >> debug_target_test_run.log - fi + python -m pytest \ + --json-report \ + --json-report-file=target_results.json \ + --tb=no \ + --no-header \ + --no-summary \ + -q \ + --disable-warnings \ + --log-level=CRITICAL \ + --log-cli-level=CRITICAL \ + --capture=sys \ + >/dev/null 2>&1 || true if [ -f target_results.json ]; then - echo "Test results file created successfully" + echo "Test results generated successfully" else - echo "::warning::Failed to create test results file for target branch" + echo "::warning::Failed to create test results file" fi - name: Extract test results and create artifacts id: extract-results - # Only run if there were no collection errors if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - echo "Processing test results for target branch: ${{ inputs.target_branch_to_compare }}" - - # Create debug file for detailed output - exec 3>&1 4>&2 - exec 1> >(tee -a debug_target_extract_results.log) 2>&1 - python -c " import json import sys import os - # Default values in case file doesn't exist or is invalid target_total = 0 target_passed = 0 target_percentage = 0 @@ -522,40 +405,34 @@ jobs: with open('target_results.json') as f: target_results = json.load(f) - # Check for collection errors by looking at exitcode or error patterns if target_results.get('exitcode', 0) > 1: - target_total = 0 # Explicitly set to 0 - no tests run when collection fails + target_total = 0 target_passed = 0 elif 'summary' in target_results and isinstance(target_results['summary'], dict): - # Normal case - extract data from summary summary = target_results['summary'] target_total = summary.get('total', 0) target_passed = summary.get('passed', 0) - # Extract passing tests and all tests if 'tests' in target_results: for test in target_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') if nodeid: - all_tests.append(nodeid) # Track all tests regardless of outcome + all_tests.append(nodeid) if outcome == 'passed': passing_tests.append(nodeid) - # Calculate percentage safely target_percentage = (target_passed / target_total * 100) if target_total > 0 else 0 except Exception as e: - pass # Use defaults on any error + pass - # Set scalar outputs only (no large arrays) with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'total={target_total}\\n') f.write(f'passed={target_passed}\\n') f.write(f'percentage={target_percentage:.2f}\\n') f.write(f'passing_count={len(passing_tests)}\\n') - # Save test lists to artifact files instead of job outputs test_data = { 'passing_tests': passing_tests, 'all_tests': all_tests @@ -565,11 +442,6 @@ jobs: json.dump(test_data, f, indent=2) " - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Target branch test results processed" - - name: Upload target branch artifacts if: always() uses: actions/upload-artifact@v4 @@ -577,20 +449,15 @@ jobs: name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | target_test_data.json - debug_target_*.log - target_test_run_output.log - collection_output.txt - error_details.txt target_results.json + collection_check.txt retention-days: 3 if-no-files-found: ignore - # Add a step to set default outputs when collection errors are detected - name: Set collection error outputs id: set-error-outputs if: steps.check-collection.outputs.has_collection_errors == 'true' run: | - echo "::warning::Setting default outputs for target branch due to collection errors" echo "total=0" >> $GITHUB_OUTPUT echo "passed=0" >> $GITHUB_OUTPUT echo "percentage=0.00" >> $GITHUB_OUTPUT @@ -618,120 +485,62 @@ jobs: - name: Check for collection errors run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_comparison_analysis.log) 2>&1 - - echo "Retrieving collection error status information" PR_COLLECTION_ERRORS="${{ needs.test-source-branch.outputs.collection_errors }}" PR_NO_TESTS="${{ needs.test-source-branch.outputs.no_tests_found }}" PR_ERROR_TYPE="${{ needs.test-source-branch.outputs.error_type }}" - PR_ERROR_DETAILS="${{ needs.test-source-branch.outputs.error_details }}" TARGET_COLLECTION_ERRORS="${{ needs.test-target-branch.outputs.collection_errors }}" - echo "PR branch collection errors: $PR_COLLECTION_ERRORS" - echo "PR branch no tests found: $PR_NO_TESTS" - echo "PR branch error type: $PR_ERROR_TYPE" - echo "Target branch collection errors: $TARGET_COLLECTION_ERRORS" - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - # Distinct error handling for PR branch if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then echo "::error::Test discovery errors in PR branch: $PR_ERROR_TYPE" - echo "::error::$PR_ERROR_DETAILS" - echo "❌ PR branch has test discovery errors. Python modules could not be imported correctly." exit 1 fi if [[ "$PR_NO_TESTS" == "true" ]]; then echo "::error::No tests were found in the PR branch" - echo "❌ PR branch has no tests detected. Please add test files that match pytest's discovery pattern." exit 1 fi - # Warning for target branch issues (not a failure) if [[ "$TARGET_COLLECTION_ERRORS" == "true" ]]; then - echo "⚠️ Target branch has test discovery errors. Tests will still be compared but results may not be accurate." - fi - - if [[ "${{ needs.test-target-branch.outputs.no_tests_found }}" == "true" ]]; then - echo "⚠️ Target branch has no tests detected. PR branch tests will still be evaluated." + echo "⚠️ Target branch has test discovery errors." fi - name: Run regression analysis from artifacts run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_regression_analysis.log) 2>&1 - - echo "Running regression analysis from artifact data..." - python3 - << 'EOF' import json import os + import glob try: - # Load test data from artifacts target_data = {} pr_data = {} - # Try to load target branch data - target_artifact_path = './artifacts/target_branch_data_*/target_test_data.json' - import glob - target_files = glob.glob(target_artifact_path) + target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') if target_files: with open(target_files[0], 'r') as f: target_data = json.load(f) - print(f"Loaded target data from {target_files[0]}") - else: - print("No target branch test data found") - # Try to load PR branch data - pr_artifact_path = './artifacts/pr_branch_data_*/pr_test_data.json' - pr_files = glob.glob(pr_artifact_path) + pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') if pr_files: with open(pr_files[0], 'r') as f: pr_data = json.load(f) - print(f"Loaded PR data from {pr_files[0]}") - else: - print("No PR branch test data found") - # Extract test arrays target_passing = target_data.get('passing_tests', []) pr_failing = pr_data.get('failing_tests', []) - print(f"Parsed {len(target_passing)} passing tests from target branch") - print(f"Parsed {len(pr_failing)} failing tests from PR branch") - - # Find regressions using set operations target_passing_set = set(target_passing) pr_failing_set = set(pr_failing) regression_tests = list(target_passing_set.intersection(pr_failing_set)) - # Write results to file if there are regressions if regression_tests: - print(f"Found {len(regression_tests)} regression(s)!") - with open("regression_details.txt", "w") as f: f.write(f"Found {len(regression_tests)} tests that were passing in target branch but now failing in PR branch:\\n\\n") for idx, test in enumerate(sorted(regression_tests), 1): f.write(f"{idx}. {test}\\n") - print("Regression details written to file") - else: - print("No regressions found") except Exception as e: - print(f"Error in regression analysis: {e}") - import traceback - print(traceback.format_exc()) + pass EOF - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Regression analysis completed" - - name: Check for regression details file id: check-regressions run: | @@ -739,39 +548,17 @@ jobs: _regression_count="0" if [ -f "regression_details.txt" ]; then - echo "Regression details file exists" - # Count regression lines (lines starting with a number and period) _current_count=$(grep -c "^[0-9]\+\." regression_details.txt || echo "0") - echo "Found $_current_count regression items in file" - if [ "$_current_count" -gt 0 ]; then _has_regressions="true" _regression_count="$_current_count" - echo "::error::Test Regressions Found: $_regression_count test(s) that were passing in target branch are now **failing** in PR branch." - echo "Regression details:" - cat regression_details.txt - else - # File exists but no regressions counted (e.g. empty or malformed) - _has_regressions="false" - _regression_count="0" + echo "::error::Test Regressions Found: $_regression_count test(s) that were passing in target branch are now failing in PR branch." fi - else - echo "No regression details file found - no regressions detected" - _has_regressions="false" - _regression_count="0" fi echo "HAS_REGRESSIONS=$_has_regressions" >> $GITHUB_OUTPUT echo "REGRESSION_COUNT=$_regression_count" >> $GITHUB_OUTPUT - if [[ "$_has_regressions" == "false" ]]; then - if [ -f regression_details.txt ] && [ "$_has_regressions" == "false" ]; then - echo "::notice::Regression details file (regression_details.txt) was found but no valid regression entries were counted by this step, or the file was empty." - else - echo "No test regressions detected by this step." - fi - fi - - name: Upload regression details artifact if: steps.check-regressions.outputs.HAS_REGRESSIONS == 'true' && steps.check-regressions.outputs.REGRESSION_COUNT > 0 uses: actions/upload-artifact@v4 @@ -780,188 +567,6 @@ jobs: path: regression_details.txt retention-days: 1 - - name: Check for skip/xfail regressions from artifacts - run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_skip_xfail_analysis.log) 2>&1 - - echo "Checking for tests that were passing in target but are now skipped or xfailed in PR" - - python3 - << 'EOF' - import json - import os - import glob - - try: - # Load test data from artifacts - target_data = {} - pr_data = {} - - # Load target and PR data - target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') - if target_files: - with open(target_files[0], 'r') as f: - target_data = json.load(f) - - pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') - if pr_files: - with open(pr_files[0], 'r') as f: - pr_data = json.load(f) - - # Extract test arrays - target_passing = target_data.get('passing_tests', []) - pr_skipped = pr_data.get('skipped_tests', []) - pr_xfailed = pr_data.get('xfailed_tests', []) - pr_skipped_with_reasons = pr_data.get('skipped_tests_with_reasons', {}) - pr_xfailed_with_reasons = pr_data.get('xfailed_tests_with_reasons', {}) - - print(f"Parsed {len(target_passing)} passing tests from target branch") - print(f"Parsed {len(pr_skipped)} skipped tests from PR branch") - print(f"Parsed {len(pr_xfailed)} xfailed tests from PR branch") - print(f"Parsed {len(pr_skipped_with_reasons)} skipped tests with reasons") - print(f"Parsed {len(pr_xfailed_with_reasons)} xfailed tests with reasons") - - # Find tests that were passing in target but are now skipped or xfailed in PR - target_passing_set = set(target_passing) - pr_skipped_set = set(pr_skipped) - pr_xfailed_set = set(pr_xfailed) - - passing_to_skipped = list(target_passing_set.intersection(pr_skipped_set)) - passing_to_xfailed = list(target_passing_set.intersection(pr_xfailed_set)) - - total_skip_xfail_regressions = len(passing_to_skipped) + len(passing_to_xfailed) - - if total_skip_xfail_regressions > 0: - print(f"Found {total_skip_xfail_regressions} tests that were passing in target but are now skipped/xfailed in PR!") - - # Build comprehensive warning message - warning_parts = [f"Skip/XFail Analysis: {total_skip_xfail_regressions} test(s) that were passing in target branch are now being skipped or xfailed in PR branch."] - - if passing_to_skipped: - warning_parts.append(f"Tests now SKIPPED ({len(passing_to_skipped)}):") - for idx, test in enumerate(sorted(passing_to_skipped), 1): - reason = pr_skipped_with_reasons.get(test, 'No reason provided') - warning_parts.append(f" {idx}. {test} - Reason: {reason}") - - if passing_to_xfailed: - warning_parts.append(f"Tests now XFAILED ({len(passing_to_xfailed)}):") - for idx, test in enumerate(sorted(passing_to_xfailed), 1): - reason = pr_xfailed_with_reasons.get(test, 'No reason provided') - warning_parts.append(f" {idx}. {test} - Reason: {reason}") - - warning_parts.append("While these changes don't fail the workflow, they indicate tests that were working before are now being bypassed. Please review these tests to ensure this is intentional.") - - # Print as single warning annotation - combined_warning = " ".join(warning_parts) - print(f"::warning::{combined_warning}") - else: - print("No skip/xfail regressions found - all previously passing tests are still running.") - except Exception as e: - print(f"Error in skip/xfail analysis: {e}") - import traceback - print(traceback.format_exc()) - EOF - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Skip/xfail regression analysis completed" - - - name: Check for test additions and removals from artifacts - run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_test_changes_analysis.log) 2>&1 - - echo "Checking for test additions and removals between target and PR branches" - - python3 - << 'EOF' - import json - import os - import glob - - try: - # Load test data from artifacts - target_data = {} - pr_data = {} - - # Load target and PR data - target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') - if target_files: - with open(target_files[0], 'r') as f: - target_data = json.load(f) - - pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') - if pr_files: - with open(pr_files[0], 'r') as f: - pr_data = json.load(f) - - # Extract test arrays - target_all = target_data.get('all_tests', []) - pr_all = pr_data.get('all_tests', []) - - print(f"Parsed {len(target_all)} total tests from target branch") - print(f"Parsed {len(pr_all)} total tests from PR branch") - - # Find test additions and removals using set operations - target_all_set = set(target_all) - pr_all_set = set(pr_all) - - removed_tests = list(target_all_set - pr_all_set) # In target but not in PR - added_tests = list(pr_all_set - target_all_set) # In PR but not in target - - # Report removed tests (warnings) - if removed_tests: - print(f"Found {len(removed_tests)} tests that were removed from target branch!") - - # Build comprehensive removal warning - removal_parts = [f"Test Removal Analysis: {len(removed_tests)} test(s) that existed in target branch are missing from PR branch."] - removal_parts.append(f"REMOVED Tests ({len(removed_tests)}):") - for idx, test in enumerate(sorted(removed_tests), 1): - removal_parts.append(f" {idx}. {test}") - removal_parts.append("These test removals should be reviewed to ensure they are intentional. If tests were renamed or moved, this may show as removal + addition.") - - # Print as single warning annotation - combined_removal_warning = " ".join(removal_parts) - print(f"::warning::{combined_removal_warning}") - else: - print("No test removals detected.") - - # Report added tests (notifications/info) - if added_tests: - print(f"Found {len(added_tests)} new tests added in PR branch!") - - # Build comprehensive addition notice - addition_parts = [f"Test Addition Analysis: {len(added_tests)} new test(s) have been added in the PR branch."] - addition_parts.append(f"NEW Tests ({len(added_tests)}):") - for idx, test in enumerate(sorted(added_tests), 1): - addition_parts.append(f" {idx}. {test}") - addition_parts.append("New tests detected - this indicates expanded test coverage!") - - # Print as single notice annotation - combined_addition_notice = " ".join(addition_parts) - print(f"::notice::{combined_addition_notice}") - else: - print("No new tests detected in PR branch.") - - # Summary - if not removed_tests and not added_tests: - print("Test suite composition is unchanged between target and PR branches.") - else: - print(f"Test suite changes: {len(added_tests)} added, {len(removed_tests)} removed") - - except Exception as e: - print(f"Error in test addition/removal analysis: {e}") - import traceback - print(traceback.format_exc()) - EOF - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Test addition/removal analysis completed" - - name: Compare test results run: | echo "Test Results Summary:" @@ -970,7 +575,6 @@ jobs: if [[ "${{ needs.test-source-branch.outputs.total }}" == "0" ]]; then echo "::error::No tests were found in the PR branch" - echo "❌ PR branch has no tests detected. Please add test files that match pytest's discovery pattern." exit 1 fi @@ -981,89 +585,29 @@ jobs: PR_TOTAL=${{ needs.test-source-branch.outputs.total }} TARGET_TOTAL=${{ needs.test-target-branch.outputs.total }} - # Handle case where target has no tests if [[ "$TARGET_TOTAL" == "0" ]]; then if [[ "$PR_PASSED" -gt 0 ]]; then echo "✅ PR branch has tests and some are passing (target branch has no tests)" exit 0 else echo "❌ PR branch has no passing tests" - echo " - Pass percentage: $PR_PERCENTAGE%" exit 1 fi fi - # Fail if any tests passed in target branch but now fail in PR branch if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then echo "❌ PR branch has test regressions from target branch" - REGRESSION_COUNT_VAL=${{ needs.perform-regression-analysis.outputs.regression_count }} - echo " - $REGRESSION_COUNT_VAL tests that were passing in target branch are now failing" - - echo "### :x: Test Regressions Detected!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**$REGRESSION_COUNT_VAL test(s) that were passing in the target branch are now failing in the PR branch.**" >> $GITHUB_STEP_SUMMARY - echo "This job (\`compare-results\`) has been marked as failed due to these regressions." >> $GITHUB_STEP_SUMMARY - - if [ -f regression_details.txt ]; then # Check if file exists (it should from previous step) - echo "Failing tests (regressions) list:" - cat regression_details.txt - else - echo "::warning::Regression details file (regression_details.txt) not found in Compare test results step. It might be available as an artifact from the 'perform-regression-analysis' job." - fi exit 1 fi - # Continue with the original comparison if no regressions if (( $(echo "$PR_PASSED >= $TARGET_PASSED" | bc -l) )) && (( $(echo "$PR_PERCENTAGE >= $TARGET_PERCENTAGE" | bc -l) )); then echo "✅ PR branch has equal or better test results than target branch" - - # Additional verbose information about improvement - if (( $(echo "$PR_PASSED > $TARGET_PASSED" | bc -l) )); then - IMPROVEMENT=$(( $PR_PASSED - $TARGET_PASSED )) - echo " - Improvement: $IMPROVEMENT more passing tests than target branch" - fi - - if (( $(echo "$PR_PERCENTAGE > $TARGET_PERCENTAGE" | bc -l) )); then - PERCENTAGE_IMPROVEMENT=$(echo "$PR_PERCENTAGE - $TARGET_PERCENTAGE" | bc -l) - echo " - Percentage improvement: +${PERCENTAGE_IMPROVEMENT}% compared to target branch" - fi - exit 0 else echo "❌ PR branch has worse test results than target branch" - echo " - Passed tests: $PR_PASSED vs $TARGET_PASSED on target branch" - echo " - Pass percentage: $PR_PERCENTAGE% vs $TARGET_PERCENTAGE% on target branch" - - # Add to job summary for general comparison failure - echo "### :x: Test Comparison Failed" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The PR branch has worse test results than the target branch:" >> $GITHUB_STEP_SUMMARY - echo "- Passed tests: $PR_PASSED (PR) vs $TARGET_PASSED (Target)" >> $GITHUB_STEP_SUMMARY - echo "- Pass percentage: $PR_PERCENTAGE% (PR) vs $TARGET_PERCENTAGE% (Target)" >> $GITHUB_STEP_SUMMARY - - # Calculate regression metrics - if (( $(echo "$PR_PASSED < $TARGET_PASSED" | bc -l) )); then - REGRESSION=$(( $TARGET_PASSED - $PR_PASSED )) - echo " - Regression: $REGRESSION fewer passing tests than target branch" - fi - - if (( $(echo "$PR_PERCENTAGE < $TARGET_PERCENTAGE" | bc -l) )); then - PERCENTAGE_REGRESSION=$(echo "$TARGET_PERCENTAGE - $PR_PERCENTAGE" | bc -l) - echo " - Percentage regression: -${PERCENTAGE_REGRESSION}% compared to target branch" - fi - exit 1 fi - - name: Upload comparison analysis logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: comparison_analysis_logs_${{ github.event.pull_request.number || github.run_id }} - path: debug_*_analysis.log - retention-days: 3 - if-no-files-found: ignore - perform-regression-analysis: needs: [test-source-branch, test-target-branch] uses: ./.github/workflows/meta-regression-analysis.yml @@ -1075,7 +619,6 @@ jobs: target_branch_artifact_name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} pr_branch_artifact_name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} - # Conditionally run notification job only if needed prepare-notification: name: Prepare Notification Data needs: @@ -1086,7 +629,6 @@ jobs: compare-results, perform-regression-analysis, ] - # Notify on collection errors, no tests found, compare result failure, or if regressions are detected if: | always() && ( @@ -1108,18 +650,18 @@ jobs: id: check_webhook_availability run: | if [ -z "${{ secrets.DISCORD_WEBHOOK_URL }}" ]; then - echo "::notice::DISCORD_WEBHOOK_URL secret is not set. Discord notifications will likely be skipped by the alert workflow if it relies on this secret." echo "webhook_available=false" >> $GITHUB_OUTPUT else echo "webhook_available=true" >> $GITHUB_OUTPUT fi + - name: Download regression details (if any) id: download_regressions if: needs.perform-regression-analysis.outputs.has_regressions == 'true' && needs.perform-regression-analysis.outputs.regression_count > 0 uses: actions/download-artifact@v4 with: name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests - path: . # Download to current directory + path: . continue-on-error: true - name: Construct Discord Notification @@ -1154,77 +696,42 @@ jobs: DOWNLOAD_REGRESSIONS_OUTCOME: ${{ steps.download_regressions.outcome }} INPUT_PING_LATEST_COMMITTER: ${{ inputs.ping_latest_committer }} run: | - # Create debug file for detailed notification construction - exec 3>&1 4>&2 - exec 1> >(tee -a debug_notification_construction.log) 2>&1 - - MESSAGE_LINES=() # Use an array to build message lines - PING_KEYS_OUTPUT="" # Will be comma-separated GitHub logins + MESSAGE_LINES=() + PING_KEYS_OUTPUT="" ARTIFACT_PATH_OUTPUT="" - echo "Raw GH_ASSIGNEES_JSON value: [$GH_ASSIGNEES_JSON]" - echo "Raw USER_MAP_JSON value: [$USER_MAP_JSON]" - - # 1. Determine Pings - Collect GitHub Logins to pass to alert-discord.yml - # Initialize PING_KEYS_OUTPUT PING_KEYS_OUTPUT="" - # Add assignees to PING_KEYS_OUTPUT if [ -n "$USER_MAP_JSON" ] && [ "$USER_MAP_JSON" != "{}" ] && command -v jq &> /dev/null; then ASSIGNEE_LOGINS_ARRAY=($(echo "$GH_ASSIGNEES_JSON" | jq -r '.[].login // empty')) - echo "GH_ASSIGNEES_JSON received: $GH_ASSIGNEES_JSON" - echo "Extracted ASSIGNEE_LOGINS_ARRAY: (${ASSIGNEE_LOGINS_ARRAY[*]})" - echo "Count of assignees extracted: ${#ASSIGNEE_LOGINS_ARRAY[@]}" - MAPPED_ASSIGNEE_COUNT=0 TEMP_PING_KEYS=() for assignee_login in "${ASSIGNEE_LOGINS_ARRAY[@]}"; do if [ -z "$assignee_login" ]; then - echo "Skipping empty assignee login." continue fi - echo "Processing assignee for ping: '$assignee_login'" - # Check if this assignee_login exists as a key in USER_MAP_JSON if echo "$USER_MAP_JSON" | jq -e --arg K "$assignee_login" '.[$K]' > /dev/null; then - echo "Assignee '$assignee_login' FOUND in USER_MAP_JSON." TEMP_PING_KEYS+=("$assignee_login") MAPPED_ASSIGNEE_COUNT=$((MAPPED_ASSIGNEE_COUNT + 1)) - else - echo "Assignee '$assignee_login' NOT FOUND in USER_MAP_JSON." fi done - echo "Total assignees found in USER_MAP_JSON and added to pings: $MAPPED_ASSIGNEE_COUNT" - if [ ${#TEMP_PING_KEYS[@]} -gt 0 ]; then PING_KEYS_OUTPUT=$(IFS=,; echo "${TEMP_PING_KEYS[*]}") - echo "Initial PING_KEYS_OUTPUT from assignees: [$PING_KEYS_OUTPUT]" - else - echo "No assignees found or GH_ASSIGNEES_JSON was empty, or no assignees were found in USER_MAP_JSON." fi - elif [ -n "$USER_MAP_JSON" ] && [ "$USER_MAP_JSON" != "{}" ] && ! command -v jq &> /dev/null; then - echo "::warning::jq is not available. Cannot determine GitHub users (assignees) for pings." - else - echo "No user map JSON or jq not found. PING_KEYS_OUTPUT (from assignees) will be empty." fi - # Add latest committer if INPUT_PING_LATEST_COMMITTER is true if [[ "$INPUT_PING_LATEST_COMMITTER" == "true" ]]; then - echo "INPUT_PING_LATEST_COMMITTER is true. Attempting to fetch latest committer for PR #${PR_NUMBER}." if command -v gh &> /dev/null && [ -n "$PR_NUMBER" ]; then LATEST_COMMITTER_LOGIN_RAW=$(gh pr view "$PR_NUMBER" --json commits --jq '.commits[-1].author.login' 2>/dev/null || echo "") if [ -n "$LATEST_COMMITTER_LOGIN_RAW" ] && [ "$LATEST_COMMITTER_LOGIN_RAW" != "null" ]; then - # Apply bot filter (e.g., names ending in [bot] or -bot) LATEST_COMMITTER_LOGIN=$(echo "$LATEST_COMMITTER_LOGIN_RAW" | grep -v -E -i '(\[bot\]$|-bot$)' || echo "") if [ -n "$LATEST_COMMITTER_LOGIN" ]; then - echo "Latest committer identified: $LATEST_COMMITTER_LOGIN" - - # Check if this committer is already in PING_KEYS_OUTPUT ALREADY_IN_LIST=0 - if [ -n "$PING_KEYS_OUTPUT" ]; then # Only check if PING_KEYS_OUTPUT is not empty + if [ -n "$PING_KEYS_OUTPUT" ]; then IFS=',' read -ra PING_ARRAY <<< "$PING_KEYS_OUTPUT" for key in "${PING_ARRAY[@]}"; do if [[ "$key" == "$LATEST_COMMITTER_LOGIN" ]]; then @@ -1240,43 +747,21 @@ jobs: else PING_KEYS_OUTPUT="$PING_KEYS_OUTPUT,$LATEST_COMMITTER_LOGIN" fi - echo "Added latest committer '$LATEST_COMMITTER_LOGIN' to PING_KEYS_OUTPUT. New list: [$PING_KEYS_OUTPUT]" - else - echo "Latest committer '$LATEST_COMMITTER_LOGIN' is already in PING_KEYS_OUTPUT (likely an assignee)." fi - else - echo "Latest committer login '$LATEST_COMMITTER_LOGIN_RAW' was filtered out (likely a bot or pattern match) or empty after filter." fi - else - echo "No latest committer login found for PR #$PR_NUMBER from gh command, or login was null." - fi - else - if ! command -v gh &> /dev/null; then - echo "::warning::gh command not available. Cannot fetch latest committer." - fi - if [ -z "$PR_NUMBER" ]; then - echo "::warning::PR_NUMBER is not set (event might not be a pull_request). Cannot fetch latest committer." fi fi fi - # Restore stdout/stderr for GitHub Actions to show final summary - exec 1>&3 2>&4 - - # Make this a standard echo for better visibility of the final list - echo "Final Ping Keys Output (GitHub Logins from test-pytest.yml): [$PING_KEYS_OUTPUT]" echo "ping_user_ids_out=$PING_KEYS_OUTPUT" >> $GITHUB_OUTPUT - # Store branch names in variables with proper quoting PR_BRANCH="${PR_BRANCH_NAME:-unknown}" TARGET_BRANCH="${TARGET_BRANCH_NAME:-unknown}" - # 2. Construct Message Body MESSAGE_LINES+=("**Pytest Comparison & Regression Analysis for PR [#${PR_NUMBER}: ${PR_TITLE}](${PR_URL})**") MESSAGE_LINES+=("Branch: [\`${PR_BRANCH}\`](${REPO_URL}/tree/${PR_BRANCH}) against [\`${TARGET_BRANCH}\`](${REPO_URL}/tree/${TARGET_BRANCH})") MESSAGE_LINES+=("---") - # Job Status Summary MESSAGE_LINES+=("**Job Status:**") LINT_STATUS="Success" if [[ "$LINT_RESULT" == "failure" ]]; then LINT_STATUS="Failed"; elif [[ "$LINT_RESULT" == "skipped" ]]; then LINT_STATUS="Skipped"; fi @@ -1295,32 +780,24 @@ jobs: MESSAGE_LINES+=("- Comparison & Regression: $COMPARE_STATUS") MESSAGE_LINES+=("---") - # Test Discovery Issues in PR Branch if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: ERROR: Test Discovery Failed in PR Branch (\`${PR_BRANCH}\`)**") MESSAGE_LINES+=(" - Type: \`${PR_ERROR_TYPE}\`") MESSAGE_LINES+=(" - Details: \`\`\`${PR_ERROR_DETAILS_TRUNCATED}\`\`\`") - MESSAGE_LINES+=(" - This usually indicates import errors or syntax issues preventing tests from being collected.") elif [[ "$PR_NO_TESTS_FOUND" == "true" ]]; then MESSAGE_LINES+=("**:warning: WARNING: No Tests Found in PR Branch (\`${PR_BRANCH}\`)**") - MESSAGE_LINES+=(" - Pytest did not discover any test files matching its patterns.") - MESSAGE_LINES+=(" - Ensure your test files are correctly named (e.g., \`test_*.py\` or \`*_test.py\`) and located.") fi - # Regression Analysis Summary if [[ "$HAS_REGRESSIONS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: REGRESSIONS DETECTED**") MESSAGE_LINES+=(" - **${REGRESSION_COUNT} test(s)** that were passing in \`${TARGET_BRANCH}\` are now **failing** in \`${PR_BRANCH}\`.") - # Calculate current message length CURRENT_MESSAGE=$(printf "%s\\n" "${MESSAGE_LINES[@]}") CURRENT_LENGTH=${#CURRENT_MESSAGE} if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then - # Read regression details REGRESSION_LIST=$(awk '/^[0-9]+\./ {sub(/^[0-9]+\. /, "- "); print}' "$REGRESSION_FILE_PATH") - # Calculate length with regression details TEMP_MESSAGE="$CURRENT_MESSAGE" TEMP_MESSAGE+="\`\`\`" TEMP_MESSAGE+="$REGRESSION_LIST" @@ -1328,25 +805,20 @@ jobs: TEMP_LENGTH=${#TEMP_MESSAGE} if [ $TEMP_LENGTH -le 2000 ]; then - # If total length would be under 2000 chars, include in message MESSAGE_LINES+=(" - **Failed Tests (Regressions):**") MESSAGE_LINES+=("\`\`\`") MESSAGE_LINES+=("$REGRESSION_LIST") MESSAGE_LINES+=("\`\`\`") - ARTIFACT_PATH_OUTPUT="" # No artifact if details are inline + ARTIFACT_PATH_OUTPUT="" else - # If would exceed 2000 chars, attach file instead MESSAGE_LINES+=(" - Details for the ${REGRESSION_COUNT} regressions are in the attached \`regression_details.txt\` file.") ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" fi else - MESSAGE_LINES+=(" (Regression details file not found or download failed; cannot list specific regressions here.)") ARTIFACT_PATH_OUTPUT="" fi elif [[ "$COMPARE_RESULT" == "failure" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then - # This case handles general comparison failures NOT due to specific regressions MESSAGE_LINES+=("**:warning: TEST RESULTS DECLINED**") - MESSAGE_LINES+=(" - The PR branch shows a decrease in test success compared to the target branch, but no specific regressions were identified by the \`meta-regression-analysis\` job.") MESSAGE_LINES+=(" - PR Branch (\`${PR_BRANCH}\`): **${PR_PASSED_TESTS}/${PR_TOTAL_TESTS} passed (${PR_PERCENTAGE}%)**") MESSAGE_LINES+=(" - Target Branch (\`${TARGET_BRANCH}\`): **${TARGET_PASSED_TESTS}/${TARGET_TOTAL_TESTS} passed (${TARGET_PERCENTAGE}%)**") elif [[ "$COMPARE_RESULT" == "success" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then @@ -1358,30 +830,17 @@ jobs: MESSAGE_LINES+=("---") MESSAGE_LINES+=("[View Workflow Run](${ACTION_RUN_URL})") - # Construct with actual newlines FINAL_MESSAGE_BODY=$(printf "%s\\n" "${MESSAGE_LINES[@]}") if [ ${#MESSAGE_LINES[@]} -gt 0 ]; then - # Remove the very last actual newline FINAL_MESSAGE_BODY="${FINAL_MESSAGE_BODY%\\n}" fi - echo "Final message body prepared in test-pytest.yml" - echo "message_body_out<> $GITHUB_OUTPUT echo "$FINAL_MESSAGE_BODY" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT echo "artifact_path_out=$ARTIFACT_PATH_OUTPUT" >> $GITHUB_OUTPUT - - name: Upload notification construction debug logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: notification_debug_logs_${{ github.event.pull_request.number || github.run_id }} - path: debug_notification_construction.log - retention-days: 3 - if-no-files-found: ignore - notify-discord: name: Send Discord Notification needs: [prepare-notification] From 00a073aadd6a6f3800fda17ced1e33ac29d35fa5 Mon Sep 17 00:00:00 2001 From: Jameson Grieve <37882431+JamesonRGrieve@users.noreply.github.com> Date: Wed, 4 Jun 2025 11:25:36 -0700 Subject: [PATCH 02/44] Update test-pytest.yml --- .github/workflows/test-pytest.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index 33c5bd4..d70ad81 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -61,11 +61,6 @@ on: value: ${{ jobs.compare-results.outputs.regression_count }} jobs: - lint: - uses: ./.github/workflows/test-lint-py.yml - permissions: - contents: write - test-source-branch: needs: lint runs-on: ${{ inputs.runs_on }} From 18c91205d5bf531659bca362166c09a8a5edd15d Mon Sep 17 00:00:00 2001 From: Jameson Grieve <37882431+JamesonRGrieve@users.noreply.github.com> Date: Wed, 4 Jun 2025 11:38:33 -0700 Subject: [PATCH 03/44] Update test-pytest.yml --- .github/workflows/test-pytest.yml | 185 ++++++++++++++++-------------- 1 file changed, 101 insertions(+), 84 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index d70ad81..afdb624 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -96,82 +96,85 @@ jobs: pip install -r requirements.txt --quiet fi - - name: Check for test collection errors + - name: Silent pytest execution and collection check id: check-collection run: | - HAS_COLLECTION_ERRORS="false" - NO_TESTS_FOUND="false" - ERROR_TYPE="none" - ERROR_DETAILS="none" - - python -m pytest --collect-only --quiet --tb=no >/dev/null 2>collection_check.txt - COLLECTION_EXIT_CODE=$? + # Use Python for completely silent execution + python3 << 'EOF' + import subprocess + import sys + import os - if [ $COLLECTION_EXIT_CODE -ne 0 ]; then - if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting" collection_check.txt; then - echo "::error::Test discovery errors detected in PR branch" - HAS_COLLECTION_ERRORS="true" + def run_silent_command(cmd, capture_output=False): + try: + if capture_output: + return subprocess.run(cmd, capture_output=True, text=True, timeout=60) + else: + with open(os.devnull, 'w') as devnull: + return subprocess.run(cmd, stdout=devnull, stderr=devnull, timeout=1800) + except: + return subprocess.CompletedProcess(cmd, 1) + + # Step 1: Collection check + collection_cmd = [sys.executable, '-m', 'pytest', '--collect-only', '--quiet', '--tb=no'] + collection_result = run_silent_command(collection_cmd, capture_output=True) + + has_collection_errors = "false" + no_tests_found = "false" + error_type = "none" + error_details = "none" + + if collection_result.returncode != 0: + if any(err in collection_result.stderr for err in ['ImportError', 'ModuleNotFoundError', 'SyntaxError', 'ERROR collecting']): + has_collection_errors = "true" + if 'ImportError' in collection_result.stderr: + error_type = "ImportError" + elif 'ModuleNotFoundError' in collection_result.stderr: + error_type = "ModuleNotFoundError" + elif 'SyntaxError' in collection_result.stderr: + error_type = "SyntaxError" + else: + error_type = "CollectionError" + error_details = collection_result.stderr[:200] if collection_result.stderr else "Collection failed" + else: + if 'collected 0 items' in collection_result.stdout or not collection_result.stdout.strip(): + no_tests_found = "true" + error_type = "NoTestsFound" + error_details = "No test files discovered" + + # Write outputs + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f'has_collection_errors={has_collection_errors}\n') + f.write(f'no_tests_found={no_tests_found}\n') + f.write(f'error_type={error_type}\n') + f.write(f'error_details={error_details}\n') + f.write(f'has_errors={str(has_collection_errors == "true" or no_tests_found == "true").lower()}\n') + + # Step 2: Run tests if collection succeeded + if has_collection_errors == "false": + test_cmd = [ + sys.executable, '-m', 'pytest', + '--json-report', + '--json-report-file=pr_results.json', + '--tb=no', + '--no-header', + '--no-summary', + '--quiet', + '--disable-warnings', + '--log-level=CRITICAL', + '--capture=sys', + '--maxfail=999999' + ] - if grep -q "ImportError" collection_check.txt; then - ERROR_TYPE="ImportError" - elif grep -q "ModuleNotFoundError" collection_check.txt; then - ERROR_TYPE="ModuleNotFoundError" - elif grep -q "SyntaxError" collection_check.txt; then - ERROR_TYPE="SyntaxError" - else - ERROR_TYPE="CollectionError" - fi + test_result = run_silent_command(test_cmd) - ERROR_DETAILS=$(head -1 collection_check.txt | tr '\n' ' ' | cut -c1-200) - fi - else - TEST_COUNT=$(python -m pytest --collect-only --quiet 2>/dev/null | grep -o "collected [0-9]* item" | grep -o "[0-9]*" || echo "0") - if [[ "$TEST_COUNT" == "0" ]]; then - echo "::warning::No tests were found in the PR branch" - NO_TESTS_FOUND="true" - ERROR_TYPE="NoTestsFound" - ERROR_DETAILS="No test files discovered" - fi - fi - - echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT - echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT - echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT - echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT - - if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then - echo "has_errors=true" >> $GITHUB_OUTPUT - else - echo "has_errors=false" >> $GITHUB_OUTPUT - fi - - - name: Run tests on PR Branch - if: steps.check-collection.outputs.has_collection_errors != 'true' - env: - PYTHONUNBUFFERED: "0" - PYTHONWARNINGS: "ignore" - PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" - PYTEST_CURRENT_TEST: "" - COLUMNS: "80" - run: | - python -m pytest \ - --json-report \ - --json-report-file=pr_results.json \ - --tb=no \ - --no-header \ - --no-summary \ - -q \ - --disable-warnings \ - --log-level=CRITICAL \ - --log-cli-level=CRITICAL \ - --capture=sys \ - >/dev/null 2>&1 || true - - if [ -f pr_results.json ]; then - echo "Test results generated successfully" - else - echo "::error::Failed to create test results file" - fi + if os.path.exists('pr_results.json'): + print("Test results generated successfully") + else: + print("::error::Failed to create test results file") + else: + print(f"::error::Skipping tests due to collection errors: {error_type}") + EOF - name: Extract test results and create artifacts id: extract-results @@ -362,19 +365,33 @@ jobs: PYTEST_CURRENT_TEST: "" COLUMNS: "80" run: | - python -m pytest \ - --json-report \ - --json-report-file=target_results.json \ - --tb=no \ - --no-header \ - --no-summary \ - -q \ - --disable-warnings \ - --log-level=CRITICAL \ - --log-cli-level=CRITICAL \ - --capture=sys \ - >/dev/null 2>&1 || true - + # Create completely isolated execution to prevent any output leakage + ( + # Close all file descriptors and redirect to /dev/null + exec 0/dev/null + exec 2>/dev/null + + # Run pytest in completely silent mode + python -m pytest \ + --json-report \ + --json-report-file=target_results.json \ + --tb=no \ + --no-header \ + --no-summary \ + --quiet \ + --disable-warnings \ + --log-level=CRITICAL \ + --log-cli-level=CRITICAL \ + --capture=sys \ + --maxfail=999999 \ + --tb=no + ) & + + # Wait for background process to complete + wait + + # Check results outside the silent execution if [ -f target_results.json ]; then echo "Test results generated successfully" else From a760433616e1debbe19549a9e7797fb1e3dc20ad Mon Sep 17 00:00:00 2001 From: Jameson Grieve <37882431+JamesonRGrieve@users.noreply.github.com> Date: Wed, 4 Jun 2025 11:38:56 -0700 Subject: [PATCH 04/44] Update test-pytest.yml --- .github/workflows/test-pytest.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index afdb624..b8e242e 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -62,7 +62,6 @@ on: jobs: test-source-branch: - needs: lint runs-on: ${{ inputs.runs_on }} outputs: total: ${{ steps.extract-results.outputs.total }} @@ -274,7 +273,6 @@ jobs: if-no-files-found: ignore test-target-branch: - needs: lint runs-on: ${{ inputs.runs_on }} outputs: total: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.total || steps.extract-results.outputs.total }} From 820063a1b073a3935c7adf35f1a50dc2fb49eb12 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 4 Jun 2025 14:26:32 -0600 Subject: [PATCH 05/44] Silent; --- .github/workflows/test-pytest.yml | 59 +++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index b8e242e..fe0a894 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -104,25 +104,51 @@ jobs: import sys import os - def run_silent_command(cmd, capture_output=False): + def run_completely_silent(cmd, timeout=1800): + """Run command with complete output suppression""" try: - if capture_output: - return subprocess.run(cmd, capture_output=True, text=True, timeout=60) - else: - with open(os.devnull, 'w') as devnull: - return subprocess.run(cmd, stdout=devnull, stderr=devnull, timeout=1800) - except: + # Suppress all output streams + result = subprocess.run( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + timeout=timeout, + env={**os.environ, + 'PYTHONUNBUFFERED': '0', + 'PYTHONWARNINGS': 'ignore', + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', + 'PYTEST_CURRENT_TEST': '', + 'COLUMNS': '80'} + ) + return result + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(cmd, 1) + except Exception: return subprocess.CompletedProcess(cmd, 1) + def run_with_captured_output(cmd, timeout=60): + """Run command and capture output for error checking""" + try: + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + env={**os.environ, 'PYTHONWARNINGS': 'ignore'} + ) + except Exception: + return subprocess.CompletedProcess(cmd, 1, stderr="Execution failed") + # Step 1: Collection check collection_cmd = [sys.executable, '-m', 'pytest', '--collect-only', '--quiet', '--tb=no'] - collection_result = run_silent_command(collection_cmd, capture_output=True) - + collection_result = run_with_captured_output(collection_cmd) + has_collection_errors = "false" no_tests_found = "false" error_type = "none" error_details = "none" - + if collection_result.returncode != 0: if any(err in collection_result.stderr for err in ['ImportError', 'ModuleNotFoundError', 'SyntaxError', 'ERROR collecting']): has_collection_errors = "true" @@ -161,11 +187,14 @@ jobs: '--quiet', '--disable-warnings', '--log-level=CRITICAL', + '--log-cli-level=CRITICAL', '--capture=sys', - '--maxfail=999999' + '--maxfail=999999', + '-v', '--tb=no' # -v for verbose JSON but --tb=no for no traceback output ] - test_result = run_silent_command(test_cmd) + # Run completely silently + test_result = run_completely_silent(test_cmd) if os.path.exists('pr_results.json'): print("Test results generated successfully") @@ -347,7 +376,7 @@ jobs: echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT - + if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then echo "has_errors=true" >> $GITHUB_OUTPUT else @@ -385,10 +414,10 @@ jobs: --maxfail=999999 \ --tb=no ) & - + # Wait for background process to complete wait - + # Check results outside the silent execution if [ -f target_results.json ]; then echo "Test results generated successfully" From 3ba8bc6316b286c3412fe23b241188444a0849df Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 4 Jun 2025 14:27:29 -0600 Subject: [PATCH 06/44] Silent; --- .github/workflows/test-pytest.yml | 107 +++++++++++++++++------------- 1 file changed, 62 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index fe0a894..a57f826 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -385,45 +385,61 @@ jobs: - name: Run tests on target branch if: steps.check-collection.outputs.has_collection_errors != 'true' - env: - PYTHONUNBUFFERED: "0" - PYTHONWARNINGS: "ignore" - PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" - PYTEST_CURRENT_TEST: "" - COLUMNS: "80" run: | - # Create completely isolated execution to prevent any output leakage - ( - # Close all file descriptors and redirect to /dev/null - exec 0/dev/null - exec 2>/dev/null - - # Run pytest in completely silent mode - python -m pytest \ - --json-report \ - --json-report-file=target_results.json \ - --tb=no \ - --no-header \ - --no-summary \ - --quiet \ - --disable-warnings \ - --log-level=CRITICAL \ - --log-cli-level=CRITICAL \ - --capture=sys \ - --maxfail=999999 \ - --tb=no - ) & - - # Wait for background process to complete - wait - - # Check results outside the silent execution - if [ -f target_results.json ]; then - echo "Test results generated successfully" - else - echo "::warning::Failed to create test results file" - fi + # Use Python for completely silent pytest execution + python3 << 'EOF' + import subprocess + import sys + import os + + def run_completely_silent(cmd, timeout=1800): + """Run command with complete output suppression""" + try: + # Suppress all output streams and set environment for silent execution + result = subprocess.run( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + timeout=timeout, + env={**os.environ, + 'PYTHONUNBUFFERED': '0', + 'PYTHONWARNINGS': 'ignore', + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', + 'PYTEST_CURRENT_TEST': '', + 'COLUMNS': '80'} + ) + return result + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(cmd, 1) + except Exception: + return subprocess.CompletedProcess(cmd, 1) + + # Run pytest completely silently + test_cmd = [ + sys.executable, '-m', 'pytest', + '--json-report', + '--json-report-file=target_results.json', + '--tb=no', + '--no-header', + '--no-summary', + '--quiet', + '--disable-warnings', + '--log-level=CRITICAL', + '--log-cli-level=CRITICAL', + '--capture=sys', + '--maxfail=999999', + '-v', '--tb=no' # -v for verbose JSON but --tb=no for no traceback output + ] + + test_result = run_completely_silent(test_cmd) + + # Check results + if os.path.exists('target_results.json'): + print("Test results generated successfully") + else: + print("::warning::Failed to create test results file") + EOF - name: Extract test results and create artifacts id: extract-results @@ -662,7 +678,6 @@ jobs: name: Prepare Notification Data needs: [ - lint, test-source-branch, test-target-branch, compare-results, @@ -706,7 +721,6 @@ jobs: - name: Construct Discord Notification id: construct_notification env: - LINT_RESULT: ${{ needs.lint.result }} SOURCE_TEST_RESULT: ${{ needs.test-source-branch.result }} TARGET_TEST_RESULT: ${{ needs.test-target-branch.result }} COMPARE_RESULT: ${{ needs.compare-results.result }} @@ -802,10 +816,6 @@ jobs: MESSAGE_LINES+=("---") MESSAGE_LINES+=("**Job Status:**") - LINT_STATUS="Success" - if [[ "$LINT_RESULT" == "failure" ]]; then LINT_STATUS="Failed"; elif [[ "$LINT_RESULT" == "skipped" ]]; then LINT_STATUS="Skipped"; fi - MESSAGE_LINES+=("- Linting: $LINT_STATUS") - SOURCE_TEST_STATUS="Success" if [[ "$SOURCE_TEST_RESULT" == "failure" ]]; then SOURCE_TEST_STATUS="Failed"; elif [[ "$SOURCE_TEST_RESULT" == "skipped" ]]; then SOURCE_TEST_STATUS="Skipped"; fi MESSAGE_LINES+=("- PR Branch Tests (\`${PR_BRANCH}\`): $SOURCE_TEST_STATUS") @@ -882,7 +892,14 @@ jobs: notify-discord: name: Send Discord Notification - needs: [prepare-notification] + needs: + [ + test-source-branch, + test-target-branch, + compare-results, + perform-regression-analysis, + prepare-notification, + ] if: | always() && needs.prepare-notification.outputs.should_notify == 'true' && From ac1a1057a45cd6985b4958fd0ab74bacdf7a5362 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 4 Jun 2025 14:32:33 -0600 Subject: [PATCH 07/44] Quiet; --- .github/workflows/test-pytest.yml | 144 ++++++++++++++++++++---------- 1 file changed, 98 insertions(+), 46 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index a57f826..2b0f661 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -98,33 +98,60 @@ jobs: - name: Silent pytest execution and collection check id: check-collection run: | - # Use Python for completely silent execution + # Use Python with OS-level file descriptor redirection for complete silence python3 << 'EOF' import subprocess import sys import os + import contextlib - def run_completely_silent(cmd, timeout=1800): - """Run command with complete output suppression""" + def run_with_complete_silence(cmd, timeout=1800): + """Run command with OS-level output suppression""" try: - # Suppress all output streams - result = subprocess.run( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, - timeout=timeout, - env={**os.environ, - 'PYTHONUNBUFFERED': '0', - 'PYTHONWARNINGS': 'ignore', - 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', - 'PYTEST_CURRENT_TEST': '', - 'COLUMNS': '80'} - ) + # Save original file descriptors + original_stdout = os.dup(1) + original_stderr = os.dup(2) + original_stdin = os.dup(0) + + try: + # Redirect to /dev/null at OS level + devnull = os.open(os.devnull, os.O_WRONLY) + devnull_read = os.open(os.devnull, os.O_RDONLY) + + os.dup2(devnull, 1) # stdout + os.dup2(devnull, 2) # stderr + os.dup2(devnull_read, 0) # stdin + + # Run the command + result = subprocess.run( + cmd, + timeout=timeout, + env={**os.environ, + 'PYTHONUNBUFFERED': '0', + 'PYTHONWARNINGS': 'ignore', + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', + 'PYTEST_CURRENT_TEST': '', + 'COLUMNS': '80', + 'PYTHONDONTWRITEBYTECODE': '1'} + ) + + os.close(devnull) + os.close(devnull_read) + + finally: + # Restore original file descriptors + os.dup2(original_stdout, 1) + os.dup2(original_stderr, 2) + os.dup2(original_stdin, 0) + os.close(original_stdout) + os.close(original_stderr) + os.close(original_stdin) + return result + except subprocess.TimeoutExpired: return subprocess.CompletedProcess(cmd, 1) - except Exception: + except Exception as e: return subprocess.CompletedProcess(cmd, 1) def run_with_captured_output(cmd, timeout=60): @@ -140,7 +167,7 @@ jobs: except Exception: return subprocess.CompletedProcess(cmd, 1, stderr="Execution failed") - # Step 1: Collection check + # Step 1: Collection check (need output for this) collection_cmd = [sys.executable, '-m', 'pytest', '--collect-only', '--quiet', '--tb=no'] collection_result = run_with_captured_output(collection_cmd) @@ -175,7 +202,7 @@ jobs: f.write(f'error_details={error_details}\n') f.write(f'has_errors={str(has_collection_errors == "true" or no_tests_found == "true").lower()}\n') - # Step 2: Run tests if collection succeeded + # Step 2: Run tests if collection succeeded - COMPLETELY SILENT if has_collection_errors == "false": test_cmd = [ sys.executable, '-m', 'pytest', @@ -189,12 +216,11 @@ jobs: '--log-level=CRITICAL', '--log-cli-level=CRITICAL', '--capture=sys', - '--maxfail=999999', - '-v', '--tb=no' # -v for verbose JSON but --tb=no for no traceback output + '--maxfail=999999' ] - # Run completely silently - test_result = run_completely_silent(test_cmd) + # Run with complete OS-level silence + test_result = run_with_complete_silence(test_cmd) if os.path.exists('pr_results.json'): print("Test results generated successfully") @@ -386,36 +412,62 @@ jobs: - name: Run tests on target branch if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - # Use Python for completely silent pytest execution + # Use Python with OS-level file descriptor redirection for complete silence python3 << 'EOF' import subprocess import sys import os - def run_completely_silent(cmd, timeout=1800): - """Run command with complete output suppression""" + def run_with_complete_silence(cmd, timeout=1800): + """Run command with OS-level output suppression""" try: - # Suppress all output streams and set environment for silent execution - result = subprocess.run( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, - timeout=timeout, - env={**os.environ, - 'PYTHONUNBUFFERED': '0', - 'PYTHONWARNINGS': 'ignore', - 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', - 'PYTEST_CURRENT_TEST': '', - 'COLUMNS': '80'} - ) + # Save original file descriptors + original_stdout = os.dup(1) + original_stderr = os.dup(2) + original_stdin = os.dup(0) + + try: + # Redirect to /dev/null at OS level + devnull = os.open(os.devnull, os.O_WRONLY) + devnull_read = os.open(os.devnull, os.O_RDONLY) + + os.dup2(devnull, 1) # stdout + os.dup2(devnull, 2) # stderr + os.dup2(devnull_read, 0) # stdin + + # Run the command + result = subprocess.run( + cmd, + timeout=timeout, + env={**os.environ, + 'PYTHONUNBUFFERED': '0', + 'PYTHONWARNINGS': 'ignore', + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', + 'PYTEST_CURRENT_TEST': '', + 'COLUMNS': '80', + 'PYTHONDONTWRITEBYTECODE': '1'} + ) + + os.close(devnull) + os.close(devnull_read) + + finally: + # Restore original file descriptors + os.dup2(original_stdout, 1) + os.dup2(original_stderr, 2) + os.dup2(original_stdin, 0) + os.close(original_stdout) + os.close(original_stderr) + os.close(original_stdin) + return result + except subprocess.TimeoutExpired: return subprocess.CompletedProcess(cmd, 1) - except Exception: + except Exception as e: return subprocess.CompletedProcess(cmd, 1) - # Run pytest completely silently + # Run pytest with complete OS-level silence test_cmd = [ sys.executable, '-m', 'pytest', '--json-report', @@ -428,11 +480,11 @@ jobs: '--log-level=CRITICAL', '--log-cli-level=CRITICAL', '--capture=sys', - '--maxfail=999999', - '-v', '--tb=no' # -v for verbose JSON but --tb=no for no traceback output + '--maxfail=999999' ] - test_result = run_completely_silent(test_cmd) + # Run with complete OS-level silence + test_result = run_with_complete_silence(test_cmd) # Check results if os.path.exists('target_results.json'): From e585ed780c148b85efbfd9137919661f2d8eaff9 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 4 Jun 2025 14:45:41 -0600 Subject: [PATCH 08/44] Quieter; --- .github/workflows/test-pytest.yml | 36 +++++++++---------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index 2b0f661..1d853bd 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -98,31 +98,26 @@ jobs: - name: Silent pytest execution and collection check id: check-collection run: | - # Use Python with OS-level file descriptor redirection for complete silence python3 << 'EOF' import subprocess import sys import os - import contextlib def run_with_complete_silence(cmd, timeout=1800): """Run command with OS-level output suppression""" try: - # Save original file descriptors original_stdout = os.dup(1) original_stderr = os.dup(2) original_stdin = os.dup(0) try: - # Redirect to /dev/null at OS level devnull = os.open(os.devnull, os.O_WRONLY) devnull_read = os.open(os.devnull, os.O_RDONLY) - os.dup2(devnull, 1) # stdout - os.dup2(devnull, 2) # stderr - os.dup2(devnull_read, 0) # stdin + os.dup2(devnull, 1) + os.dup2(devnull, 2) + os.dup2(devnull_read, 0) - # Run the command result = subprocess.run( cmd, timeout=timeout, @@ -139,7 +134,6 @@ jobs: os.close(devnull_read) finally: - # Restore original file descriptors os.dup2(original_stdout, 1) os.dup2(original_stderr, 2) os.dup2(original_stdin, 0) @@ -167,7 +161,7 @@ jobs: except Exception: return subprocess.CompletedProcess(cmd, 1, stderr="Execution failed") - # Step 1: Collection check (need output for this) + # Collection check collection_cmd = [sys.executable, '-m', 'pytest', '--collect-only', '--quiet', '--tb=no'] collection_result = run_with_captured_output(collection_cmd) @@ -194,7 +188,6 @@ jobs: error_type = "NoTestsFound" error_details = "No test files discovered" - # Write outputs with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'has_collection_errors={has_collection_errors}\n') f.write(f'no_tests_found={no_tests_found}\n') @@ -202,7 +195,7 @@ jobs: f.write(f'error_details={error_details}\n') f.write(f'has_errors={str(has_collection_errors == "true" or no_tests_found == "true").lower()}\n') - # Step 2: Run tests if collection succeeded - COMPLETELY SILENT + # Run tests silently if collection succeeded if has_collection_errors == "false": test_cmd = [ sys.executable, '-m', 'pytest', @@ -219,8 +212,7 @@ jobs: '--maxfail=999999' ] - # Run with complete OS-level silence - test_result = run_with_complete_silence(test_cmd) + run_with_complete_silence(test_cmd) if os.path.exists('pr_results.json'): print("Test results generated successfully") @@ -412,7 +404,6 @@ jobs: - name: Run tests on target branch if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - # Use Python with OS-level file descriptor redirection for complete silence python3 << 'EOF' import subprocess import sys @@ -421,21 +412,18 @@ jobs: def run_with_complete_silence(cmd, timeout=1800): """Run command with OS-level output suppression""" try: - # Save original file descriptors original_stdout = os.dup(1) original_stderr = os.dup(2) original_stdin = os.dup(0) try: - # Redirect to /dev/null at OS level devnull = os.open(os.devnull, os.O_WRONLY) devnull_read = os.open(os.devnull, os.O_RDONLY) - os.dup2(devnull, 1) # stdout - os.dup2(devnull, 2) # stderr - os.dup2(devnull_read, 0) # stdin + os.dup2(devnull, 1) + os.dup2(devnull, 2) + os.dup2(devnull_read, 0) - # Run the command result = subprocess.run( cmd, timeout=timeout, @@ -452,7 +440,6 @@ jobs: os.close(devnull_read) finally: - # Restore original file descriptors os.dup2(original_stdout, 1) os.dup2(original_stderr, 2) os.dup2(original_stdin, 0) @@ -467,7 +454,6 @@ jobs: except Exception as e: return subprocess.CompletedProcess(cmd, 1) - # Run pytest with complete OS-level silence test_cmd = [ sys.executable, '-m', 'pytest', '--json-report', @@ -483,10 +469,8 @@ jobs: '--maxfail=999999' ] - # Run with complete OS-level silence - test_result = run_with_complete_silence(test_cmd) + run_with_complete_silence(test_cmd) - # Check results if os.path.exists('target_results.json'): print("Test results generated successfully") else: From e2d82587597a576f304a5cff3927aead5e2dbd68 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 10:08:22 -0600 Subject: [PATCH 09/44] Remove old pytest workflow; Add usage; --- .github/workflows/test-py-pytest.yml | 1196 +++++++------------------- .github/workflows/test-pytest.yml | 952 -------------------- README.md | 4 +- usage.mmd | 71 ++ 4 files changed, 383 insertions(+), 1840 deletions(-) delete mode 100644 .github/workflows/test-pytest.yml create mode 100644 usage.mmd diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index 155aa61..1d853bd 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -38,9 +38,6 @@ on: pr_percentage: description: "Pass percentage in PR/source branch" value: ${{ jobs.test-source-branch.outputs.percentage }} - pr_failing_tests: - description: "List of failing tests in PR/source branch (JSON string)" - value: ${{ jobs.test-source-branch.outputs.failing_tests }} pr_collection_errors: description: "PR branch has collection errors" value: ${{ jobs.test-source-branch.outputs.collection_errors }} @@ -56,9 +53,6 @@ on: target_percentage: description: "Pass percentage in target branch" value: ${{ jobs.test-target-branch.outputs.percentage }} - target_passing_tests: - description: "List of passing tests in target branch (JSON string)" - value: ${{ jobs.test-target-branch.outputs.passing_tests }} has_regressions: description: "Boolean indicating if regressions were found" value: ${{ jobs.compare-results.outputs.has_regressions }} @@ -67,13 +61,7 @@ on: value: ${{ jobs.compare-results.outputs.regression_count }} jobs: - lint: - uses: ./.github/workflows/test-py-lint.yml - permissions: - contents: write # Lint job might push changes - test-source-branch: - needs: lint runs-on: ${{ inputs.runs_on }} outputs: total: ${{ steps.extract-results.outputs.total }} @@ -84,12 +72,9 @@ jobs: has_errors: ${{ steps.check-collection.outputs.has_errors }} error_type: ${{ steps.check-collection.outputs.error_type }} error_details: ${{ steps.check-collection.outputs.error_details }} - failing_tests: ${{ steps.extract-results.outputs.failing_tests }} - skipped_tests: ${{ steps.extract-results.outputs.skipped_tests }} - xfailed_tests: ${{ steps.extract-results.outputs.xfailed_tests }} - all_tests: ${{ steps.extract-results.outputs.all_tests }} - skipped_tests_with_reasons: ${{ steps.extract-results.outputs.skipped_tests_with_reasons }} - xfailed_tests_with_reasons: ${{ steps.extract-results.outputs.xfailed_tests_with_reasons }} + failing_count: ${{ steps.extract-results.outputs.failing_count }} + skipped_count: ${{ steps.extract-results.outputs.skipped_count }} + xfailed_count: ${{ steps.extract-results.outputs.xfailed_count }} steps: - name: Checkout PR Branch @@ -104,141 +89,147 @@ jobs: - name: Install dependencies run: | - set -x - python -m pip install --upgrade pip - pip install pytest pytest-json-report pytest-asyncio + python -m pip install --upgrade pip --quiet + pip install pytest pytest-json-report pytest-asyncio --quiet if [ -f requirements.txt ]; then - pip install -r requirements.txt + pip install -r requirements.txt --quiet fi - - name: Check for test collection errors + - name: Silent pytest execution and collection check id: check-collection run: | - set -x - # Create verbose debug file for artifact - exec 3>&1 4>&2 - exec 1> >(tee -a debug_collection.log) 2>&1 - - echo "Running pytest collection check..." - python -m pytest --collect-only -v > collection_output.txt 2>&1 || true - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - # Set default values - HAS_COLLECTION_ERRORS="false" - NO_TESTS_FOUND="false" - ERROR_TYPE="none" - ERROR_DETAILS="none" - - if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt; then - echo "::error::Test discovery errors detected in PR branch - Python modules could not be imported correctly" - - # Attempt to identify specific error type - if grep -q "ImportError" collection_output.txt; then - ERROR_TYPE="ImportError" - elif grep -q "ModuleNotFoundError" collection_output.txt; then - ERROR_TYPE="ModuleNotFoundError" - elif grep -q "SyntaxError" collection_output.txt; then - ERROR_TYPE="SyntaxError" - elif grep -q "ERROR collecting" collection_output.txt; then - ERROR_TYPE="CollectionError" - elif grep -q "Interrupted:" collection_output.txt; then - ERROR_TYPE="Interrupted" - else - ERROR_TYPE="UnknownError" - fi - - echo "PR branch discovery error type: $ERROR_TYPE" - - ERROR_FILE=$(grep -o "ERROR collecting.*\.py" collection_output.txt | grep -o "[a-zA-Z0-9_/]*\.py" || echo "Unknown file") - - if [[ "$ERROR_FILE" != "Unknown file" ]]; then - echo "Error in file $ERROR_FILE" - grep -A 15 "$ERROR_FILE" collection_output.txt > error_details.txt - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') - else - # If we couldn't find a specific file, get general error info - grep -A 15 "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt | head -20 > error_details.txt - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') - fi - - echo "::error::PR branch discovery error details: ${ERROR_DETAILS:0:200}..." - HAS_COLLECTION_ERRORS="true" - else - echo "No discovery errors detected in PR branch" - - TEST_COUNT=$(grep -o "collected [0-9]* item" collection_output.txt | grep -o "[0-9]*" || echo "0") - - if [[ "$TEST_COUNT" == "0" ]]; then - echo "::warning::No tests were found in the PR branch" - NO_TESTS_FOUND="true" - ERROR_TYPE="NoTestsFound" - ERROR_DETAILS="No test files were discovered that match pytest's test discovery pattern" - else - echo "Found $TEST_COUNT tests in PR branch" - fi - fi - - # Set all the outputs - echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT - echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT - echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT - echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT - - # For backward compatibility - if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then - echo "has_errors=true" >> $GITHUB_OUTPUT - else - echo "has_errors=false" >> $GITHUB_OUTPUT - fi + python3 << 'EOF' + import subprocess + import sys + import os - # Save full collection output to debug file for artifact - echo "=== FULL COLLECTION OUTPUT ===" >> debug_collection.log - cat collection_output.txt >> debug_collection.log + def run_with_complete_silence(cmd, timeout=1800): + """Run command with OS-level output suppression""" + try: + original_stdout = os.dup(1) + original_stderr = os.dup(2) + original_stdin = os.dup(0) + + try: + devnull = os.open(os.devnull, os.O_WRONLY) + devnull_read = os.open(os.devnull, os.O_RDONLY) + + os.dup2(devnull, 1) + os.dup2(devnull, 2) + os.dup2(devnull_read, 0) + + result = subprocess.run( + cmd, + timeout=timeout, + env={**os.environ, + 'PYTHONUNBUFFERED': '0', + 'PYTHONWARNINGS': 'ignore', + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', + 'PYTEST_CURRENT_TEST': '', + 'COLUMNS': '80', + 'PYTHONDONTWRITEBYTECODE': '1'} + ) + + os.close(devnull) + os.close(devnull_read) + + finally: + os.dup2(original_stdout, 1) + os.dup2(original_stderr, 2) + os.dup2(original_stdin, 0) + os.close(original_stdout) + os.close(original_stderr) + os.close(original_stdin) + + return result + + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(cmd, 1) + except Exception as e: + return subprocess.CompletedProcess(cmd, 1) + + def run_with_captured_output(cmd, timeout=60): + """Run command and capture output for error checking""" + try: + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + env={**os.environ, 'PYTHONWARNINGS': 'ignore'} + ) + except Exception: + return subprocess.CompletedProcess(cmd, 1, stderr="Execution failed") + + # Collection check + collection_cmd = [sys.executable, '-m', 'pytest', '--collect-only', '--quiet', '--tb=no'] + collection_result = run_with_captured_output(collection_cmd) + + has_collection_errors = "false" + no_tests_found = "false" + error_type = "none" + error_details = "none" + + if collection_result.returncode != 0: + if any(err in collection_result.stderr for err in ['ImportError', 'ModuleNotFoundError', 'SyntaxError', 'ERROR collecting']): + has_collection_errors = "true" + if 'ImportError' in collection_result.stderr: + error_type = "ImportError" + elif 'ModuleNotFoundError' in collection_result.stderr: + error_type = "ModuleNotFoundError" + elif 'SyntaxError' in collection_result.stderr: + error_type = "SyntaxError" + else: + error_type = "CollectionError" + error_details = collection_result.stderr[:200] if collection_result.stderr else "Collection failed" + else: + if 'collected 0 items' in collection_result.stdout or not collection_result.stdout.strip(): + no_tests_found = "true" + error_type = "NoTestsFound" + error_details = "No test files discovered" - - name: Run tests on PR Branch - if: steps.check-collection.outputs.has_collection_errors != 'true' - run: | - set -x - # Create verbose debug file for artifact - exec 3>&1 4>&2 - exec 1> >(tee -a debug_test_run.log) 2>&1 - - echo "Running tests on PR branch..." - python -m pytest -vv --json-report --json-report-file=pr_results.json || true - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - if [ -f pr_results.json ]; then - echo "Test results file successfully created for PR branch" - # Save first 200 chars to debug file only - echo "=== JSON RESULTS PREVIEW ===" >> debug_test_run.log - head -c 200 pr_results.json >> debug_test_run.log - else - echo "::error::Failed to create test results file for PR branch" - fi + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f'has_collection_errors={has_collection_errors}\n') + f.write(f'no_tests_found={no_tests_found}\n') + f.write(f'error_type={error_type}\n') + f.write(f'error_details={error_details}\n') + f.write(f'has_errors={str(has_collection_errors == "true" or no_tests_found == "true").lower()}\n') + + # Run tests silently if collection succeeded + if has_collection_errors == "false": + test_cmd = [ + sys.executable, '-m', 'pytest', + '--json-report', + '--json-report-file=pr_results.json', + '--tb=no', + '--no-header', + '--no-summary', + '--quiet', + '--disable-warnings', + '--log-level=CRITICAL', + '--log-cli-level=CRITICAL', + '--capture=sys', + '--maxfail=999999' + ] + + run_with_complete_silence(test_cmd) + + if os.path.exists('pr_results.json'): + print("Test results generated successfully") + else: + print("::error::Failed to create test results file") + else: + print(f"::error::Skipping tests due to collection errors: {error_type}") + EOF - - name: Extract test results + - name: Extract test results and create artifacts id: extract-results run: | - set -x - echo "PR_BRANCH=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV - echo "Processing test results for PR branch: $PR_BRANCH" - - # Create debug file for detailed output - exec 3>&1 4>&2 - exec 1> >(tee -a debug_extract_results.log) 2>&1 - python -c " import json import sys import os - print('Starting test results extraction script for PR branch') - - # Default values in case file doesn't exist or is invalid pr_total = 0 pr_passed = 0 pr_percentage = 0 @@ -250,52 +241,37 @@ jobs: xfailed_tests_with_reasons = {} try: - print('Attempting to open pr_results.json') with open('pr_results.json') as f: pr_results = json.load(f) - print(f'JSON loaded successfully, keys: {list(pr_results.keys())}') - # Check for collection errors by looking at exitcode or error patterns if pr_results.get('exitcode', 0) > 1: - print('Detected non-zero exitcode, likely a collection error') - if 'collectors' in pr_results and pr_results['collectors']: - print(f'Collection errors found: {pr_results[\"collectors\"]}') - pr_total = 0 # Explicitly set to 0 - no tests run when collection fails + pr_total = 0 pr_passed = 0 elif 'summary' in pr_results and isinstance(pr_results['summary'], dict): - # Normal case - extract data from summary summary = pr_results['summary'] pr_total = summary.get('total', 0) pr_passed = summary.get('passed', 0) - print(f'Results extracted from summary - Total: {pr_total}, Passed: {pr_passed}') - # Extract all tests by outcome and collect all test nodeids with reasons if 'tests' in pr_results: - print('Extracting failing, skipped, xfailed, and all tests with reasons') for test in pr_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') if nodeid: - all_tests.append(nodeid) # Track all tests regardless of outcome + all_tests.append(nodeid) if outcome in ['failed', 'error']: failing_tests.append(nodeid) elif outcome == 'skipped': skipped_tests.append(nodeid) - # Extract skip reason skip_reason = 'No reason provided' if 'longrepr' in test and test['longrepr']: - # longrepr can be a string or list, handle both longrepr = test['longrepr'] if isinstance(longrepr, list) and longrepr: skip_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' elif isinstance(longrepr, str): skip_reason = longrepr - elif 'call' in test and test['call'] and 'longrepr' in test['call']: - skip_reason = str(test['call']['longrepr']) skipped_tests_with_reasons[nodeid] = skip_reason.strip() elif outcome == 'xfailed': xfailed_tests.append(nodeid) - # Extract xfail reason xfail_reason = 'No reason provided' if 'longrepr' in test and test['longrepr']: longrepr = test['longrepr'] @@ -303,144 +279,47 @@ jobs: xfail_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' elif isinstance(longrepr, str): xfail_reason = longrepr - elif 'call' in test and test['call'] and 'longrepr' in test['call']: - xfail_reason = str(test['call']['longrepr']) xfailed_tests_with_reasons[nodeid] = xfail_reason.strip() - - print(f'Found {len(failing_tests)} failing tests') - print(f'Found {len(skipped_tests)} skipped tests') - print(f'Found {len(xfailed_tests)} xfailed tests') - print(f'Found {len(all_tests)} total discovered tests') - else: - print('No valid summary structure found') - # Calculate percentage safely pr_percentage = (pr_passed / pr_total * 100) if pr_total > 0 else 0 - print(f'Pass percentage calculated: {pr_percentage:.2f}%') - except FileNotFoundError as e: - print(f'File not found error: {e}') - except KeyError as e: - print(f'Missing key in results file: {e}') - if 'pr_results' in locals(): - print(f'Available keys: {list(pr_results.keys())}') - if 'summary' in pr_results: - print(f'Summary structure: {pr_results[\"summary\"]}') except Exception as e: - print(f'Error processing results: {e}') - import traceback - print(f'Full exception: {traceback.format_exc()}') - - print(f'Total tests: {pr_total}') - print(f'Passed tests: {pr_passed}') - print(f'Pass percentage: {pr_percentage:.2f}%') - print(f'Failing tests: {len(failing_tests)}') - print(f'Skipped tests: {len(skipped_tests)}') - print(f'Xfailed tests: {len(xfailed_tests)}') - print(f'All discovered tests: {len(all_tests)}') - - # Set outputs for GitHub Actions - print('Writing results to GITHUB_OUTPUT') + pass + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'total={pr_total}\\n') f.write(f'passed={pr_passed}\\n') f.write(f'percentage={pr_percentage:.2f}\\n') - # Write test lists as compact JSON strings to avoid issues with large outputs - if failing_tests: - f.write(f'failing_tests={json.dumps(failing_tests)}\\n') - else: - f.write('failing_tests=[]\\n') - if skipped_tests: - f.write(f'skipped_tests={json.dumps(skipped_tests)}\\n') - else: - f.write('skipped_tests=[]\\n') - if xfailed_tests: - f.write(f'xfailed_tests={json.dumps(xfailed_tests)}\\n') - else: - f.write('xfailed_tests=[]\\n') - if all_tests: - f.write(f'all_tests={json.dumps(all_tests)}\\n') - else: - f.write('all_tests=[]\\n') - # Write test reason mappings - if skipped_tests_with_reasons: - f.write(f'skipped_tests_with_reasons={json.dumps(skipped_tests_with_reasons)}\\n') - else: - f.write('skipped_tests_with_reasons={}\\n') - if xfailed_tests_with_reasons: - f.write(f'xfailed_tests_with_reasons={json.dumps(xfailed_tests_with_reasons)}\\n') - else: - f.write('xfailed_tests_with_reasons={}\\n') - - print('Results extraction completed') + f.write(f'failing_count={len(failing_tests)}\\n') + f.write(f'skipped_count={len(skipped_tests)}\\n') + f.write(f'xfailed_count={len(xfailed_tests)}\\n') + + test_data = { + 'failing_tests': failing_tests, + 'skipped_tests': skipped_tests, + 'xfailed_tests': xfailed_tests, + 'all_tests': all_tests, + 'skipped_tests_with_reasons': skipped_tests_with_reasons, + 'xfailed_tests_with_reasons': xfailed_tests_with_reasons + } + + with open('pr_test_data.json', 'w') as f: + json.dump(test_data, f, indent=2) " - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "PR branch test results processed: ${{ steps.extract-results.outputs.passed }}/${{ steps.extract-results.outputs.total }} tests passed (${{ steps.extract-results.outputs.percentage }}%)" - - - name: Upload PR branch debug logs + - name: Upload PR branch artifacts if: always() uses: actions/upload-artifact@v4 with: - name: pr_branch_debug_logs_${{ github.event.pull_request.number || github.run_id }} + name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | - debug_*.log - collection_output.txt - error_details.txt + pr_test_data.json pr_results.json + collection_check.txt retention-days: 3 if-no-files-found: ignore - - name: Create PR branch test data artifact - if: always() - run: | - set -x - echo "Creating PR branch test data for regression analysis..." - - # Create directory for artifact - mkdir -p pr_test_data - - # Extract failing tests from outputs and save to JSON file - python3 - << 'EOF' - import json - import os - - # Get failing tests from the step outputs (if available) - failing_tests_str = '''${{ steps.extract-results.outputs.failing_tests || '[]' }}''' - - try: - if failing_tests_str and failing_tests_str != '[]': - failing_tests = json.loads(failing_tests_str) - else: - failing_tests = [] - - print(f"Saving {len(failing_tests)} failing tests to artifact") - - # Save to JSON file for artifact - with open('pr_test_data/failing_items.json', 'w') as f: - json.dump(failing_tests, f, indent=2) - - print("PR branch test data saved successfully") - - except Exception as e: - print(f"Error saving PR branch test data: {e}") - # Create empty file as fallback - with open('pr_test_data/failing_items.json', 'w') as f: - json.dump([], f) - EOF - - - name: Upload PR branch test data artifact - uses: actions/upload-artifact@v4 - if: always() - with: - name: pr_branch_test_data_${{ github.event.pull_request.number || github.run_id }} - path: pr_test_data/ - retention-days: 1 - test-target-branch: - needs: lint runs-on: ${{ inputs.runs_on }} outputs: total: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.total || steps.extract-results.outputs.total }} @@ -451,8 +330,7 @@ jobs: has_errors: ${{ steps.check-collection.outputs.has_errors }} error_type: ${{ steps.check-collection.outputs.error_type }} error_details: ${{ steps.check-collection.outputs.error_details }} - passing_tests: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.passing_tests || steps.extract-results.outputs.passing_tests }} - all_tests: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.all_tests || steps.extract-results.outputs.all_tests }} + passing_count: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.passing_count || steps.extract-results.outputs.passing_count }} steps: - name: Checkout target branch @@ -468,142 +346,146 @@ jobs: - name: Install dependencies run: | - set -x - python -m pip install --upgrade pip - pip install pytest pytest-json-report pytest-asyncio + python -m pip install --upgrade pip --quiet + pip install pytest pytest-json-report pytest-asyncio --quiet if [ -f requirements.txt ]; then - pip install -r requirements.txt + pip install -r requirements.txt --quiet fi - name: Check for test collection errors id: check-collection run: | - set -x - # Create verbose debug file for artifact - exec 3>&1 4>&2 - exec 1> >(tee -a debug_target_collection.log) 2>&1 - - echo "Running pytest collection check..." - python -m pytest --collect-only -v > collection_output.txt 2>&1 || true - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - # Set default values HAS_COLLECTION_ERRORS="false" NO_TESTS_FOUND="false" ERROR_TYPE="none" ERROR_DETAILS="none" - if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt; then - echo "::warning::Test discovery errors detected in target branch - Python modules could not be imported correctly" - - # Attempt to identify specific error type - if grep -q "ImportError" collection_output.txt; then - ERROR_TYPE="ImportError" - elif grep -q "ModuleNotFoundError" collection_output.txt; then - ERROR_TYPE="ModuleNotFoundError" - elif grep -q "SyntaxError" collection_output.txt; then - ERROR_TYPE="SyntaxError" - elif grep -q "ERROR collecting" collection_output.txt; then - ERROR_TYPE="CollectionError" - elif grep -q "Interrupted:" collection_output.txt; then - ERROR_TYPE="Interrupted" - else - ERROR_TYPE="UnknownError" - fi - - echo "Target branch discovery error type: $ERROR_TYPE" - - ERROR_FILE=$(grep -o "ERROR collecting.*\.py" collection_output.txt | grep -o "[a-zA-Z0-9_/]*\.py" || echo "Unknown file") - - if [[ "$ERROR_FILE" != "Unknown file" ]]; then - echo "Error in file $ERROR_FILE" - grep -A 15 "$ERROR_FILE" collection_output.txt > error_details.txt - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') - else - # If we couldn't find a specific file, get general error info - grep -A 15 "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt | head -20 > error_details.txt - ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') + python -m pytest --collect-only --quiet --tb=no >/dev/null 2>collection_check.txt + COLLECTION_EXIT_CODE=$? + + if [ $COLLECTION_EXIT_CODE -ne 0 ]; then + if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting" collection_check.txt; then + echo "::warning::Test discovery errors detected in target branch" + HAS_COLLECTION_ERRORS="true" + + if grep -q "ImportError" collection_check.txt; then + ERROR_TYPE="ImportError" + elif grep -q "ModuleNotFoundError" collection_check.txt; then + ERROR_TYPE="ModuleNotFoundError" + elif grep -q "SyntaxError" collection_check.txt; then + ERROR_TYPE="SyntaxError" + else + ERROR_TYPE="CollectionError" + fi + + ERROR_DETAILS=$(head -1 collection_check.txt | tr '\n' ' ' | cut -c1-200) fi - - echo "::warning::Target branch discovery error details: ${ERROR_DETAILS:0:200}..." - HAS_COLLECTION_ERRORS="true" else - echo "No discovery errors detected in target branch" - - TEST_COUNT=$(grep -o "collected [0-9]* item" collection_output.txt | grep -o "[0-9]*" || echo "0") - + TEST_COUNT=$(python -m pytest --collect-only --quiet 2>/dev/null | grep -o "collected [0-9]* item" | grep -o "[0-9]*" || echo "0") if [[ "$TEST_COUNT" == "0" ]]; then echo "::warning::No tests were found in the target branch" NO_TESTS_FOUND="true" ERROR_TYPE="NoTestsFound" - ERROR_DETAILS="No test files were discovered in target branch that match pytest's test discovery pattern" - else - echo "Found $TEST_COUNT tests in target branch" + ERROR_DETAILS="No test files discovered" fi fi - # Set all the outputs echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT - # For backward compatibility if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then echo "has_errors=true" >> $GITHUB_OUTPUT else echo "has_errors=false" >> $GITHUB_OUTPUT fi - # Save full collection output to debug file for artifact - echo "=== FULL COLLECTION OUTPUT ===" >> debug_target_collection.log - cat collection_output.txt >> debug_target_collection.log - - name: Run tests on target branch if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - set -x - # Create verbose debug file for artifact - exec 3>&1 4>&2 - exec 1> >(tee -a debug_target_test_run.log) 2>&1 - - echo "Running tests on target branch..." - python -m pytest -vv --json-report --json-report-file=target_results.json || true - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - if [ -f target_results.json ]; then - echo "Test results file successfully created for target branch" - # Save first 200 chars to debug file only - echo "=== JSON RESULTS PREVIEW ===" >> debug_target_test_run.log - head -c 200 target_results.json >> debug_target_test_run.log - else - echo "::warning::Failed to create test results file for target branch" - fi + python3 << 'EOF' + import subprocess + import sys + import os + + def run_with_complete_silence(cmd, timeout=1800): + """Run command with OS-level output suppression""" + try: + original_stdout = os.dup(1) + original_stderr = os.dup(2) + original_stdin = os.dup(0) + + try: + devnull = os.open(os.devnull, os.O_WRONLY) + devnull_read = os.open(os.devnull, os.O_RDONLY) + + os.dup2(devnull, 1) + os.dup2(devnull, 2) + os.dup2(devnull_read, 0) + + result = subprocess.run( + cmd, + timeout=timeout, + env={**os.environ, + 'PYTHONUNBUFFERED': '0', + 'PYTHONWARNINGS': 'ignore', + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', + 'PYTEST_CURRENT_TEST': '', + 'COLUMNS': '80', + 'PYTHONDONTWRITEBYTECODE': '1'} + ) + + os.close(devnull) + os.close(devnull_read) + + finally: + os.dup2(original_stdout, 1) + os.dup2(original_stderr, 2) + os.dup2(original_stdin, 0) + os.close(original_stdout) + os.close(original_stderr) + os.close(original_stdin) + + return result + + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(cmd, 1) + except Exception as e: + return subprocess.CompletedProcess(cmd, 1) + + test_cmd = [ + sys.executable, '-m', 'pytest', + '--json-report', + '--json-report-file=target_results.json', + '--tb=no', + '--no-header', + '--no-summary', + '--quiet', + '--disable-warnings', + '--log-level=CRITICAL', + '--log-cli-level=CRITICAL', + '--capture=sys', + '--maxfail=999999' + ] + + run_with_complete_silence(test_cmd) + + if os.path.exists('target_results.json'): + print("Test results generated successfully") + else: + print("::warning::Failed to create test results file") + EOF - - name: Extract test results + - name: Extract test results and create artifacts id: extract-results - # Only run if there were no collection errors if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - set -x - echo "Processing test results for target branch: ${{ inputs.target_branch_to_compare }}" - - # Create debug file for detailed output - exec 3>&1 4>&2 - exec 1> >(tee -a debug_target_extract_results.log) 2>&1 - python -c " import json import sys import os - print('Starting test results extraction script for target branch') - - # Default values in case file doesn't exist or is invalid target_total = 0 target_passed = 0 target_percentage = 0 @@ -611,158 +493,66 @@ jobs: all_tests = [] try: - print('Attempting to open target_results.json') with open('target_results.json') as f: target_results = json.load(f) - print(f'JSON loaded successfully, keys: {list(target_results.keys())}') - # Check for collection errors by looking at exitcode or error patterns if target_results.get('exitcode', 0) > 1: - print('Detected non-zero exitcode, likely a collection error') - if 'collectors' in target_results and target_results['collectors']: - print(f'Collection errors found: {target_results[\"collectors\"]}') - target_total = 0 # Explicitly set to 0 - no tests run when collection fails + target_total = 0 target_passed = 0 elif 'summary' in target_results and isinstance(target_results['summary'], dict): - # Normal case - extract data from summary summary = target_results['summary'] target_total = summary.get('total', 0) target_passed = summary.get('passed', 0) - print(f'Results extracted from summary - Total: {target_total}, Passed: {target_passed}') - # Extract passing tests and all tests if 'tests' in target_results: - print('Extracting passing tests and all discovered tests') for test in target_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') if nodeid: - all_tests.append(nodeid) # Track all tests regardless of outcome + all_tests.append(nodeid) if outcome == 'passed': passing_tests.append(nodeid) - - print(f'Found {len(passing_tests)} passing tests') - print(f'Found {len(all_tests)} total discovered tests') - else: - print('No valid summary structure found') - # Calculate percentage safely target_percentage = (target_passed / target_total * 100) if target_total > 0 else 0 - print(f'Pass percentage calculated: {target_percentage:.2f}%') - except FileNotFoundError as e: - print(f'File not found error: {e}') - except KeyError as e: - print(f'Missing key in results file: {e}') - if 'target_results' in locals(): - print(f'Available keys: {list(target_results.keys())}') - if 'summary' in target_results: - print(f'Summary structure: {target_results[\"summary\"]}') except Exception as e: - print(f'Error processing results: {e}') - import traceback - print(f'Full exception: {traceback.format_exc()}') - - print(f'Total tests: {target_total}') - print(f'Passed tests: {target_passed}') - print(f'Pass percentage: {target_percentage:.2f}%') - print(f'Passing tests: {len(passing_tests)}') - print(f'All discovered tests: {len(all_tests)}') - - # Set outputs for GitHub Actions - print('Writing results to GITHUB_OUTPUT') + pass + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'total={target_total}\\n') f.write(f'passed={target_passed}\\n') f.write(f'percentage={target_percentage:.2f}\\n') - # Write test lists as compact JSON strings to avoid issues with large outputs - if passing_tests: - f.write(f'passing_tests={json.dumps(passing_tests)}\\n') - else: - f.write('passing_tests=[]\\n') - if all_tests: - f.write(f'all_tests={json.dumps(all_tests)}\\n') - else: - f.write('all_tests=[]\\n') + f.write(f'passing_count={len(passing_tests)}\\n') - print('Results extraction completed') - " - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 + test_data = { + 'passing_tests': passing_tests, + 'all_tests': all_tests + } - echo "Target branch test results processed: ${{ steps.extract-results.outputs.passed }}/${{ steps.extract-results.outputs.total }} tests passed (${{ steps.extract-results.outputs.percentage }}%)" + with open('target_test_data.json', 'w') as f: + json.dump(test_data, f, indent=2) + " - - name: Upload target branch debug logs + - name: Upload target branch artifacts if: always() uses: actions/upload-artifact@v4 with: - name: target_branch_debug_logs_${{ github.event.pull_request.number || github.run_id }} + name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | - debug_target_*.log - collection_output.txt - error_details.txt + target_test_data.json target_results.json + collection_check.txt retention-days: 3 if-no-files-found: ignore - - name: Create target branch test data artifact - if: always() - run: | - set -x - echo "Creating target branch test data for regression analysis..." - - # Create directory for artifact - mkdir -p target_test_data - - # Extract passing tests from outputs and save to JSON file - python3 - << 'EOF' - import json - import os - - # Get passing tests from the step outputs (if available) - passing_tests_str = '''${{ steps.extract-results.outputs.passing_tests || steps.set-error-outputs.outputs.passing_tests || '[]' }}''' - - try: - if passing_tests_str and passing_tests_str != '[]': - passing_tests = json.loads(passing_tests_str) - else: - passing_tests = [] - - print(f"Saving {len(passing_tests)} passing tests to artifact") - - # Save to JSON file for artifact - with open('target_test_data/passing_items.json', 'w') as f: - json.dump(passing_tests, f, indent=2) - - print("Target branch test data saved successfully") - - except Exception as e: - print(f"Error saving target branch test data: {e}") - # Create empty file as fallback - with open('target_test_data/passing_items.json', 'w') as f: - json.dump([], f) - EOF - - - name: Upload target branch test data artifact - uses: actions/upload-artifact@v4 - if: always() - with: - name: target_branch_test_data_${{ github.event.pull_request.number || github.run_id }} - path: target_test_data/ - retention-days: 1 - - # Add a step to set default outputs when collection errors are detected - name: Set collection error outputs id: set-error-outputs if: steps.check-collection.outputs.has_collection_errors == 'true' run: | - echo "::warning::Setting default outputs for target branch due to collection errors" echo "total=0" >> $GITHUB_OUTPUT echo "passed=0" >> $GITHUB_OUTPUT echo "percentage=0.00" >> $GITHUB_OUTPUT - echo "passing_tests=[]" >> $GITHUB_OUTPUT - echo "all_tests=[]" >> $GITHUB_OUTPUT + echo "passing_count=0" >> $GITHUB_OUTPUT compare-results: needs: [test-source-branch, test-target-branch] @@ -777,104 +567,71 @@ jobs: sudo apt-get update -y sudo apt-get install -y bc + - name: Download test data artifacts + uses: actions/download-artifact@v4 + with: + pattern: "*_branch_data_${{ github.event.pull_request.number || github.run_id }}" + path: ./artifacts + merge-multiple: false + - name: Check for collection errors run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_comparison_analysis.log) 2>&1 - - echo "Retrieving collection error status information" PR_COLLECTION_ERRORS="${{ needs.test-source-branch.outputs.collection_errors }}" PR_NO_TESTS="${{ needs.test-source-branch.outputs.no_tests_found }}" PR_ERROR_TYPE="${{ needs.test-source-branch.outputs.error_type }}" - PR_ERROR_DETAILS="${{ needs.test-source-branch.outputs.error_details }}" TARGET_COLLECTION_ERRORS="${{ needs.test-target-branch.outputs.collection_errors }}" - echo "PR branch collection errors: $PR_COLLECTION_ERRORS" - echo "PR branch no tests found: $PR_NO_TESTS" - echo "PR branch error type: $PR_ERROR_TYPE" - echo "Target branch collection errors: $TARGET_COLLECTION_ERRORS" - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - # Distinct error handling for PR branch if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then echo "::error::Test discovery errors in PR branch: $PR_ERROR_TYPE" - echo "::error::$PR_ERROR_DETAILS" - echo "❌ PR branch has test discovery errors. Python modules could not be imported correctly." exit 1 fi if [[ "$PR_NO_TESTS" == "true" ]]; then echo "::error::No tests were found in the PR branch" - echo "❌ PR branch has no tests detected. Please add test files that match pytest's discovery pattern." exit 1 fi - # Warning for target branch issues (not a failure) if [[ "$TARGET_COLLECTION_ERRORS" == "true" ]]; then - echo "⚠️ Target branch has test discovery errors. Tests will still be compared but results may not be accurate." + echo "⚠️ Target branch has test discovery errors." fi - if [[ "${{ needs.test-target-branch.outputs.no_tests_found }}" == "true" ]]; then - echo "⚠️ Target branch has no tests detected. PR branch tests will still be evaluated." - fi - - # Split the regression check into separate steps for better control - # NOTE: These regression analysis steps are redundant with the perform-regression-analysis job - # but are kept for now to ensure backward compatibility. They should be removed in a future cleanup. - - name: Run regression analysis + - name: Run regression analysis from artifacts run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_regression_analysis.log) 2>&1 - - echo "Running regression analysis..." - python3 - << 'EOF' import json import os + import glob try: - # Parse the inputs - target_passing_str = '''${{ needs.test-target-branch.outputs.passing_tests }}''' - pr_failing_str = '''${{ needs.test-source-branch.outputs.failing_tests }}''' + target_data = {} + pr_data = {} + + target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') + if target_files: + with open(target_files[0], 'r') as f: + target_data = json.load(f) - # Parse JSON - target_passing = json.loads(target_passing_str) if target_passing_str and target_passing_str != '[]' else [] - pr_failing = json.loads(pr_failing_str) if pr_failing_str and pr_failing_str != '[]' else [] + pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') + if pr_files: + with open(pr_files[0], 'r') as f: + pr_data = json.load(f) - print(f"Parsed {len(target_passing)} passing tests from target branch") - print(f"Parsed {len(pr_failing)} failing tests from PR branch") + target_passing = target_data.get('passing_tests', []) + pr_failing = pr_data.get('failing_tests', []) - # Find regressions using set operations target_passing_set = set(target_passing) pr_failing_set = set(pr_failing) regression_tests = list(target_passing_set.intersection(pr_failing_set)) - # Write results to file if there are regressions if regression_tests: - print(f"Found {len(regression_tests)} regression(s)!") - with open("regression_details.txt", "w") as f: f.write(f"Found {len(regression_tests)} tests that were passing in target branch but now failing in PR branch:\\n\\n") for idx, test in enumerate(sorted(regression_tests), 1): f.write(f"{idx}. {test}\\n") - print("Regression details written to file") - else: - print("No regressions found") except Exception as e: - print(f"Error in regression analysis: {e}") - import traceback - print(traceback.format_exc()) + pass EOF - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Regression analysis completed" - - name: Check for regression details file id: check-regressions run: | @@ -882,209 +639,25 @@ jobs: _regression_count="0" if [ -f "regression_details.txt" ]; then - echo "Regression details file exists" - # Count regression lines (lines starting with a number and period) _current_count=$(grep -c "^[0-9]\+\." regression_details.txt || echo "0") - echo "Found $_current_count regression items in file" - if [ "$_current_count" -gt 0 ]; then _has_regressions="true" _regression_count="$_current_count" - echo "::error::Test Regressions Found: $_regression_count test(s) that were passing in target branch are now **failing** in PR branch." - echo "Regression details:" - cat regression_details.txt - else - # File exists but no regressions counted (e.g. empty or malformed) - _has_regressions="false" - _regression_count="0" + echo "::error::Test Regressions Found: $_regression_count test(s) that were passing in target branch are now failing in PR branch." fi - else - echo "No regression details file found - no regressions detected" - _has_regressions="false" - _regression_count="0" fi echo "HAS_REGRESSIONS=$_has_regressions" >> $GITHUB_OUTPUT echo "REGRESSION_COUNT=$_regression_count" >> $GITHUB_OUTPUT - if [[ "$_has_regressions" == "false" ]]; then - if [ -f regression_details.txt ] && [ "$_has_regressions" == "false" ]; then - echo "::notice::Regression details file (regression_details.txt) was found but no valid regression entries were counted by this step, or the file was empty." - else - echo "No test regressions detected by this step." - fi - fi - - name: Upload regression details artifact if: steps.check-regressions.outputs.HAS_REGRESSIONS == 'true' && steps.check-regressions.outputs.REGRESSION_COUNT > 0 uses: actions/upload-artifact@v4 with: - # The artifact name from reusable-regression-analyzer.yml name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests path: regression_details.txt retention-days: 1 - - name: Check for skip/xfail regressions - run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_skip_xfail_analysis.log) 2>&1 - - echo "Checking for tests that were passing in target but are now skipped or xfailed in PR" - - python3 - << 'EOF' - import json - import os - - try: - # Parse the inputs - target_passing_str = '''${{ needs.test-target-branch.outputs.passing_tests }}''' - pr_skipped_str = '''${{ needs.test-source-branch.outputs.skipped_tests }}''' - pr_xfailed_str = '''${{ needs.test-source-branch.outputs.xfailed_tests }}''' - pr_skipped_with_reasons_str = '''${{ needs.test-source-branch.outputs.skipped_tests_with_reasons }}''' - pr_xfailed_with_reasons_str = '''${{ needs.test-source-branch.outputs.xfailed_tests_with_reasons }}''' - - # Parse JSON - target_passing = json.loads(target_passing_str) if target_passing_str and target_passing_str != '[]' else [] - pr_skipped = json.loads(pr_skipped_str) if pr_skipped_str and pr_skipped_str != '[]' else [] - pr_xfailed = json.loads(pr_xfailed_str) if pr_xfailed_str and pr_xfailed_str != '[]' else [] - pr_skipped_with_reasons = json.loads(pr_skipped_with_reasons_str) if pr_skipped_with_reasons_str and pr_skipped_with_reasons_str != '{}' else {} - pr_xfailed_with_reasons = json.loads(pr_xfailed_with_reasons_str) if pr_xfailed_with_reasons_str and pr_xfailed_with_reasons_str != '{}' else {} - - print(f"Parsed {len(target_passing)} passing tests from target branch") - print(f"Parsed {len(pr_skipped)} skipped tests from PR branch") - print(f"Parsed {len(pr_xfailed)} xfailed tests from PR branch") - print(f"Parsed {len(pr_skipped_with_reasons)} skipped tests with reasons") - print(f"Parsed {len(pr_xfailed_with_reasons)} xfailed tests with reasons") - - # Find tests that were passing in target but are now skipped or xfailed in PR - target_passing_set = set(target_passing) - pr_skipped_set = set(pr_skipped) - pr_xfailed_set = set(pr_xfailed) - - passing_to_skipped = list(target_passing_set.intersection(pr_skipped_set)) - passing_to_xfailed = list(target_passing_set.intersection(pr_xfailed_set)) - - total_skip_xfail_regressions = len(passing_to_skipped) + len(passing_to_xfailed) - - if total_skip_xfail_regressions > 0: - print(f"Found {total_skip_xfail_regressions} tests that were passing in target but are now skipped/xfailed in PR!") - - # Build comprehensive warning message - warning_parts = [f"Skip/XFail Analysis: {total_skip_xfail_regressions} test(s) that were passing in target branch are now being skipped or xfailed in PR branch."] - - if passing_to_skipped: - warning_parts.append(f"Tests now SKIPPED ({len(passing_to_skipped)}):") - for idx, test in enumerate(sorted(passing_to_skipped), 1): - reason = pr_skipped_with_reasons.get(test, 'No reason provided') - warning_parts.append(f" {idx}. {test} - Reason: {reason}") - - if passing_to_xfailed: - warning_parts.append(f"Tests now XFAILED ({len(passing_to_xfailed)}):") - for idx, test in enumerate(sorted(passing_to_xfailed), 1): - reason = pr_xfailed_with_reasons.get(test, 'No reason provided') - warning_parts.append(f" {idx}. {test} - Reason: {reason}") - - warning_parts.append("While these changes don't fail the workflow, they indicate tests that were working before are now being bypassed. Please review these tests to ensure this is intentional.") - - # Print as single warning annotation - combined_warning = " ".join(warning_parts) - print(f"::warning::{combined_warning}") - else: - print("No skip/xfail regressions found - all previously passing tests are still running.") - except Exception as e: - print(f"Error in skip/xfail analysis: {e}") - import traceback - print(traceback.format_exc()) - EOF - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Skip/xfail regression analysis completed" - - - name: Check for test additions and removals - run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_test_changes_analysis.log) 2>&1 - - echo "Checking for test additions and removals between target and PR branches" - - python3 - << 'EOF' - import json - import os - - try: - # Parse the inputs - target_all_str = '''${{ needs.test-target-branch.outputs.all_tests }}''' - pr_all_str = '''${{ needs.test-source-branch.outputs.all_tests }}''' - - # Parse JSON - target_all = json.loads(target_all_str) if target_all_str and target_all_str != '[]' else [] - pr_all = json.loads(pr_all_str) if pr_all_str and pr_all_str != '[]' else [] - - print(f"Parsed {len(target_all)} total tests from target branch") - print(f"Parsed {len(pr_all)} total tests from PR branch") - - # Find test additions and removals using set operations - target_all_set = set(target_all) - pr_all_set = set(pr_all) - - removed_tests = list(target_all_set - pr_all_set) # In target but not in PR - added_tests = list(pr_all_set - target_all_set) # In PR but not in target - - # Report removed tests (warnings) - if removed_tests: - print(f"Found {len(removed_tests)} tests that were removed from target branch!") - - # Build comprehensive removal warning - removal_parts = [f"Test Removal Analysis: {len(removed_tests)} test(s) that existed in target branch are missing from PR branch."] - removal_parts.append(f"REMOVED Tests ({len(removed_tests)}):") - for idx, test in enumerate(sorted(removed_tests), 1): - removal_parts.append(f" {idx}. {test}") - removal_parts.append("These test removals should be reviewed to ensure they are intentional. If tests were renamed or moved, this may show as removal + addition.") - - # Print as single warning annotation - combined_removal_warning = " ".join(removal_parts) - print(f"::warning::{combined_removal_warning}") - else: - print("No test removals detected.") - - # Report added tests (notifications/info) - if added_tests: - print(f"Found {len(added_tests)} new tests added in PR branch!") - - # Build comprehensive addition notice - addition_parts = [f"Test Addition Analysis: {len(added_tests)} new test(s) have been added in the PR branch."] - addition_parts.append(f"NEW Tests ({len(added_tests)}):") - for idx, test in enumerate(sorted(added_tests), 1): - addition_parts.append(f" {idx}. {test}") - addition_parts.append("New tests detected - this indicates expanded test coverage!") - - # Print as single notice annotation - combined_addition_notice = " ".join(addition_parts) - print(f"::notice::{combined_addition_notice}") - else: - print("No new tests detected in PR branch.") - - # Summary - if not removed_tests and not added_tests: - print("Test suite composition is unchanged between target and PR branches.") - else: - print(f"Test suite changes: {len(added_tests)} added, {len(removed_tests)} removed") - - except Exception as e: - print(f"Error in test addition/removal analysis: {e}") - import traceback - print(traceback.format_exc()) - EOF - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Test addition/removal analysis completed" - - name: Compare test results run: | echo "Test Results Summary:" @@ -1093,7 +666,6 @@ jobs: if [[ "${{ needs.test-source-branch.outputs.total }}" == "0" ]]; then echo "::error::No tests were found in the PR branch" - echo "❌ PR branch has no tests detected. Please add test files that match pytest's discovery pattern." exit 1 fi @@ -1104,89 +676,29 @@ jobs: PR_TOTAL=${{ needs.test-source-branch.outputs.total }} TARGET_TOTAL=${{ needs.test-target-branch.outputs.total }} - # Handle case where target has no tests if [[ "$TARGET_TOTAL" == "0" ]]; then if [[ "$PR_PASSED" -gt 0 ]]; then echo "✅ PR branch has tests and some are passing (target branch has no tests)" exit 0 else echo "❌ PR branch has no passing tests" - echo " - Pass percentage: $PR_PERCENTAGE%" exit 1 fi fi - # Fail if any tests passed in target branch but now fail in PR branch if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then echo "❌ PR branch has test regressions from target branch" - REGRESSION_COUNT_VAL=${{ needs.perform-regression-analysis.outputs.regression_count }} - echo " - $REGRESSION_COUNT_VAL tests that were passing in target branch are now failing" - - echo "### :x: Test Regressions Detected!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**$REGRESSION_COUNT_VAL test(s) that were passing in the target branch are now failing in the PR branch.**" >> $GITHUB_STEP_SUMMARY - echo "This job (\`compare-results\`) has been marked as failed due to these regressions." >> $GITHUB_STEP_SUMMARY - - if [ -f regression_details.txt ]; then # Check if file exists (it should from previous step) - echo "Failing tests (regressions) list:" - cat regression_details.txt - else - echo "::warning::Regression details file (regression_details.txt) not found in Compare test results step. It might be available as an artifact from the 'perform-regression-analysis' job." - fi exit 1 fi - # Continue with the original comparison if no regressions if (( $(echo "$PR_PASSED >= $TARGET_PASSED" | bc -l) )) && (( $(echo "$PR_PERCENTAGE >= $TARGET_PERCENTAGE" | bc -l) )); then echo "✅ PR branch has equal or better test results than target branch" - - # Additional verbose information about improvement - if (( $(echo "$PR_PASSED > $TARGET_PASSED" | bc -l) )); then - IMPROVEMENT=$(( $PR_PASSED - $TARGET_PASSED )) - echo " - Improvement: $IMPROVEMENT more passing tests than target branch" - fi - - if (( $(echo "$PR_PERCENTAGE > $TARGET_PERCENTAGE" | bc -l) )); then - PERCENTAGE_IMPROVEMENT=$(echo "$PR_PERCENTAGE - $TARGET_PERCENTAGE" | bc -l) - echo " - Percentage improvement: +${PERCENTAGE_IMPROVEMENT}% compared to target branch" - fi - exit 0 else echo "❌ PR branch has worse test results than target branch" - echo " - Passed tests: $PR_PASSED vs $TARGET_PASSED on target branch" - echo " - Pass percentage: $PR_PERCENTAGE% vs $TARGET_PERCENTAGE% on target branch" - - # Add to job summary for general comparison failure - echo "### :x: Test Comparison Failed" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The PR branch has worse test results than the target branch:" >> $GITHUB_STEP_SUMMARY - echo "- Passed tests: $PR_PASSED (PR) vs $TARGET_PASSED (Target)" >> $GITHUB_STEP_SUMMARY - echo "- Pass percentage: $PR_PERCENTAGE% (PR) vs $TARGET_PERCENTAGE% (Target)" >> $GITHUB_STEP_SUMMARY - - # Calculate regression metrics - if (( $(echo "$PR_PASSED < $TARGET_PASSED" | bc -l) )); then - REGRESSION=$(( $TARGET_PASSED - $PR_PASSED )) - echo " - Regression: $REGRESSION fewer passing tests than target branch" - fi - - if (( $(echo "$PR_PERCENTAGE < $TARGET_PERCENTAGE" | bc -l) )); then - PERCENTAGE_REGRESSION=$(echo "$TARGET_PERCENTAGE - $PR_PERCENTAGE" | bc -l) - echo " - Percentage regression: -${PERCENTAGE_REGRESSION}% compared to target branch" - fi - exit 1 fi - - name: Upload comparison analysis logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: comparison_analysis_logs_${{ github.event.pull_request.number || github.run_id }} - path: debug_*_analysis.log - retention-days: 3 - if-no-files-found: ignore - perform-regression-analysis: needs: [test-source-branch, test-target-branch] uses: ./.github/workflows/meta-regression-analysis.yml @@ -1195,23 +707,18 @@ jobs: item_type_plural: "tests" pr_number: ${{ github.event.pull_request.number }} run_id: ${{ github.run_id }} - target_branch_artifact_name: target_branch_test_data_${{ github.event.pull_request.number || github.run_id }} - pr_branch_artifact_name: pr_branch_test_data_${{ github.event.pull_request.number || github.run_id }} - # Secrets are not needed for this reusable workflow currently - # secrets: inherit + target_branch_artifact_name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} + pr_branch_artifact_name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} - # Conditionally run notification job only if needed prepare-notification: name: Prepare Notification Data needs: [ - lint, test-source-branch, test-target-branch, compare-results, perform-regression-analysis, ] - # Notify on collection errors, no tests found, compare result failure, or if regressions are detected if: | always() && ( @@ -1233,26 +740,23 @@ jobs: id: check_webhook_availability run: | if [ -z "${{ secrets.DISCORD_WEBHOOK_URL }}" ]; then - echo "::notice::DISCORD_WEBHOOK_URL secret is not set. Discord notifications will likely be skipped by the alert workflow if it relies on this secret." echo "webhook_available=false" >> $GITHUB_OUTPUT else - echo "::debug::DISCORD_WEBHOOK_URL secret is present." echo "webhook_available=true" >> $GITHUB_OUTPUT fi + - name: Download regression details (if any) id: download_regressions if: needs.perform-regression-analysis.outputs.has_regressions == 'true' && needs.perform-regression-analysis.outputs.regression_count > 0 uses: actions/download-artifact@v4 with: - # The artifact name from reusable-regression-analyzer.yml name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests - path: . # Download to current directory + path: . continue-on-error: true - name: Construct Discord Notification id: construct_notification env: - LINT_RESULT: ${{ needs.lint.result }} SOURCE_TEST_RESULT: ${{ needs.test-source-branch.result }} TARGET_TEST_RESULT: ${{ needs.test-target-branch.result }} COMPARE_RESULT: ${{ needs.compare-results.result }} @@ -1276,82 +780,47 @@ jobs: REPO_URL: ${{ github.server_url }}/${{ github.repository }} ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_ASSIGNEES_JSON: ${{ toJson(github.event.pull_request.assignees) }} - USER_MAP_JSON: ${{ secrets.DISCORD_USER_MAP || '{}' }} + USER_MAP_JSON: ${{ secrets.DISCORD_USER_MAP }} REGRESSION_FILE_PATH: "regression_details.txt" DOWNLOAD_REGRESSIONS_OUTCOME: ${{ steps.download_regressions.outcome }} INPUT_PING_LATEST_COMMITTER: ${{ inputs.ping_latest_committer }} run: | - # Create debug file for detailed notification construction - exec 3>&1 4>&2 - exec 1> >(tee -a debug_notification_construction.log) 2>&1 - - MESSAGE_LINES=() # Use an array to build message lines - PING_KEYS_OUTPUT="" # Will be comma-separated GitHub logins + MESSAGE_LINES=() + PING_KEYS_OUTPUT="" ARTIFACT_PATH_OUTPUT="" - echo "Raw GH_ASSIGNEES_JSON value: [$GH_ASSIGNEES_JSON]" - echo "Raw USER_MAP_JSON value: [$USER_MAP_JSON]" - - # 1. Determine Pings - Collect GitHub Logins to pass to alert-discord.yml - # Initialize PING_KEYS_OUTPUT PING_KEYS_OUTPUT="" - # Add assignees to PING_KEYS_OUTPUT if [ -n "$USER_MAP_JSON" ] && [ "$USER_MAP_JSON" != "{}" ] && command -v jq &> /dev/null; then ASSIGNEE_LOGINS_ARRAY=($(echo "$GH_ASSIGNEES_JSON" | jq -r '.[].login // empty')) - echo "GH_ASSIGNEES_JSON received: $GH_ASSIGNEES_JSON" - echo "Extracted ASSIGNEE_LOGINS_ARRAY: (${ASSIGNEE_LOGINS_ARRAY[*]})" - echo "Count of assignees extracted: ${#ASSIGNEE_LOGINS_ARRAY[@]}" - MAPPED_ASSIGNEE_COUNT=0 TEMP_PING_KEYS=() for assignee_login in "${ASSIGNEE_LOGINS_ARRAY[@]}"; do if [ -z "$assignee_login" ]; then - echo "Skipping empty assignee login." continue fi - echo "Processing assignee for ping: '$assignee_login'" - # Check if this assignee_login exists as a key in USER_MAP_JSON if echo "$USER_MAP_JSON" | jq -e --arg K "$assignee_login" '.[$K]' > /dev/null; then - echo "Assignee '$assignee_login' FOUND in USER_MAP_JSON." TEMP_PING_KEYS+=("$assignee_login") MAPPED_ASSIGNEE_COUNT=$((MAPPED_ASSIGNEE_COUNT + 1)) - else - echo "Assignee '$assignee_login' NOT FOUND in USER_MAP_JSON." fi done - echo "Total assignees found in USER_MAP_JSON and added to pings: $MAPPED_ASSIGNEE_COUNT" - if [ ${#TEMP_PING_KEYS[@]} -gt 0 ]; then PING_KEYS_OUTPUT=$(IFS=,; echo "${TEMP_PING_KEYS[*]}") - echo "Initial PING_KEYS_OUTPUT from assignees: [$PING_KEYS_OUTPUT]" - else - echo "No assignees found or GH_ASSIGNEES_JSON was empty, or no assignees were found in USER_MAP_JSON." fi - elif [ -n "$USER_MAP_JSON" ] && [ "$USER_MAP_JSON" != "{}" ] && ! command -v jq &> /dev/null; then - echo "::warning::jq is not available. Cannot determine GitHub users (assignees) for pings." - else - echo "No user map JSON or jq not found. PING_KEYS_OUTPUT (from assignees) will be empty." fi - # Add latest committer if INPUT_PING_LATEST_COMMITTER is true if [[ "$INPUT_PING_LATEST_COMMITTER" == "true" ]]; then - echo "INPUT_PING_LATEST_COMMITTER is true. Attempting to fetch latest committer for PR #${PR_NUMBER}." if command -v gh &> /dev/null && [ -n "$PR_NUMBER" ]; then LATEST_COMMITTER_LOGIN_RAW=$(gh pr view "$PR_NUMBER" --json commits --jq '.commits[-1].author.login' 2>/dev/null || echo "") if [ -n "$LATEST_COMMITTER_LOGIN_RAW" ] && [ "$LATEST_COMMITTER_LOGIN_RAW" != "null" ]; then - # Apply bot filter (e.g., names ending in [bot] or -bot) LATEST_COMMITTER_LOGIN=$(echo "$LATEST_COMMITTER_LOGIN_RAW" | grep -v -E -i '(\[bot\]$|-bot$)' || echo "") if [ -n "$LATEST_COMMITTER_LOGIN" ]; then - echo "Latest committer identified: $LATEST_COMMITTER_LOGIN" - - # Check if this committer is already in PING_KEYS_OUTPUT ALREADY_IN_LIST=0 - if [ -n "$PING_KEYS_OUTPUT" ]; then # Only check if PING_KEYS_OUTPUT is not empty + if [ -n "$PING_KEYS_OUTPUT" ]; then IFS=',' read -ra PING_ARRAY <<< "$PING_KEYS_OUTPUT" for key in "${PING_ARRAY[@]}"; do if [[ "$key" == "$LATEST_COMMITTER_LOGIN" ]]; then @@ -1367,48 +836,22 @@ jobs: else PING_KEYS_OUTPUT="$PING_KEYS_OUTPUT,$LATEST_COMMITTER_LOGIN" fi - echo "Added latest committer '$LATEST_COMMITTER_LOGIN' to PING_KEYS_OUTPUT. New list: [$PING_KEYS_OUTPUT]" - else - echo "Latest committer '$LATEST_COMMITTER_LOGIN' is already in PING_KEYS_OUTPUT (likely an assignee)." fi - else - echo "Latest committer login '$LATEST_COMMITTER_LOGIN_RAW' was filtered out (likely a bot or pattern match) or empty after filter." fi - else - echo "No latest committer login found for PR #$PR_NUMBER from gh command, or login was null." - fi - else - if ! command -v gh &> /dev/null; then - echo "::warning::gh command not available. Cannot fetch latest committer." - fi - if [ -z "$PR_NUMBER" ]; then - echo "::warning::PR_NUMBER is not set (event might not be a pull_request). Cannot fetch latest committer." fi fi fi - # Restore stdout/stderr for GitHub Actions to show final summary - exec 1>&3 2>&4 - - # Make this a standard echo for better visibility of the final list - echo "Final Ping Keys Output (GitHub Logins from test-pytest.yml): [$PING_KEYS_OUTPUT]" echo "ping_user_ids_out=$PING_KEYS_OUTPUT" >> $GITHUB_OUTPUT - # Store branch names in variables with proper quoting PR_BRANCH="${PR_BRANCH_NAME:-unknown}" TARGET_BRANCH="${TARGET_BRANCH_NAME:-unknown}" - # 2. Construct Message Body MESSAGE_LINES+=("**Pytest Comparison & Regression Analysis for PR [#${PR_NUMBER}: ${PR_TITLE}](${PR_URL})**") MESSAGE_LINES+=("Branch: [\`${PR_BRANCH}\`](${REPO_URL}/tree/${PR_BRANCH}) against [\`${TARGET_BRANCH}\`](${REPO_URL}/tree/${TARGET_BRANCH})") MESSAGE_LINES+=("---") - # Job Status Summary MESSAGE_LINES+=("**Job Status:**") - LINT_STATUS="Success" - if [[ "$LINT_RESULT" == "failure" ]]; then LINT_STATUS="Failed"; elif [[ "$LINT_RESULT" == "skipped" ]]; then LINT_STATUS="Skipped"; fi - MESSAGE_LINES+=("- Linting: $LINT_STATUS") - SOURCE_TEST_STATUS="Success" if [[ "$SOURCE_TEST_RESULT" == "failure" ]]; then SOURCE_TEST_STATUS="Failed"; elif [[ "$SOURCE_TEST_RESULT" == "skipped" ]]; then SOURCE_TEST_STATUS="Skipped"; fi MESSAGE_LINES+=("- PR Branch Tests (\`${PR_BRANCH}\`): $SOURCE_TEST_STATUS") @@ -1422,32 +865,24 @@ jobs: MESSAGE_LINES+=("- Comparison & Regression: $COMPARE_STATUS") MESSAGE_LINES+=("---") - # Test Discovery Issues in PR Branch if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: ERROR: Test Discovery Failed in PR Branch (\`${PR_BRANCH}\`)**") MESSAGE_LINES+=(" - Type: \`${PR_ERROR_TYPE}\`") MESSAGE_LINES+=(" - Details: \`\`\`${PR_ERROR_DETAILS_TRUNCATED}\`\`\`") - MESSAGE_LINES+=(" - This usually indicates import errors or syntax issues preventing tests from being collected.") elif [[ "$PR_NO_TESTS_FOUND" == "true" ]]; then MESSAGE_LINES+=("**:warning: WARNING: No Tests Found in PR Branch (\`${PR_BRANCH}\`)**") - MESSAGE_LINES+=(" - Pytest did not discover any test files matching its patterns.") - MESSAGE_LINES+=(" - Ensure your test files are correctly named (e.g., \`test_*.py\` or \`*_test.py\`) and located.") fi - # Regression Analysis Summary if [[ "$HAS_REGRESSIONS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: REGRESSIONS DETECTED**") MESSAGE_LINES+=(" - **${REGRESSION_COUNT} test(s)** that were passing in \`${TARGET_BRANCH}\` are now **failing** in \`${PR_BRANCH}\`.") - # Calculate current message length CURRENT_MESSAGE=$(printf "%s\\n" "${MESSAGE_LINES[@]}") CURRENT_LENGTH=${#CURRENT_MESSAGE} if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then - # Read regression details REGRESSION_LIST=$(awk '/^[0-9]+\./ {sub(/^[0-9]+\. /, "- "); print}' "$REGRESSION_FILE_PATH") - # Calculate length with regression details TEMP_MESSAGE="$CURRENT_MESSAGE" TEMP_MESSAGE+="\`\`\`" TEMP_MESSAGE+="$REGRESSION_LIST" @@ -1455,25 +890,20 @@ jobs: TEMP_LENGTH=${#TEMP_MESSAGE} if [ $TEMP_LENGTH -le 2000 ]; then - # If total length would be under 2000 chars, include in message MESSAGE_LINES+=(" - **Failed Tests (Regressions):**") MESSAGE_LINES+=("\`\`\`") MESSAGE_LINES+=("$REGRESSION_LIST") MESSAGE_LINES+=("\`\`\`") - ARTIFACT_PATH_OUTPUT="" # No artifact if details are inline + ARTIFACT_PATH_OUTPUT="" else - # If would exceed 2000 chars, attach file instead MESSAGE_LINES+=(" - Details for the ${REGRESSION_COUNT} regressions are in the attached \`regression_details.txt\` file.") ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" fi else - MESSAGE_LINES+=(" (Regression details file not found or download failed; cannot list specific regressions here.)") ARTIFACT_PATH_OUTPUT="" fi elif [[ "$COMPARE_RESULT" == "failure" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then - # This case handles general comparison failures NOT due to specific regressions MESSAGE_LINES+=("**:warning: TEST RESULTS DECLINED**") - MESSAGE_LINES+=(" - The PR branch shows a decrease in test success compared to the target branch, but no specific regressions were identified by the \`meta-regression-analysis\` job.") MESSAGE_LINES+=(" - PR Branch (\`${PR_BRANCH}\`): **${PR_PASSED_TESTS}/${PR_TOTAL_TESTS} passed (${PR_PERCENTAGE}%)**") MESSAGE_LINES+=(" - Target Branch (\`${TARGET_BRANCH}\`): **${TARGET_PASSED_TESTS}/${TARGET_TOTAL_TESTS} passed (${TARGET_PERCENTAGE}%)**") elif [[ "$COMPARE_RESULT" == "success" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then @@ -1485,35 +915,27 @@ jobs: MESSAGE_LINES+=("---") MESSAGE_LINES+=("[View Workflow Run](${ACTION_RUN_URL})") - # Construct with actual newlines FINAL_MESSAGE_BODY=$(printf "%s\\n" "${MESSAGE_LINES[@]}") if [ ${#MESSAGE_LINES[@]} -gt 0 ]; then - # Remove the very last actual newline FINAL_MESSAGE_BODY="${FINAL_MESSAGE_BODY%\\n}" fi - echo "Final message body prepared in test-pytest.yml" - echo "message_body_out<> $GITHUB_OUTPUT echo "$FINAL_MESSAGE_BODY" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT echo "artifact_path_out=$ARTIFACT_PATH_OUTPUT" >> $GITHUB_OUTPUT - - name: Upload notification construction debug logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: notification_debug_logs_${{ github.event.pull_request.number || github.run_id }} - path: debug_notification_construction.log - retention-days: 3 - if-no-files-found: ignore - - # Even if webhook checks are handled inside the alert workflow, - # we still need to pass the secret to satisfy GitHub's workflow validation notify-discord: name: Send Discord Notification - needs: [prepare-notification] + needs: + [ + test-source-branch, + test-target-branch, + compare-results, + perform-regression-analysis, + prepare-notification, + ] if: | always() && needs.prepare-notification.outputs.should_notify == 'true' && diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml deleted file mode 100644 index 1d853bd..0000000 --- a/.github/workflows/test-pytest.yml +++ /dev/null @@ -1,952 +0,0 @@ -name: Reusable Compare Pytest Results - -on: - workflow_call: - inputs: - target_branch_to_compare: - description: "The target branch to compare against (e.g., main, refs/heads/main)." - required: true - type: string - python-version: - description: "Python version to use for testing." - required: false - type: string - default: "3.10" - ping_latest_committer: - description: "If true, the latest committer on the PR will be added to the ping list." - required: false - type: boolean - default: false - runs_on: - required: false - type: string - default: "ubuntu-latest" - secrets: - DISCORD_WEBHOOK_URL: - description: "Discord Webhook URL for failure notifications. If not provided, notifications are skipped." - required: false - DISCORD_USER_MAP: - description: 'JSON string mapping GitHub usernames to Discord User IDs (e.g., {"user1":"id1"}). If not provided, users won''t be pinged.' - required: false - outputs: - pr_total: - description: "Total tests in PR/source branch" - value: ${{ jobs.test-source-branch.outputs.total }} - pr_passed: - description: "Passed tests in PR/source branch" - value: ${{ jobs.test-source-branch.outputs.passed }} - pr_percentage: - description: "Pass percentage in PR/source branch" - value: ${{ jobs.test-source-branch.outputs.percentage }} - pr_collection_errors: - description: "PR branch has collection errors" - value: ${{ jobs.test-source-branch.outputs.collection_errors }} - pr_no_tests_found: - description: "PR branch has no tests found" - value: ${{ jobs.test-source-branch.outputs.no_tests_found }} - target_total: - description: "Total tests in target branch" - value: ${{ jobs.test-target-branch.outputs.total }} - target_passed: - description: "Passed tests in target branch" - value: ${{ jobs.test-target-branch.outputs.passed }} - target_percentage: - description: "Pass percentage in target branch" - value: ${{ jobs.test-target-branch.outputs.percentage }} - has_regressions: - description: "Boolean indicating if regressions were found" - value: ${{ jobs.compare-results.outputs.has_regressions }} - regression_count: - description: "Number of test regressions found" - value: ${{ jobs.compare-results.outputs.regression_count }} - -jobs: - test-source-branch: - runs-on: ${{ inputs.runs_on }} - outputs: - total: ${{ steps.extract-results.outputs.total }} - passed: ${{ steps.extract-results.outputs.passed }} - percentage: ${{ steps.extract-results.outputs.percentage }} - collection_errors: ${{ steps.check-collection.outputs.has_collection_errors }} - no_tests_found: ${{ steps.check-collection.outputs.no_tests_found }} - has_errors: ${{ steps.check-collection.outputs.has_errors }} - error_type: ${{ steps.check-collection.outputs.error_type }} - error_details: ${{ steps.check-collection.outputs.error_details }} - failing_count: ${{ steps.extract-results.outputs.failing_count }} - skipped_count: ${{ steps.extract-results.outputs.skipped_count }} - xfailed_count: ${{ steps.extract-results.outputs.xfailed_count }} - - steps: - - name: Checkout PR Branch - uses: actions/checkout@v4.2.2 - with: - submodules: "recursive" - - - name: Set up Python - uses: actions/setup-python@v5.3.0 - with: - python-version: "${{ inputs.python-version }}" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip --quiet - pip install pytest pytest-json-report pytest-asyncio --quiet - if [ -f requirements.txt ]; then - pip install -r requirements.txt --quiet - fi - - - name: Silent pytest execution and collection check - id: check-collection - run: | - python3 << 'EOF' - import subprocess - import sys - import os - - def run_with_complete_silence(cmd, timeout=1800): - """Run command with OS-level output suppression""" - try: - original_stdout = os.dup(1) - original_stderr = os.dup(2) - original_stdin = os.dup(0) - - try: - devnull = os.open(os.devnull, os.O_WRONLY) - devnull_read = os.open(os.devnull, os.O_RDONLY) - - os.dup2(devnull, 1) - os.dup2(devnull, 2) - os.dup2(devnull_read, 0) - - result = subprocess.run( - cmd, - timeout=timeout, - env={**os.environ, - 'PYTHONUNBUFFERED': '0', - 'PYTHONWARNINGS': 'ignore', - 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', - 'PYTEST_CURRENT_TEST': '', - 'COLUMNS': '80', - 'PYTHONDONTWRITEBYTECODE': '1'} - ) - - os.close(devnull) - os.close(devnull_read) - - finally: - os.dup2(original_stdout, 1) - os.dup2(original_stderr, 2) - os.dup2(original_stdin, 0) - os.close(original_stdout) - os.close(original_stderr) - os.close(original_stdin) - - return result - - except subprocess.TimeoutExpired: - return subprocess.CompletedProcess(cmd, 1) - except Exception as e: - return subprocess.CompletedProcess(cmd, 1) - - def run_with_captured_output(cmd, timeout=60): - """Run command and capture output for error checking""" - try: - return subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=timeout, - env={**os.environ, 'PYTHONWARNINGS': 'ignore'} - ) - except Exception: - return subprocess.CompletedProcess(cmd, 1, stderr="Execution failed") - - # Collection check - collection_cmd = [sys.executable, '-m', 'pytest', '--collect-only', '--quiet', '--tb=no'] - collection_result = run_with_captured_output(collection_cmd) - - has_collection_errors = "false" - no_tests_found = "false" - error_type = "none" - error_details = "none" - - if collection_result.returncode != 0: - if any(err in collection_result.stderr for err in ['ImportError', 'ModuleNotFoundError', 'SyntaxError', 'ERROR collecting']): - has_collection_errors = "true" - if 'ImportError' in collection_result.stderr: - error_type = "ImportError" - elif 'ModuleNotFoundError' in collection_result.stderr: - error_type = "ModuleNotFoundError" - elif 'SyntaxError' in collection_result.stderr: - error_type = "SyntaxError" - else: - error_type = "CollectionError" - error_details = collection_result.stderr[:200] if collection_result.stderr else "Collection failed" - else: - if 'collected 0 items' in collection_result.stdout or not collection_result.stdout.strip(): - no_tests_found = "true" - error_type = "NoTestsFound" - error_details = "No test files discovered" - - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f'has_collection_errors={has_collection_errors}\n') - f.write(f'no_tests_found={no_tests_found}\n') - f.write(f'error_type={error_type}\n') - f.write(f'error_details={error_details}\n') - f.write(f'has_errors={str(has_collection_errors == "true" or no_tests_found == "true").lower()}\n') - - # Run tests silently if collection succeeded - if has_collection_errors == "false": - test_cmd = [ - sys.executable, '-m', 'pytest', - '--json-report', - '--json-report-file=pr_results.json', - '--tb=no', - '--no-header', - '--no-summary', - '--quiet', - '--disable-warnings', - '--log-level=CRITICAL', - '--log-cli-level=CRITICAL', - '--capture=sys', - '--maxfail=999999' - ] - - run_with_complete_silence(test_cmd) - - if os.path.exists('pr_results.json'): - print("Test results generated successfully") - else: - print("::error::Failed to create test results file") - else: - print(f"::error::Skipping tests due to collection errors: {error_type}") - EOF - - - name: Extract test results and create artifacts - id: extract-results - run: | - python -c " - import json - import sys - import os - - pr_total = 0 - pr_passed = 0 - pr_percentage = 0 - failing_tests = [] - skipped_tests = [] - xfailed_tests = [] - all_tests = [] - skipped_tests_with_reasons = {} - xfailed_tests_with_reasons = {} - - try: - with open('pr_results.json') as f: - pr_results = json.load(f) - - if pr_results.get('exitcode', 0) > 1: - pr_total = 0 - pr_passed = 0 - elif 'summary' in pr_results and isinstance(pr_results['summary'], dict): - summary = pr_results['summary'] - pr_total = summary.get('total', 0) - pr_passed = summary.get('passed', 0) - - if 'tests' in pr_results: - for test in pr_results['tests']: - outcome = test.get('outcome') - nodeid = test.get('nodeid', '') - if nodeid: - all_tests.append(nodeid) - if outcome in ['failed', 'error']: - failing_tests.append(nodeid) - elif outcome == 'skipped': - skipped_tests.append(nodeid) - skip_reason = 'No reason provided' - if 'longrepr' in test and test['longrepr']: - longrepr = test['longrepr'] - if isinstance(longrepr, list) and longrepr: - skip_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' - elif isinstance(longrepr, str): - skip_reason = longrepr - skipped_tests_with_reasons[nodeid] = skip_reason.strip() - elif outcome == 'xfailed': - xfailed_tests.append(nodeid) - xfail_reason = 'No reason provided' - if 'longrepr' in test and test['longrepr']: - longrepr = test['longrepr'] - if isinstance(longrepr, list) and longrepr: - xfail_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' - elif isinstance(longrepr, str): - xfail_reason = longrepr - xfailed_tests_with_reasons[nodeid] = xfail_reason.strip() - - pr_percentage = (pr_passed / pr_total * 100) if pr_total > 0 else 0 - - except Exception as e: - pass - - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f'total={pr_total}\\n') - f.write(f'passed={pr_passed}\\n') - f.write(f'percentage={pr_percentage:.2f}\\n') - f.write(f'failing_count={len(failing_tests)}\\n') - f.write(f'skipped_count={len(skipped_tests)}\\n') - f.write(f'xfailed_count={len(xfailed_tests)}\\n') - - test_data = { - 'failing_tests': failing_tests, - 'skipped_tests': skipped_tests, - 'xfailed_tests': xfailed_tests, - 'all_tests': all_tests, - 'skipped_tests_with_reasons': skipped_tests_with_reasons, - 'xfailed_tests_with_reasons': xfailed_tests_with_reasons - } - - with open('pr_test_data.json', 'w') as f: - json.dump(test_data, f, indent=2) - " - - - name: Upload PR branch artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} - path: | - pr_test_data.json - pr_results.json - collection_check.txt - retention-days: 3 - if-no-files-found: ignore - - test-target-branch: - runs-on: ${{ inputs.runs_on }} - outputs: - total: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.total || steps.extract-results.outputs.total }} - passed: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.passed || steps.extract-results.outputs.passed }} - percentage: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.percentage || steps.extract-results.outputs.percentage }} - collection_errors: ${{ steps.check-collection.outputs.has_collection_errors }} - no_tests_found: ${{ steps.check-collection.outputs.no_tests_found }} - has_errors: ${{ steps.check-collection.outputs.has_errors }} - error_type: ${{ steps.check-collection.outputs.error_type }} - error_details: ${{ steps.check-collection.outputs.error_details }} - passing_count: ${{ steps.check-collection.outputs.has_collection_errors == 'true' && steps.set-error-outputs.outputs.passing_count || steps.extract-results.outputs.passing_count }} - - steps: - - name: Checkout target branch - uses: actions/checkout@v4.2.2 - with: - submodules: "recursive" - ref: ${{ inputs.target_branch_to_compare }} - - - name: Set up Python - uses: actions/setup-python@v5.3.0 - with: - python-version: "${{ inputs.python-version }}" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip --quiet - pip install pytest pytest-json-report pytest-asyncio --quiet - if [ -f requirements.txt ]; then - pip install -r requirements.txt --quiet - fi - - - name: Check for test collection errors - id: check-collection - run: | - HAS_COLLECTION_ERRORS="false" - NO_TESTS_FOUND="false" - ERROR_TYPE="none" - ERROR_DETAILS="none" - - python -m pytest --collect-only --quiet --tb=no >/dev/null 2>collection_check.txt - COLLECTION_EXIT_CODE=$? - - if [ $COLLECTION_EXIT_CODE -ne 0 ]; then - if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting" collection_check.txt; then - echo "::warning::Test discovery errors detected in target branch" - HAS_COLLECTION_ERRORS="true" - - if grep -q "ImportError" collection_check.txt; then - ERROR_TYPE="ImportError" - elif grep -q "ModuleNotFoundError" collection_check.txt; then - ERROR_TYPE="ModuleNotFoundError" - elif grep -q "SyntaxError" collection_check.txt; then - ERROR_TYPE="SyntaxError" - else - ERROR_TYPE="CollectionError" - fi - - ERROR_DETAILS=$(head -1 collection_check.txt | tr '\n' ' ' | cut -c1-200) - fi - else - TEST_COUNT=$(python -m pytest --collect-only --quiet 2>/dev/null | grep -o "collected [0-9]* item" | grep -o "[0-9]*" || echo "0") - if [[ "$TEST_COUNT" == "0" ]]; then - echo "::warning::No tests were found in the target branch" - NO_TESTS_FOUND="true" - ERROR_TYPE="NoTestsFound" - ERROR_DETAILS="No test files discovered" - fi - fi - - echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT - echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT - echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT - echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT - - if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then - echo "has_errors=true" >> $GITHUB_OUTPUT - else - echo "has_errors=false" >> $GITHUB_OUTPUT - fi - - - name: Run tests on target branch - if: steps.check-collection.outputs.has_collection_errors != 'true' - run: | - python3 << 'EOF' - import subprocess - import sys - import os - - def run_with_complete_silence(cmd, timeout=1800): - """Run command with OS-level output suppression""" - try: - original_stdout = os.dup(1) - original_stderr = os.dup(2) - original_stdin = os.dup(0) - - try: - devnull = os.open(os.devnull, os.O_WRONLY) - devnull_read = os.open(os.devnull, os.O_RDONLY) - - os.dup2(devnull, 1) - os.dup2(devnull, 2) - os.dup2(devnull_read, 0) - - result = subprocess.run( - cmd, - timeout=timeout, - env={**os.environ, - 'PYTHONUNBUFFERED': '0', - 'PYTHONWARNINGS': 'ignore', - 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', - 'PYTEST_CURRENT_TEST': '', - 'COLUMNS': '80', - 'PYTHONDONTWRITEBYTECODE': '1'} - ) - - os.close(devnull) - os.close(devnull_read) - - finally: - os.dup2(original_stdout, 1) - os.dup2(original_stderr, 2) - os.dup2(original_stdin, 0) - os.close(original_stdout) - os.close(original_stderr) - os.close(original_stdin) - - return result - - except subprocess.TimeoutExpired: - return subprocess.CompletedProcess(cmd, 1) - except Exception as e: - return subprocess.CompletedProcess(cmd, 1) - - test_cmd = [ - sys.executable, '-m', 'pytest', - '--json-report', - '--json-report-file=target_results.json', - '--tb=no', - '--no-header', - '--no-summary', - '--quiet', - '--disable-warnings', - '--log-level=CRITICAL', - '--log-cli-level=CRITICAL', - '--capture=sys', - '--maxfail=999999' - ] - - run_with_complete_silence(test_cmd) - - if os.path.exists('target_results.json'): - print("Test results generated successfully") - else: - print("::warning::Failed to create test results file") - EOF - - - name: Extract test results and create artifacts - id: extract-results - if: steps.check-collection.outputs.has_collection_errors != 'true' - run: | - python -c " - import json - import sys - import os - - target_total = 0 - target_passed = 0 - target_percentage = 0 - passing_tests = [] - all_tests = [] - - try: - with open('target_results.json') as f: - target_results = json.load(f) - - if target_results.get('exitcode', 0) > 1: - target_total = 0 - target_passed = 0 - elif 'summary' in target_results and isinstance(target_results['summary'], dict): - summary = target_results['summary'] - target_total = summary.get('total', 0) - target_passed = summary.get('passed', 0) - - if 'tests' in target_results: - for test in target_results['tests']: - outcome = test.get('outcome') - nodeid = test.get('nodeid', '') - if nodeid: - all_tests.append(nodeid) - if outcome == 'passed': - passing_tests.append(nodeid) - - target_percentage = (target_passed / target_total * 100) if target_total > 0 else 0 - - except Exception as e: - pass - - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f'total={target_total}\\n') - f.write(f'passed={target_passed}\\n') - f.write(f'percentage={target_percentage:.2f}\\n') - f.write(f'passing_count={len(passing_tests)}\\n') - - test_data = { - 'passing_tests': passing_tests, - 'all_tests': all_tests - } - - with open('target_test_data.json', 'w') as f: - json.dump(test_data, f, indent=2) - " - - - name: Upload target branch artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} - path: | - target_test_data.json - target_results.json - collection_check.txt - retention-days: 3 - if-no-files-found: ignore - - - name: Set collection error outputs - id: set-error-outputs - if: steps.check-collection.outputs.has_collection_errors == 'true' - run: | - echo "total=0" >> $GITHUB_OUTPUT - echo "passed=0" >> $GITHUB_OUTPUT - echo "percentage=0.00" >> $GITHUB_OUTPUT - echo "passing_count=0" >> $GITHUB_OUTPUT - - compare-results: - needs: [test-source-branch, test-target-branch] - runs-on: ${{ inputs.runs_on }} - outputs: - has_regressions: ${{ needs.perform-regression-analysis.outputs.has_regressions }} - regression_count: ${{ needs.perform-regression-analysis.outputs.regression_count }} - - steps: - - name: Install bc - run: | - sudo apt-get update -y - sudo apt-get install -y bc - - - name: Download test data artifacts - uses: actions/download-artifact@v4 - with: - pattern: "*_branch_data_${{ github.event.pull_request.number || github.run_id }}" - path: ./artifacts - merge-multiple: false - - - name: Check for collection errors - run: | - PR_COLLECTION_ERRORS="${{ needs.test-source-branch.outputs.collection_errors }}" - PR_NO_TESTS="${{ needs.test-source-branch.outputs.no_tests_found }}" - PR_ERROR_TYPE="${{ needs.test-source-branch.outputs.error_type }}" - TARGET_COLLECTION_ERRORS="${{ needs.test-target-branch.outputs.collection_errors }}" - - if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then - echo "::error::Test discovery errors in PR branch: $PR_ERROR_TYPE" - exit 1 - fi - - if [[ "$PR_NO_TESTS" == "true" ]]; then - echo "::error::No tests were found in the PR branch" - exit 1 - fi - - if [[ "$TARGET_COLLECTION_ERRORS" == "true" ]]; then - echo "⚠️ Target branch has test discovery errors." - fi - - - name: Run regression analysis from artifacts - run: | - python3 - << 'EOF' - import json - import os - import glob - - try: - target_data = {} - pr_data = {} - - target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') - if target_files: - with open(target_files[0], 'r') as f: - target_data = json.load(f) - - pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') - if pr_files: - with open(pr_files[0], 'r') as f: - pr_data = json.load(f) - - target_passing = target_data.get('passing_tests', []) - pr_failing = pr_data.get('failing_tests', []) - - target_passing_set = set(target_passing) - pr_failing_set = set(pr_failing) - regression_tests = list(target_passing_set.intersection(pr_failing_set)) - - if regression_tests: - with open("regression_details.txt", "w") as f: - f.write(f"Found {len(regression_tests)} tests that were passing in target branch but now failing in PR branch:\\n\\n") - for idx, test in enumerate(sorted(regression_tests), 1): - f.write(f"{idx}. {test}\\n") - except Exception as e: - pass - EOF - - - name: Check for regression details file - id: check-regressions - run: | - _has_regressions="false" - _regression_count="0" - - if [ -f "regression_details.txt" ]; then - _current_count=$(grep -c "^[0-9]\+\." regression_details.txt || echo "0") - if [ "$_current_count" -gt 0 ]; then - _has_regressions="true" - _regression_count="$_current_count" - echo "::error::Test Regressions Found: $_regression_count test(s) that were passing in target branch are now failing in PR branch." - fi - fi - - echo "HAS_REGRESSIONS=$_has_regressions" >> $GITHUB_OUTPUT - echo "REGRESSION_COUNT=$_regression_count" >> $GITHUB_OUTPUT - - - name: Upload regression details artifact - if: steps.check-regressions.outputs.HAS_REGRESSIONS == 'true' && steps.check-regressions.outputs.REGRESSION_COUNT > 0 - uses: actions/upload-artifact@v4 - with: - name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests - path: regression_details.txt - retention-days: 1 - - - name: Compare test results - run: | - echo "Test Results Summary:" - echo "Target branch (${{ inputs.target_branch_to_compare }}): ${{ needs.test-target-branch.outputs.passed }}/${{ needs.test-target-branch.outputs.total }} tests passed (${{ needs.test-target-branch.outputs.percentage }}%)" - echo "PR branch: ${{ needs.test-source-branch.outputs.passed }}/${{ needs.test-source-branch.outputs.total }} tests passed (${{ needs.test-source-branch.outputs.percentage }}%)" - - if [[ "${{ needs.test-source-branch.outputs.total }}" == "0" ]]; then - echo "::error::No tests were found in the PR branch" - exit 1 - fi - - PR_PASSED=${{ needs.test-source-branch.outputs.passed }} - TARGET_PASSED=${{ needs.test-target-branch.outputs.passed }} - PR_PERCENTAGE=${{ needs.test-source-branch.outputs.percentage }} - TARGET_PERCENTAGE=${{ needs.test-target-branch.outputs.percentage }} - PR_TOTAL=${{ needs.test-source-branch.outputs.total }} - TARGET_TOTAL=${{ needs.test-target-branch.outputs.total }} - - if [[ "$TARGET_TOTAL" == "0" ]]; then - if [[ "$PR_PASSED" -gt 0 ]]; then - echo "✅ PR branch has tests and some are passing (target branch has no tests)" - exit 0 - else - echo "❌ PR branch has no passing tests" - exit 1 - fi - fi - - if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then - echo "❌ PR branch has test regressions from target branch" - exit 1 - fi - - if (( $(echo "$PR_PASSED >= $TARGET_PASSED" | bc -l) )) && (( $(echo "$PR_PERCENTAGE >= $TARGET_PERCENTAGE" | bc -l) )); then - echo "✅ PR branch has equal or better test results than target branch" - exit 0 - else - echo "❌ PR branch has worse test results than target branch" - exit 1 - fi - - perform-regression-analysis: - needs: [test-source-branch, test-target-branch] - uses: ./.github/workflows/meta-regression-analysis.yml - with: - item_type_singular: "test" - item_type_plural: "tests" - pr_number: ${{ github.event.pull_request.number }} - run_id: ${{ github.run_id }} - target_branch_artifact_name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} - pr_branch_artifact_name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} - - prepare-notification: - name: Prepare Notification Data - needs: - [ - test-source-branch, - test-target-branch, - compare-results, - perform-regression-analysis, - ] - if: | - always() && - ( - needs.test-source-branch.outputs.collection_errors == 'true' || - needs.test-source-branch.outputs.no_tests_found == 'true' || - needs.compare-results.result == 'failure' || - needs.perform-regression-analysis.outputs.has_regressions == 'true' - ) - runs-on: ${{ inputs.runs_on }} - outputs: - message_body: ${{ steps.construct_notification.outputs.message_body_out }} - ping_user_ids: ${{ steps.construct_notification.outputs.ping_user_ids_out }} - artifact_path: ${{ steps.construct_notification.outputs.artifact_path_out }} - should_notify: "true" - webhook_available_for_alert: ${{ steps.check_webhook_availability.outputs.webhook_available }} - - steps: - - name: Check for Discord Webhook URL - id: check_webhook_availability - run: | - if [ -z "${{ secrets.DISCORD_WEBHOOK_URL }}" ]; then - echo "webhook_available=false" >> $GITHUB_OUTPUT - else - echo "webhook_available=true" >> $GITHUB_OUTPUT - fi - - - name: Download regression details (if any) - id: download_regressions - if: needs.perform-regression-analysis.outputs.has_regressions == 'true' && needs.perform-regression-analysis.outputs.regression_count > 0 - uses: actions/download-artifact@v4 - with: - name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests - path: . - continue-on-error: true - - - name: Construct Discord Notification - id: construct_notification - env: - SOURCE_TEST_RESULT: ${{ needs.test-source-branch.result }} - TARGET_TEST_RESULT: ${{ needs.test-target-branch.result }} - COMPARE_RESULT: ${{ needs.compare-results.result }} - PR_COLLECTION_ERRORS: ${{ needs.test-source-branch.outputs.collection_errors }} - PR_NO_TESTS_FOUND: ${{ needs.test-source-branch.outputs.no_tests_found }} - PR_ERROR_TYPE: ${{ needs.test-source-branch.outputs.error_type }} - PR_ERROR_DETAILS_TRUNCATED: ${{ needs.test-source-branch.outputs.error_details }} - HAS_REGRESSIONS: ${{ needs.perform-regression-analysis.outputs.has_regressions }} - REGRESSION_COUNT: ${{ needs.perform-regression-analysis.outputs.regression_count }} - PR_TOTAL_TESTS: ${{ needs.test-source-branch.outputs.total }} - PR_PASSED_TESTS: ${{ needs.test-source-branch.outputs.passed }} - PR_PERCENTAGE: ${{ needs.test-source-branch.outputs.percentage }} - TARGET_TOTAL_TESTS: ${{ needs.test-target-branch.outputs.total }} - TARGET_PASSED_TESTS: ${{ needs.test-target-branch.outputs.passed }} - TARGET_PERCENTAGE: ${{ needs.test-target-branch.outputs.percentage }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_TITLE: ${{ github.event.pull_request.title }} - PR_URL: ${{ github.event.pull_request.html_url }} - TARGET_BRANCH_NAME: ${{ inputs.target_branch_to_compare }} - PR_BRANCH_NAME: ${{ github.head_ref }} - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_ASSIGNEES_JSON: ${{ toJson(github.event.pull_request.assignees) }} - USER_MAP_JSON: ${{ secrets.DISCORD_USER_MAP }} - REGRESSION_FILE_PATH: "regression_details.txt" - DOWNLOAD_REGRESSIONS_OUTCOME: ${{ steps.download_regressions.outcome }} - INPUT_PING_LATEST_COMMITTER: ${{ inputs.ping_latest_committer }} - run: | - MESSAGE_LINES=() - PING_KEYS_OUTPUT="" - ARTIFACT_PATH_OUTPUT="" - - PING_KEYS_OUTPUT="" - - if [ -n "$USER_MAP_JSON" ] && [ "$USER_MAP_JSON" != "{}" ] && command -v jq &> /dev/null; then - ASSIGNEE_LOGINS_ARRAY=($(echo "$GH_ASSIGNEES_JSON" | jq -r '.[].login // empty')) - MAPPED_ASSIGNEE_COUNT=0 - TEMP_PING_KEYS=() - - for assignee_login in "${ASSIGNEE_LOGINS_ARRAY[@]}"; do - if [ -z "$assignee_login" ]; then - continue - fi - if echo "$USER_MAP_JSON" | jq -e --arg K "$assignee_login" '.[$K]' > /dev/null; then - TEMP_PING_KEYS+=("$assignee_login") - MAPPED_ASSIGNEE_COUNT=$((MAPPED_ASSIGNEE_COUNT + 1)) - fi - done - - if [ ${#TEMP_PING_KEYS[@]} -gt 0 ]; then - PING_KEYS_OUTPUT=$(IFS=,; echo "${TEMP_PING_KEYS[*]}") - fi - fi - - if [[ "$INPUT_PING_LATEST_COMMITTER" == "true" ]]; then - if command -v gh &> /dev/null && [ -n "$PR_NUMBER" ]; then - LATEST_COMMITTER_LOGIN_RAW=$(gh pr view "$PR_NUMBER" --json commits --jq '.commits[-1].author.login' 2>/dev/null || echo "") - - if [ -n "$LATEST_COMMITTER_LOGIN_RAW" ] && [ "$LATEST_COMMITTER_LOGIN_RAW" != "null" ]; then - LATEST_COMMITTER_LOGIN=$(echo "$LATEST_COMMITTER_LOGIN_RAW" | grep -v -E -i '(\[bot\]$|-bot$)' || echo "") - - if [ -n "$LATEST_COMMITTER_LOGIN" ]; then - ALREADY_IN_LIST=0 - if [ -n "$PING_KEYS_OUTPUT" ]; then - IFS=',' read -ra PING_ARRAY <<< "$PING_KEYS_OUTPUT" - for key in "${PING_ARRAY[@]}"; do - if [[ "$key" == "$LATEST_COMMITTER_LOGIN" ]]; then - ALREADY_IN_LIST=1 - break - fi - done - fi - - if [[ "$ALREADY_IN_LIST" -eq 0 ]]; then - if [ -z "$PING_KEYS_OUTPUT" ]; then - PING_KEYS_OUTPUT="$LATEST_COMMITTER_LOGIN" - else - PING_KEYS_OUTPUT="$PING_KEYS_OUTPUT,$LATEST_COMMITTER_LOGIN" - fi - fi - fi - fi - fi - fi - - echo "ping_user_ids_out=$PING_KEYS_OUTPUT" >> $GITHUB_OUTPUT - - PR_BRANCH="${PR_BRANCH_NAME:-unknown}" - TARGET_BRANCH="${TARGET_BRANCH_NAME:-unknown}" - - MESSAGE_LINES+=("**Pytest Comparison & Regression Analysis for PR [#${PR_NUMBER}: ${PR_TITLE}](${PR_URL})**") - MESSAGE_LINES+=("Branch: [\`${PR_BRANCH}\`](${REPO_URL}/tree/${PR_BRANCH}) against [\`${TARGET_BRANCH}\`](${REPO_URL}/tree/${TARGET_BRANCH})") - MESSAGE_LINES+=("---") - - MESSAGE_LINES+=("**Job Status:**") - SOURCE_TEST_STATUS="Success" - if [[ "$SOURCE_TEST_RESULT" == "failure" ]]; then SOURCE_TEST_STATUS="Failed"; elif [[ "$SOURCE_TEST_RESULT" == "skipped" ]]; then SOURCE_TEST_STATUS="Skipped"; fi - MESSAGE_LINES+=("- PR Branch Tests (\`${PR_BRANCH}\`): $SOURCE_TEST_STATUS") - - TARGET_TEST_STATUS="Success" - if [[ "$TARGET_TEST_RESULT" == "failure" ]]; then TARGET_TEST_STATUS="Failed"; elif [[ "$TARGET_TEST_RESULT" == "skipped" ]]; then TARGET_TEST_STATUS="Skipped"; fi - MESSAGE_LINES+=("- Target Branch Tests (\`${TARGET_BRANCH}\`): $TARGET_TEST_STATUS") - - COMPARE_STATUS="Success" - if [[ "$COMPARE_RESULT" == "failure" ]]; then COMPARE_STATUS="Failed"; elif [[ "$COMPARE_RESULT" == "skipped" ]]; then COMPARE_STATUS="Skipped"; fi - MESSAGE_LINES+=("- Comparison & Regression: $COMPARE_STATUS") - MESSAGE_LINES+=("---") - - if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then - MESSAGE_LINES+=("**:red_circle: ERROR: Test Discovery Failed in PR Branch (\`${PR_BRANCH}\`)**") - MESSAGE_LINES+=(" - Type: \`${PR_ERROR_TYPE}\`") - MESSAGE_LINES+=(" - Details: \`\`\`${PR_ERROR_DETAILS_TRUNCATED}\`\`\`") - elif [[ "$PR_NO_TESTS_FOUND" == "true" ]]; then - MESSAGE_LINES+=("**:warning: WARNING: No Tests Found in PR Branch (\`${PR_BRANCH}\`)**") - fi - - if [[ "$HAS_REGRESSIONS" == "true" ]]; then - MESSAGE_LINES+=("**:red_circle: REGRESSIONS DETECTED**") - MESSAGE_LINES+=(" - **${REGRESSION_COUNT} test(s)** that were passing in \`${TARGET_BRANCH}\` are now **failing** in \`${PR_BRANCH}\`.") - - CURRENT_MESSAGE=$(printf "%s\\n" "${MESSAGE_LINES[@]}") - CURRENT_LENGTH=${#CURRENT_MESSAGE} - - if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then - REGRESSION_LIST=$(awk '/^[0-9]+\./ {sub(/^[0-9]+\. /, "- "); print}' "$REGRESSION_FILE_PATH") - - TEMP_MESSAGE="$CURRENT_MESSAGE" - TEMP_MESSAGE+="\`\`\`" - TEMP_MESSAGE+="$REGRESSION_LIST" - TEMP_MESSAGE+="\`\`\`" - TEMP_LENGTH=${#TEMP_MESSAGE} - - if [ $TEMP_LENGTH -le 2000 ]; then - MESSAGE_LINES+=(" - **Failed Tests (Regressions):**") - MESSAGE_LINES+=("\`\`\`") - MESSAGE_LINES+=("$REGRESSION_LIST") - MESSAGE_LINES+=("\`\`\`") - ARTIFACT_PATH_OUTPUT="" - else - MESSAGE_LINES+=(" - Details for the ${REGRESSION_COUNT} regressions are in the attached \`regression_details.txt\` file.") - ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" - fi - else - ARTIFACT_PATH_OUTPUT="" - fi - elif [[ "$COMPARE_RESULT" == "failure" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then - MESSAGE_LINES+=("**:warning: TEST RESULTS DECLINED**") - MESSAGE_LINES+=(" - PR Branch (\`${PR_BRANCH}\`): **${PR_PASSED_TESTS}/${PR_TOTAL_TESTS} passed (${PR_PERCENTAGE}%)**") - MESSAGE_LINES+=(" - Target Branch (\`${TARGET_BRANCH}\`): **${TARGET_PASSED_TESTS}/${TARGET_TOTAL_TESTS} passed (${TARGET_PERCENTAGE}%)**") - elif [[ "$COMPARE_RESULT" == "success" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then - MESSAGE_LINES+=("**:white_check_mark: NO REGRESSIONS DETECTED**") - MESSAGE_LINES+=(" - PR Branch (\`${PR_BRANCH}\`): **${PR_PASSED_TESTS}/${PR_TOTAL_TESTS} passed (${PR_PERCENTAGE}%)**") - MESSAGE_LINES+=(" - Target Branch (\`${TARGET_BRANCH}\`): **${TARGET_PASSED_TESTS}/${TARGET_TOTAL_TESTS} passed (${TARGET_PERCENTAGE}%)**") - fi - - MESSAGE_LINES+=("---") - MESSAGE_LINES+=("[View Workflow Run](${ACTION_RUN_URL})") - - FINAL_MESSAGE_BODY=$(printf "%s\\n" "${MESSAGE_LINES[@]}") - if [ ${#MESSAGE_LINES[@]} -gt 0 ]; then - FINAL_MESSAGE_BODY="${FINAL_MESSAGE_BODY%\\n}" - fi - - echo "message_body_out<> $GITHUB_OUTPUT - echo "$FINAL_MESSAGE_BODY" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - echo "artifact_path_out=$ARTIFACT_PATH_OUTPUT" >> $GITHUB_OUTPUT - - notify-discord: - name: Send Discord Notification - needs: - [ - test-source-branch, - test-target-branch, - compare-results, - perform-regression-analysis, - prepare-notification, - ] - if: | - always() && - needs.prepare-notification.outputs.should_notify == 'true' && - needs.prepare-notification.outputs.webhook_available_for_alert == 'true' - uses: ./.github/workflows/alert-discord.yml - with: - message_body: ${{ needs.prepare-notification.outputs.message_body }} - ping_user_ids: ${{ needs.prepare-notification.outputs.ping_user_ids }} - artifact_paths: ${{ needs.prepare-notification.outputs.artifact_path }} - should_notify: ${{ needs.prepare-notification.outputs.should_notify }} - runs_on: ${{ inputs.runs_on }} - secrets: - DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} - DISCORD_USER_MAP: ${{ secrets.DISCORD_USER_MAP }} diff --git a/README.md b/README.md index 6fa326b..162905d 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,6 @@ - `test-`: Offers an action to run a test suite. *If an action is language/framework specific, a `lang-` section follows the above prefix before the descriptor. ## Triggers: -The `triggers` folder contains trigger actions that go into repositories to trigger the core actions. \ No newline at end of file +The `triggers` folder contains trigger actions that go into repositories to trigger the core actions. +- `on-pr-`: Triggers on PR made to specified branches. +- `on-push-`: Triggers on push made to specified branches. diff --git a/usage.mmd b/usage.mmd new file mode 100644 index 0000000..fcaf4b4 --- /dev/null +++ b/usage.mmd @@ -0,0 +1,71 @@ +flowchart LR + TIME["on-day"] + subgraph CM ["Common"] + PR_DEV["on-pr-dev"] + PR_MAIN["on-pr-main"] + PUSH["on-push"] + PUBLISH["on-publish"] + ISSUE_CLOSE["on-issue-close"] + ISSUE["on-issue"] + subgraph FE ["Frontend Repositories"] + FE_PUSH["on-push"] + FE_PUBLISH["on-publish"] + end + subgraph BE ["Backend Repositories"] + BE_PUSH["on-push"] + BE_PUBLISH["on-publish"] + end + end + subgraph WF ["Workflows Repository"] + WF_BUILD_NEXT["build-next"] + WF_ALERT_DISCORD["alert-discord"] + WF_GH_PR_ASSIGNEES["gh-pr-assignees"] + WF_GH_PROJECTS_QA["gh-projects-qa #4"] + WF_GH_PROJECTS_TIME["gh-projects-time #2"] + WF_GH_PROJECTS_BLOCKERS["gh-projects-blockers #3"] + WF_GH_SUBMODULES["gh-submodules"] + WF_GH_TRIGGER_SYNC["gh-trigger-sync #6"] + WF_GH_META_SYNC["gh-meta-sync #5"] + WF_GH_TEST_DOWNSTREAM["gh-test-downstream"] + WF_META_REGRESSION["meta-regression-analysis"] + WF_PUBLISH_GHCR["publish-ghcr"] + WF_PUBLISH_PY_PYPI["publish-py-pypi"] + WF_PUBLISH_STORYBOOK["publish-storybook"] + WF_PUBLISH_TS_NPM["publish-ts-npm"] + WF_TEST_PY_LINT["test-py-lint"] + WF_TEST_PY_PYTEST["test-py-pytest"] + WF_TEST_STORYBOOK["test-storybook"] + WF_TEST_TS_LINT["test-ts-lint"] + end + + DONE["Passed - If PR, Mergable"] + + FE_PUSH --> WF_TEST_TS_LINT + WF_TEST_TS_LINT -.-> WF_BUILD_NEXT + WF_BUILD_NEXT -.-> WF_TEST_STORYBOOK + WF_TEST_STORYBOOK -.-> WF_META_REGRESSION + FE_PUBLISH -->|If has package| WF_PUBLISH_TS_NPM + + BE_PUSH --> WF_TEST_PY_LINT + WF_TEST_PY_LINT -.-> WF_TEST_PY_PYTEST + WF_TEST_PY_PYTEST -.-> WF_META_REGRESSION + BE_PUBLISH --> WF_PUBLISH_PY_PYPI + + PUSH --> |Push to dev or main
Require meta-regression-analysis| WF_PUBLISH_GHCR + PR_MAIN --> WF_GH_TEST_DOWNSTREAM + PUBLISH -->|Require meta-regression-analysis| WF_PUBLISH_GHCR + PUSH --> WF_GH_TRIGGER_SYNC + PR_MAIN --> WF_GH_SUBMODULES + PR_DEV --> WF_GH_SUBMODULES + PR_MAIN --> WF_GH_PR_ASSIGNEES + PR_DEV --> WF_GH_PR_ASSIGNEES + ISSUE_CLOSE --> WF_GH_PROJECTS_TIME + ISSUE --> WF_GH_PROJECTS_BLOCKERS + WF_META_REGRESSION -.->|Pass
If push to dev or main| WF_PUBLISH_GHCR + WF_PUBLISH_GHCR --> DONE + WF_META_REGRESSION -->|Pass
If PR to dev| DONE + WF_META_REGRESSION -.->|Pass
If storybook| WF_PUBLISH_STORYBOOK + WF_META_REGRESSION -.->|Fail| WF_ALERT_DISCORD + WF_GH_TRIGGER_SYNC -.-> WF_GH_META_SYNC + + TIME --> WF_GH_PROJECTS_QA \ No newline at end of file From c231dc2beca0b7d5f46d93bcd5f2980062288351 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 10:14:32 -0600 Subject: [PATCH 10/44] Revert pytest; --- .github/workflows/test-py-pytest.yml | 989 ++++++++++++++++++++------- 1 file changed, 739 insertions(+), 250 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index 1d853bd..aa79b65 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -89,147 +89,136 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip --quiet - pip install pytest pytest-json-report pytest-asyncio --quiet + python -m pip install --upgrade pip + pip install pytest pytest-json-report pytest-asyncio if [ -f requirements.txt ]; then - pip install -r requirements.txt --quiet + pip install -r requirements.txt fi - - name: Silent pytest execution and collection check + - name: Check for test collection errors id: check-collection run: | - python3 << 'EOF' - import subprocess - import sys - import os + # Create verbose debug file for artifact + exec 3>&1 4>&2 + exec 1> >(tee -a debug_collection.log) 2>&1 - def run_with_complete_silence(cmd, timeout=1800): - """Run command with OS-level output suppression""" - try: - original_stdout = os.dup(1) - original_stderr = os.dup(2) - original_stdin = os.dup(0) - - try: - devnull = os.open(os.devnull, os.O_WRONLY) - devnull_read = os.open(os.devnull, os.O_RDONLY) - - os.dup2(devnull, 1) - os.dup2(devnull, 2) - os.dup2(devnull_read, 0) - - result = subprocess.run( - cmd, - timeout=timeout, - env={**os.environ, - 'PYTHONUNBUFFERED': '0', - 'PYTHONWARNINGS': 'ignore', - 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', - 'PYTEST_CURRENT_TEST': '', - 'COLUMNS': '80', - 'PYTHONDONTWRITEBYTECODE': '1'} - ) - - os.close(devnull) - os.close(devnull_read) - - finally: - os.dup2(original_stdout, 1) - os.dup2(original_stderr, 2) - os.dup2(original_stdin, 0) - os.close(original_stdout) - os.close(original_stderr) - os.close(original_stdin) - - return result - - except subprocess.TimeoutExpired: - return subprocess.CompletedProcess(cmd, 1) - except Exception as e: - return subprocess.CompletedProcess(cmd, 1) - - def run_with_captured_output(cmd, timeout=60): - """Run command and capture output for error checking""" - try: - return subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=timeout, - env={**os.environ, 'PYTHONWARNINGS': 'ignore'} - ) - except Exception: - return subprocess.CompletedProcess(cmd, 1, stderr="Execution failed") - - # Collection check - collection_cmd = [sys.executable, '-m', 'pytest', '--collect-only', '--quiet', '--tb=no'] - collection_result = run_with_captured_output(collection_cmd) - - has_collection_errors = "false" - no_tests_found = "false" - error_type = "none" - error_details = "none" - - if collection_result.returncode != 0: - if any(err in collection_result.stderr for err in ['ImportError', 'ModuleNotFoundError', 'SyntaxError', 'ERROR collecting']): - has_collection_errors = "true" - if 'ImportError' in collection_result.stderr: - error_type = "ImportError" - elif 'ModuleNotFoundError' in collection_result.stderr: - error_type = "ModuleNotFoundError" - elif 'SyntaxError' in collection_result.stderr: - error_type = "SyntaxError" - else: - error_type = "CollectionError" - error_details = collection_result.stderr[:200] if collection_result.stderr else "Collection failed" - else: - if 'collected 0 items' in collection_result.stdout or not collection_result.stdout.strip(): - no_tests_found = "true" - error_type = "NoTestsFound" - error_details = "No test files discovered" + echo "Running pytest collection check..." + python -m pytest --collect-only -v > collection_output.txt 2>&1 || true - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f'has_collection_errors={has_collection_errors}\n') - f.write(f'no_tests_found={no_tests_found}\n') - f.write(f'error_type={error_type}\n') - f.write(f'error_details={error_details}\n') - f.write(f'has_errors={str(has_collection_errors == "true" or no_tests_found == "true").lower()}\n') - - # Run tests silently if collection succeeded - if has_collection_errors == "false": - test_cmd = [ - sys.executable, '-m', 'pytest', - '--json-report', - '--json-report-file=pr_results.json', - '--tb=no', - '--no-header', - '--no-summary', - '--quiet', - '--disable-warnings', - '--log-level=CRITICAL', - '--log-cli-level=CRITICAL', - '--capture=sys', - '--maxfail=999999' - ] - - run_with_complete_silence(test_cmd) - - if os.path.exists('pr_results.json'): - print("Test results generated successfully") - else: - print("::error::Failed to create test results file") - else: - print(f"::error::Skipping tests due to collection errors: {error_type}") - EOF + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + # Set default values + HAS_COLLECTION_ERRORS="false" + NO_TESTS_FOUND="false" + ERROR_TYPE="none" + ERROR_DETAILS="none" + + if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt; then + echo "::error::Test discovery errors detected in PR branch - Python modules could not be imported correctly" + + # Attempt to identify specific error type + if grep -q "ImportError" collection_output.txt; then + ERROR_TYPE="ImportError" + elif grep -q "ModuleNotFoundError" collection_output.txt; then + ERROR_TYPE="ModuleNotFoundError" + elif grep -q "SyntaxError" collection_output.txt; then + ERROR_TYPE="SyntaxError" + elif grep -q "ERROR collecting" collection_output.txt; then + ERROR_TYPE="CollectionError" + elif grep -q "Interrupted:" collection_output.txt; then + ERROR_TYPE="Interrupted" + else + ERROR_TYPE="UnknownError" + fi + + echo "PR branch discovery error type: $ERROR_TYPE" + + ERROR_FILE=$(grep -o "ERROR collecting.*\.py" collection_output.txt | grep -o "[a-zA-Z0-9_/]*\.py" || echo "Unknown file") + + if [[ "$ERROR_FILE" != "Unknown file" ]]; then + echo "Error in file $ERROR_FILE" + grep -A 15 "$ERROR_FILE" collection_output.txt > error_details.txt + ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') + else + grep -A 15 "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt | head -20 > error_details.txt + ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') + fi + + echo "::error::PR branch discovery error details: ${ERROR_DETAILS:0:200}..." + HAS_COLLECTION_ERRORS="true" + else + echo "No discovery errors detected in PR branch" + + TEST_COUNT=$(grep -o "collected [0-9]* item" collection_output.txt | grep -o "[0-9]*" || echo "0") + + if [[ "$TEST_COUNT" == "0" ]]; then + echo "::warning::No tests were found in the PR branch" + NO_TESTS_FOUND="true" + ERROR_TYPE="NoTestsFound" + ERROR_DETAILS="No test files were discovered that match pytest's test discovery pattern" + else + echo "Found $TEST_COUNT tests in PR branch" + fi + fi + + # Set all the outputs + echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT + echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT + echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT + echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT + + # For backward compatibility + if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then + echo "has_errors=true" >> $GITHUB_OUTPUT + else + echo "has_errors=false" >> $GITHUB_OUTPUT + fi + + # Save full collection output to debug file for artifact + echo "=== FULL COLLECTION OUTPUT ===" >> debug_collection.log + cat collection_output.txt >> debug_collection.log + + - name: Run tests on PR Branch + if: steps.check-collection.outputs.has_collection_errors != 'true' + run: | + # Create verbose debug file for artifact + exec 3>&1 4>&2 + exec 1> >(tee -a debug_test_run.log) 2>&1 + + echo "Running tests on PR branch..." + python -m pytest -vv --json-report --json-report-file=pr_results.json || true + + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + if [ -f pr_results.json ]; then + echo "Test results file successfully created for PR branch" + # Save first 200 chars to debug file only + echo "=== JSON RESULTS PREVIEW ===" >> debug_test_run.log + head -c 200 pr_results.json >> debug_test_run.log + else + echo "::error::Failed to create test results file for PR branch" + fi - name: Extract test results and create artifacts id: extract-results run: | + echo "PR_BRANCH=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV + echo "Processing test results for PR branch: $PR_BRANCH" + + # Create debug file for detailed output + exec 3>&1 4>&2 + exec 1> >(tee -a debug_extract_results.log) 2>&1 + python -c " import json import sys import os + print('Starting test results extraction script for PR branch') + + # Default values in case file doesn't exist or is invalid pr_total = 0 pr_passed = 0 pr_percentage = 0 @@ -241,37 +230,52 @@ jobs: xfailed_tests_with_reasons = {} try: + print('Attempting to open pr_results.json') with open('pr_results.json') as f: pr_results = json.load(f) + print(f'JSON loaded successfully, keys: {list(pr_results.keys())}') + # Check for collection errors by looking at exitcode or error patterns if pr_results.get('exitcode', 0) > 1: - pr_total = 0 + print('Detected non-zero exitcode, likely a collection error') + if 'collectors' in pr_results and pr_results['collectors']: + print(f'Collection errors found: {pr_results[\"collectors\"]}') + pr_total = 0 # Explicitly set to 0 - no tests run when collection fails pr_passed = 0 elif 'summary' in pr_results and isinstance(pr_results['summary'], dict): + # Normal case - extract data from summary summary = pr_results['summary'] pr_total = summary.get('total', 0) pr_passed = summary.get('passed', 0) + print(f'Results extracted from summary - Total: {pr_total}, Passed: {pr_passed}') + # Extract all tests by outcome and collect all test nodeids with reasons if 'tests' in pr_results: + print('Extracting failing, skipped, xfailed, and all tests with reasons') for test in pr_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') if nodeid: - all_tests.append(nodeid) + all_tests.append(nodeid) # Track all tests regardless of outcome if outcome in ['failed', 'error']: failing_tests.append(nodeid) elif outcome == 'skipped': skipped_tests.append(nodeid) + # Extract skip reason skip_reason = 'No reason provided' if 'longrepr' in test and test['longrepr']: + # longrepr can be a string or list, handle both longrepr = test['longrepr'] if isinstance(longrepr, list) and longrepr: skip_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' elif isinstance(longrepr, str): skip_reason = longrepr + elif 'call' in test and test['call'] and 'longrepr' in test['call']: + skip_reason = str(test['call']['longrepr']) skipped_tests_with_reasons[nodeid] = skip_reason.strip() elif outcome == 'xfailed': xfailed_tests.append(nodeid) + # Extract xfail reason xfail_reason = 'No reason provided' if 'longrepr' in test and test['longrepr']: longrepr = test['longrepr'] @@ -279,13 +283,43 @@ jobs: xfail_reason = str(longrepr[0]) if longrepr[0] else 'No reason provided' elif isinstance(longrepr, str): xfail_reason = longrepr + elif 'call' in test and test['call'] and 'longrepr' in test['call']: + xfail_reason = str(test['call']['longrepr']) xfailed_tests_with_reasons[nodeid] = xfail_reason.strip() + + print(f'Found {len(failing_tests)} failing tests') + print(f'Found {len(skipped_tests)} skipped tests') + print(f'Found {len(xfailed_tests)} xfailed tests') + print(f'Found {len(all_tests)} total discovered tests') + else: + print('No valid summary structure found') + # Calculate percentage safely pr_percentage = (pr_passed / pr_total * 100) if pr_total > 0 else 0 + print(f'Pass percentage calculated: {pr_percentage:.2f}%') + except FileNotFoundError as e: + print(f'File not found error: {e}') + except KeyError as e: + print(f'Missing key in results file: {e}') + if 'pr_results' in locals(): + print(f'Available keys: {list(pr_results.keys())}') + if 'summary' in pr_results: + print(f'Summary structure: {pr_results[\"summary\"]}') except Exception as e: - pass - + print(f'Error processing results: {e}') + import traceback + print(f'Full exception: {traceback.format_exc()}') + + print(f'Total tests: {pr_total}') + print(f'Passed tests: {pr_passed}') + print(f'Pass percentage: {pr_percentage:.2f}%') + print(f'Failing tests: {len(failing_tests)}') + print(f'Skipped tests: {len(skipped_tests)}') + print(f'Xfailed tests: {len(xfailed_tests)}') + print(f'All discovered tests: {len(all_tests)}') + + # Set scalar outputs only (no large arrays) with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'total={pr_total}\\n') f.write(f'passed={pr_passed}\\n') @@ -294,6 +328,7 @@ jobs: f.write(f'skipped_count={len(skipped_tests)}\\n') f.write(f'xfailed_count={len(xfailed_tests)}\\n') + # Save test lists to artifact files instead of job outputs test_data = { 'failing_tests': failing_tests, 'skipped_tests': skipped_tests, @@ -305,8 +340,16 @@ jobs: with open('pr_test_data.json', 'w') as f: json.dump(test_data, f, indent=2) + + print('Test data saved to pr_test_data.json for artifact') + print('Results extraction completed') " + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + echo "PR branch test results processed: ${{ steps.extract-results.outputs.passed }}/${{ steps.extract-results.outputs.total }} tests passed (${{ steps.extract-results.outputs.percentage }}%)" + - name: Upload PR branch artifacts if: always() uses: actions/upload-artifact@v4 @@ -314,8 +357,10 @@ jobs: name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | pr_test_data.json + debug_*.log + collection_output.txt + error_details.txt pr_results.json - collection_check.txt retention-days: 3 if-no-files-found: ignore @@ -346,146 +391,137 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip --quiet - pip install pytest pytest-json-report pytest-asyncio --quiet + python -m pip install --upgrade pip + pip install pytest pytest-json-report pytest-asyncio if [ -f requirements.txt ]; then - pip install -r requirements.txt --quiet + pip install -r requirements.txt fi - name: Check for test collection errors id: check-collection run: | + # Create verbose debug file for artifact + exec 3>&1 4>&2 + exec 1> >(tee -a debug_target_collection.log) 2>&1 + + echo "Running pytest collection check..." + python -m pytest --collect-only -v > collection_output.txt 2>&1 || true + + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + # Set default values HAS_COLLECTION_ERRORS="false" NO_TESTS_FOUND="false" ERROR_TYPE="none" ERROR_DETAILS="none" - python -m pytest --collect-only --quiet --tb=no >/dev/null 2>collection_check.txt - COLLECTION_EXIT_CODE=$? - - if [ $COLLECTION_EXIT_CODE -ne 0 ]; then - if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting" collection_check.txt; then - echo "::warning::Test discovery errors detected in target branch" - HAS_COLLECTION_ERRORS="true" - - if grep -q "ImportError" collection_check.txt; then - ERROR_TYPE="ImportError" - elif grep -q "ModuleNotFoundError" collection_check.txt; then - ERROR_TYPE="ModuleNotFoundError" - elif grep -q "SyntaxError" collection_check.txt; then - ERROR_TYPE="SyntaxError" - else - ERROR_TYPE="CollectionError" - fi - - ERROR_DETAILS=$(head -1 collection_check.txt | tr '\n' ' ' | cut -c1-200) + if grep -q "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt; then + echo "::warning::Test discovery errors detected in target branch - Python modules could not be imported correctly" + + # Attempt to identify specific error type + if grep -q "ImportError" collection_output.txt; then + ERROR_TYPE="ImportError" + elif grep -q "ModuleNotFoundError" collection_output.txt; then + ERROR_TYPE="ModuleNotFoundError" + elif grep -q "SyntaxError" collection_output.txt; then + ERROR_TYPE="SyntaxError" + elif grep -q "ERROR collecting" collection_output.txt; then + ERROR_TYPE="CollectionError" + elif grep -q "Interrupted:" collection_output.txt; then + ERROR_TYPE="Interrupted" + else + ERROR_TYPE="UnknownError" fi + + echo "Target branch discovery error type: $ERROR_TYPE" + + ERROR_FILE=$(grep -o "ERROR collecting.*\.py" collection_output.txt | grep -o "[a-zA-Z0-9_/]*\.py" || echo "Unknown file") + + if [[ "$ERROR_FILE" != "Unknown file" ]]; then + echo "Error in file $ERROR_FILE" + grep -A 15 "$ERROR_FILE" collection_output.txt > error_details.txt + ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') + else + grep -A 15 "ImportError\|ModuleNotFoundError\|SyntaxError\|ERROR collecting\|Interrupted:" collection_output.txt | head -20 > error_details.txt + ERROR_DETAILS=$(cat error_details.txt | tr '\n' ' ' | sed 's/"/\\"/g') + fi + + echo "::warning::Target branch discovery error details: ${ERROR_DETAILS:0:200}..." + HAS_COLLECTION_ERRORS="true" else - TEST_COUNT=$(python -m pytest --collect-only --quiet 2>/dev/null | grep -o "collected [0-9]* item" | grep -o "[0-9]*" || echo "0") + echo "No discovery errors detected in target branch" + + TEST_COUNT=$(grep -o "collected [0-9]* item" collection_output.txt | grep -o "[0-9]*" || echo "0") + if [[ "$TEST_COUNT" == "0" ]]; then echo "::warning::No tests were found in the target branch" NO_TESTS_FOUND="true" ERROR_TYPE="NoTestsFound" - ERROR_DETAILS="No test files discovered" + ERROR_DETAILS="No test files were discovered in target branch that match pytest's test discovery pattern" + else + echo "Found $TEST_COUNT tests in target branch" fi fi + # Set all the outputs echo "has_collection_errors=$HAS_COLLECTION_ERRORS" >> $GITHUB_OUTPUT echo "no_tests_found=$NO_TESTS_FOUND" >> $GITHUB_OUTPUT echo "error_type=$ERROR_TYPE" >> $GITHUB_OUTPUT echo "error_details=$ERROR_DETAILS" >> $GITHUB_OUTPUT + # For backward compatibility if [[ "$HAS_COLLECTION_ERRORS" == "true" || "$NO_TESTS_FOUND" == "true" ]]; then echo "has_errors=true" >> $GITHUB_OUTPUT else echo "has_errors=false" >> $GITHUB_OUTPUT fi + # Save full collection output to debug file for artifact + echo "=== FULL COLLECTION OUTPUT ===" >> debug_target_collection.log + cat collection_output.txt >> debug_target_collection.log + - name: Run tests on target branch if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - python3 << 'EOF' - import subprocess - import sys - import os + # Create verbose debug file for artifact + exec 3>&1 4>&2 + exec 1> >(tee -a debug_target_test_run.log) 2>&1 - def run_with_complete_silence(cmd, timeout=1800): - """Run command with OS-level output suppression""" - try: - original_stdout = os.dup(1) - original_stderr = os.dup(2) - original_stdin = os.dup(0) - - try: - devnull = os.open(os.devnull, os.O_WRONLY) - devnull_read = os.open(os.devnull, os.O_RDONLY) - - os.dup2(devnull, 1) - os.dup2(devnull, 2) - os.dup2(devnull_read, 0) - - result = subprocess.run( - cmd, - timeout=timeout, - env={**os.environ, - 'PYTHONUNBUFFERED': '0', - 'PYTHONWARNINGS': 'ignore', - 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1', - 'PYTEST_CURRENT_TEST': '', - 'COLUMNS': '80', - 'PYTHONDONTWRITEBYTECODE': '1'} - ) - - os.close(devnull) - os.close(devnull_read) - - finally: - os.dup2(original_stdout, 1) - os.dup2(original_stderr, 2) - os.dup2(original_stdin, 0) - os.close(original_stdout) - os.close(original_stderr) - os.close(original_stdin) - - return result - - except subprocess.TimeoutExpired: - return subprocess.CompletedProcess(cmd, 1) - except Exception as e: - return subprocess.CompletedProcess(cmd, 1) - - test_cmd = [ - sys.executable, '-m', 'pytest', - '--json-report', - '--json-report-file=target_results.json', - '--tb=no', - '--no-header', - '--no-summary', - '--quiet', - '--disable-warnings', - '--log-level=CRITICAL', - '--log-cli-level=CRITICAL', - '--capture=sys', - '--maxfail=999999' - ] - - run_with_complete_silence(test_cmd) - - if os.path.exists('target_results.json'): - print("Test results generated successfully") - else: - print("::warning::Failed to create test results file") - EOF + echo "Running tests on target branch..." + python -m pytest -vv --json-report --json-report-file=target_results.json || true + + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + if [ -f target_results.json ]; then + echo "Test results file successfully created for target branch" + # Save first 200 chars to debug file only + echo "=== JSON RESULTS PREVIEW ===" >> debug_target_test_run.log + head -c 200 target_results.json >> debug_target_test_run.log + else + echo "::warning::Failed to create test results file for target branch" + fi - name: Extract test results and create artifacts id: extract-results + # Only run if there were no collection errors if: steps.check-collection.outputs.has_collection_errors != 'true' run: | + echo "Processing test results for target branch: ${{ inputs.target_branch_to_compare }}" + + # Create debug file for detailed output + exec 3>&1 4>&2 + exec 1> >(tee -a debug_target_extract_results.log) 2>&1 + python -c " import json import sys import os + print('Starting test results extraction script for target branch') + + # Default values in case file doesn't exist or is invalid target_total = 0 target_passed = 0 target_percentage = 0 @@ -493,37 +529,72 @@ jobs: all_tests = [] try: + print('Attempting to open target_results.json') with open('target_results.json') as f: target_results = json.load(f) + print(f'JSON loaded successfully, keys: {list(target_results.keys())}') + # Check for collection errors by looking at exitcode or error patterns if target_results.get('exitcode', 0) > 1: - target_total = 0 + print('Detected non-zero exitcode, likely a collection error') + if 'collectors' in target_results and target_results['collectors']: + print(f'Collection errors found: {target_results[\"collectors\"]}') + target_total = 0 # Explicitly set to 0 - no tests run when collection fails target_passed = 0 elif 'summary' in target_results and isinstance(target_results['summary'], dict): + # Normal case - extract data from summary summary = target_results['summary'] target_total = summary.get('total', 0) target_passed = summary.get('passed', 0) + print(f'Results extracted from summary - Total: {target_total}, Passed: {target_passed}') + # Extract passing tests and all tests if 'tests' in target_results: + print('Extracting passing tests and all discovered tests') for test in target_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') if nodeid: - all_tests.append(nodeid) + all_tests.append(nodeid) # Track all tests regardless of outcome if outcome == 'passed': passing_tests.append(nodeid) + + print(f'Found {len(passing_tests)} passing tests') + print(f'Found {len(all_tests)} total discovered tests') + else: + print('No valid summary structure found') + # Calculate percentage safely target_percentage = (target_passed / target_total * 100) if target_total > 0 else 0 + print(f'Pass percentage calculated: {target_percentage:.2f}%') + except FileNotFoundError as e: + print(f'File not found error: {e}') + except KeyError as e: + print(f'Missing key in results file: {e}') + if 'target_results' in locals(): + print(f'Available keys: {list(target_results.keys())}') + if 'summary' in target_results: + print(f'Summary structure: {target_results[\"summary\"]}') except Exception as e: - pass + print(f'Error processing results: {e}') + import traceback + print(f'Full exception: {traceback.format_exc()}') + print(f'Total tests: {target_total}') + print(f'Passed tests: {target_passed}') + print(f'Pass percentage: {target_percentage:.2f}%') + print(f'Passing tests: {len(passing_tests)}') + print(f'All discovered tests: {len(all_tests)}') + + # Set scalar outputs only (no large arrays) with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'total={target_total}\\n') f.write(f'passed={target_passed}\\n') f.write(f'percentage={target_percentage:.2f}\\n') f.write(f'passing_count={len(passing_tests)}\\n') + # Save test lists to artifact files instead of job outputs test_data = { 'passing_tests': passing_tests, 'all_tests': all_tests @@ -531,8 +602,16 @@ jobs: with open('target_test_data.json', 'w') as f: json.dump(test_data, f, indent=2) + + print('Test data saved to target_test_data.json for artifact') + print('Results extraction completed') " + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + echo "Target branch test results processed: ${{ steps.extract-results.outputs.passed }}/${{ steps.extract-results.outputs.total }} tests passed (${{ steps.extract-results.outputs.percentage }}%)" + - name: Upload target branch artifacts if: always() uses: actions/upload-artifact@v4 @@ -540,15 +619,19 @@ jobs: name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | target_test_data.json + debug_target_*.log + collection_output.txt + error_details.txt target_results.json - collection_check.txt retention-days: 3 if-no-files-found: ignore + # Add a step to set default outputs when collection errors are detected - name: Set collection error outputs id: set-error-outputs if: steps.check-collection.outputs.has_collection_errors == 'true' run: | + echo "::warning::Setting default outputs for target branch due to collection errors" echo "total=0" >> $GITHUB_OUTPUT echo "passed=0" >> $GITHUB_OUTPUT echo "percentage=0.00" >> $GITHUB_OUTPUT @@ -576,62 +659,120 @@ jobs: - name: Check for collection errors run: | + # Create analysis debug file + exec 3>&1 4>&2 + exec 1> >(tee -a debug_comparison_analysis.log) 2>&1 + + echo "Retrieving collection error status information" PR_COLLECTION_ERRORS="${{ needs.test-source-branch.outputs.collection_errors }}" PR_NO_TESTS="${{ needs.test-source-branch.outputs.no_tests_found }}" PR_ERROR_TYPE="${{ needs.test-source-branch.outputs.error_type }}" + PR_ERROR_DETAILS="${{ needs.test-source-branch.outputs.error_details }}" TARGET_COLLECTION_ERRORS="${{ needs.test-target-branch.outputs.collection_errors }}" + echo "PR branch collection errors: $PR_COLLECTION_ERRORS" + echo "PR branch no tests found: $PR_NO_TESTS" + echo "PR branch error type: $PR_ERROR_TYPE" + echo "Target branch collection errors: $TARGET_COLLECTION_ERRORS" + + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + # Distinct error handling for PR branch if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then echo "::error::Test discovery errors in PR branch: $PR_ERROR_TYPE" + echo "::error::$PR_ERROR_DETAILS" + echo "❌ PR branch has test discovery errors. Python modules could not be imported correctly." exit 1 fi if [[ "$PR_NO_TESTS" == "true" ]]; then echo "::error::No tests were found in the PR branch" + echo "❌ PR branch has no tests detected. Please add test files that match pytest's discovery pattern." exit 1 fi + # Warning for target branch issues (not a failure) if [[ "$TARGET_COLLECTION_ERRORS" == "true" ]]; then - echo "⚠️ Target branch has test discovery errors." + echo "⚠️ Target branch has test discovery errors. Tests will still be compared but results may not be accurate." + fi + + if [[ "${{ needs.test-target-branch.outputs.no_tests_found }}" == "true" ]]; then + echo "⚠️ Target branch has no tests detected. PR branch tests will still be evaluated." fi - name: Run regression analysis from artifacts run: | + # Create analysis debug file + exec 3>&1 4>&2 + exec 1> >(tee -a debug_regression_analysis.log) 2>&1 + + echo "Running regression analysis from artifact data..." + python3 - << 'EOF' import json import os - import glob try: + # Load test data from artifacts target_data = {} pr_data = {} - target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') + # Try to load target branch data + target_artifact_path = './artifacts/target_branch_data_*/target_test_data.json' + import glob + target_files = glob.glob(target_artifact_path) if target_files: with open(target_files[0], 'r') as f: target_data = json.load(f) + print(f"Loaded target data from {target_files[0]}") + else: + print("No target branch test data found") - pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') + # Try to load PR branch data + pr_artifact_path = './artifacts/pr_branch_data_*/pr_test_data.json' + pr_files = glob.glob(pr_artifact_path) if pr_files: with open(pr_files[0], 'r') as f: pr_data = json.load(f) + print(f"Loaded PR data from {pr_files[0]}") + else: + print("No PR branch test data found") + # Extract test arrays target_passing = target_data.get('passing_tests', []) pr_failing = pr_data.get('failing_tests', []) + print(f"Parsed {len(target_passing)} passing tests from target branch") + print(f"Parsed {len(pr_failing)} failing tests from PR branch") + + # Find regressions using set operations target_passing_set = set(target_passing) pr_failing_set = set(pr_failing) regression_tests = list(target_passing_set.intersection(pr_failing_set)) + # Write results to file if there are regressions if regression_tests: + print(f"Found {len(regression_tests)} regression(s)!") + with open("regression_details.txt", "w") as f: f.write(f"Found {len(regression_tests)} tests that were passing in target branch but now failing in PR branch:\\n\\n") for idx, test in enumerate(sorted(regression_tests), 1): f.write(f"{idx}. {test}\\n") + print("Regression details written to file") + else: + print("No regressions found") except Exception as e: - pass + print(f"Error in regression analysis: {e}") + import traceback + print(traceback.format_exc()) EOF + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + echo "Regression analysis completed" + - name: Check for regression details file id: check-regressions run: | @@ -639,17 +780,39 @@ jobs: _regression_count="0" if [ -f "regression_details.txt" ]; then + echo "Regression details file exists" + # Count regression lines (lines starting with a number and period) _current_count=$(grep -c "^[0-9]\+\." regression_details.txt || echo "0") + echo "Found $_current_count regression items in file" + if [ "$_current_count" -gt 0 ]; then _has_regressions="true" _regression_count="$_current_count" - echo "::error::Test Regressions Found: $_regression_count test(s) that were passing in target branch are now failing in PR branch." + echo "::error::Test Regressions Found: $_regression_count test(s) that were passing in target branch are now **failing** in PR branch." + echo "Regression details:" + cat regression_details.txt + else + # File exists but no regressions counted (e.g. empty or malformed) + _has_regressions="false" + _regression_count="0" fi + else + echo "No regression details file found - no regressions detected" + _has_regressions="false" + _regression_count="0" fi echo "HAS_REGRESSIONS=$_has_regressions" >> $GITHUB_OUTPUT echo "REGRESSION_COUNT=$_regression_count" >> $GITHUB_OUTPUT + if [[ "$_has_regressions" == "false" ]]; then + if [ -f regression_details.txt ] && [ "$_has_regressions" == "false" ]; then + echo "::notice::Regression details file (regression_details.txt) was found but no valid regression entries were counted by this step, or the file was empty." + else + echo "No test regressions detected by this step." + fi + fi + - name: Upload regression details artifact if: steps.check-regressions.outputs.HAS_REGRESSIONS == 'true' && steps.check-regressions.outputs.REGRESSION_COUNT > 0 uses: actions/upload-artifact@v4 @@ -658,6 +821,188 @@ jobs: path: regression_details.txt retention-days: 1 + - name: Check for skip/xfail regressions from artifacts + run: | + # Create analysis debug file + exec 3>&1 4>&2 + exec 1> >(tee -a debug_skip_xfail_analysis.log) 2>&1 + + echo "Checking for tests that were passing in target but are now skipped or xfailed in PR" + + python3 - << 'EOF' + import json + import os + import glob + + try: + # Load test data from artifacts + target_data = {} + pr_data = {} + + # Load target and PR data + target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') + if target_files: + with open(target_files[0], 'r') as f: + target_data = json.load(f) + + pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') + if pr_files: + with open(pr_files[0], 'r') as f: + pr_data = json.load(f) + + # Extract test arrays + target_passing = target_data.get('passing_tests', []) + pr_skipped = pr_data.get('skipped_tests', []) + pr_xfailed = pr_data.get('xfailed_tests', []) + pr_skipped_with_reasons = pr_data.get('skipped_tests_with_reasons', {}) + pr_xfailed_with_reasons = pr_data.get('xfailed_tests_with_reasons', {}) + + print(f"Parsed {len(target_passing)} passing tests from target branch") + print(f"Parsed {len(pr_skipped)} skipped tests from PR branch") + print(f"Parsed {len(pr_xfailed)} xfailed tests from PR branch") + print(f"Parsed {len(pr_skipped_with_reasons)} skipped tests with reasons") + print(f"Parsed {len(pr_xfailed_with_reasons)} xfailed tests with reasons") + + # Find tests that were passing in target but are now skipped or xfailed in PR + target_passing_set = set(target_passing) + pr_skipped_set = set(pr_skipped) + pr_xfailed_set = set(pr_xfailed) + + passing_to_skipped = list(target_passing_set.intersection(pr_skipped_set)) + passing_to_xfailed = list(target_passing_set.intersection(pr_xfailed_set)) + + total_skip_xfail_regressions = len(passing_to_skipped) + len(passing_to_xfailed) + + if total_skip_xfail_regressions > 0: + print(f"Found {total_skip_xfail_regressions} tests that were passing in target but are now skipped/xfailed in PR!") + + # Build comprehensive warning message + warning_parts = [f"Skip/XFail Analysis: {total_skip_xfail_regressions} test(s) that were passing in target branch are now being skipped or xfailed in PR branch."] + + if passing_to_skipped: + warning_parts.append(f"Tests now SKIPPED ({len(passing_to_skipped)}):") + for idx, test in enumerate(sorted(passing_to_skipped), 1): + reason = pr_skipped_with_reasons.get(test, 'No reason provided') + warning_parts.append(f" {idx}. {test} - Reason: {reason}") + + if passing_to_xfailed: + warning_parts.append(f"Tests now XFAILED ({len(passing_to_xfailed)}):") + for idx, test in enumerate(sorted(passing_to_xfailed), 1): + reason = pr_xfailed_with_reasons.get(test, 'No reason provided') + warning_parts.append(f" {idx}. {test} - Reason: {reason}") + + warning_parts.append("While these changes don't fail the workflow, they indicate tests that were working before are now being bypassed. Please review these tests to ensure this is intentional.") + + # Print as single warning annotation + combined_warning = " ".join(warning_parts) + print(f"::warning::{combined_warning}") + else: + print("No skip/xfail regressions found - all previously passing tests are still running.") + except Exception as e: + print(f"Error in skip/xfail analysis: {e}") + import traceback + print(traceback.format_exc()) + EOF + + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + echo "Skip/xfail regression analysis completed" + + - name: Check for test additions and removals from artifacts + run: | + # Create analysis debug file + exec 3>&1 4>&2 + exec 1> >(tee -a debug_test_changes_analysis.log) 2>&1 + + echo "Checking for test additions and removals between target and PR branches" + + python3 - << 'EOF' + import json + import os + import glob + + try: + # Load test data from artifacts + target_data = {} + pr_data = {} + + # Load target and PR data + target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') + if target_files: + with open(target_files[0], 'r') as f: + target_data = json.load(f) + + pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') + if pr_files: + with open(pr_files[0], 'r') as f: + pr_data = json.load(f) + + # Extract test arrays + target_all = target_data.get('all_tests', []) + pr_all = pr_data.get('all_tests', []) + + print(f"Parsed {len(target_all)} total tests from target branch") + print(f"Parsed {len(pr_all)} total tests from PR branch") + + # Find test additions and removals using set operations + target_all_set = set(target_all) + pr_all_set = set(pr_all) + + removed_tests = list(target_all_set - pr_all_set) # In target but not in PR + added_tests = list(pr_all_set - target_all_set) # In PR but not in target + + # Report removed tests (warnings) + if removed_tests: + print(f"Found {len(removed_tests)} tests that were removed from target branch!") + + # Build comprehensive removal warning + removal_parts = [f"Test Removal Analysis: {len(removed_tests)} test(s) that existed in target branch are missing from PR branch."] + removal_parts.append(f"REMOVED Tests ({len(removed_tests)}):") + for idx, test in enumerate(sorted(removed_tests), 1): + removal_parts.append(f" {idx}. {test}") + removal_parts.append("These test removals should be reviewed to ensure they are intentional. If tests were renamed or moved, this may show as removal + addition.") + + # Print as single warning annotation + combined_removal_warning = " ".join(removal_parts) + print(f"::warning::{combined_removal_warning}") + else: + print("No test removals detected.") + + # Report added tests (notifications/info) + if added_tests: + print(f"Found {len(added_tests)} new tests added in PR branch!") + + # Build comprehensive addition notice + addition_parts = [f"Test Addition Analysis: {len(added_tests)} new test(s) have been added in the PR branch."] + addition_parts.append(f"NEW Tests ({len(added_tests)}):") + for idx, test in enumerate(sorted(added_tests), 1): + addition_parts.append(f" {idx}. {test}") + addition_parts.append("New tests detected - this indicates expanded test coverage!") + + # Print as single notice annotation + combined_addition_notice = " ".join(addition_parts) + print(f"::notice::{combined_addition_notice}") + else: + print("No new tests detected in PR branch.") + + # Summary + if not removed_tests and not added_tests: + print("Test suite composition is unchanged between target and PR branches.") + else: + print(f"Test suite changes: {len(added_tests)} added, {len(removed_tests)} removed") + + except Exception as e: + print(f"Error in test addition/removal analysis: {e}") + import traceback + print(traceback.format_exc()) + EOF + + # Restore stdout/stderr for GitHub Actions + exec 1>&3 2>&4 + + echo "Test addition/removal analysis completed" + - name: Compare test results run: | echo "Test Results Summary:" @@ -666,6 +1011,7 @@ jobs: if [[ "${{ needs.test-source-branch.outputs.total }}" == "0" ]]; then echo "::error::No tests were found in the PR branch" + echo "❌ PR branch has no tests detected. Please add test files that match pytest's discovery pattern." exit 1 fi @@ -676,29 +1022,89 @@ jobs: PR_TOTAL=${{ needs.test-source-branch.outputs.total }} TARGET_TOTAL=${{ needs.test-target-branch.outputs.total }} + # Handle case where target has no tests if [[ "$TARGET_TOTAL" == "0" ]]; then if [[ "$PR_PASSED" -gt 0 ]]; then echo "✅ PR branch has tests and some are passing (target branch has no tests)" exit 0 else echo "❌ PR branch has no passing tests" + echo " - Pass percentage: $PR_PERCENTAGE%" exit 1 fi fi + # Fail if any tests passed in target branch but now fail in PR branch if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then echo "❌ PR branch has test regressions from target branch" + REGRESSION_COUNT_VAL=${{ needs.perform-regression-analysis.outputs.regression_count }} + echo " - $REGRESSION_COUNT_VAL tests that were passing in target branch are now failing" + + echo "### :x: Test Regressions Detected!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**$REGRESSION_COUNT_VAL test(s) that were passing in the target branch are now failing in the PR branch.**" >> $GITHUB_STEP_SUMMARY + echo "This job (\`compare-results\`) has been marked as failed due to these regressions." >> $GITHUB_STEP_SUMMARY + + if [ -f regression_details.txt ]; then # Check if file exists (it should from previous step) + echo "Failing tests (regressions) list:" + cat regression_details.txt + else + echo "::warning::Regression details file (regression_details.txt) not found in Compare test results step. It might be available as an artifact from the 'perform-regression-analysis' job." + fi exit 1 fi + # Continue with the original comparison if no regressions if (( $(echo "$PR_PASSED >= $TARGET_PASSED" | bc -l) )) && (( $(echo "$PR_PERCENTAGE >= $TARGET_PERCENTAGE" | bc -l) )); then echo "✅ PR branch has equal or better test results than target branch" + + # Additional verbose information about improvement + if (( $(echo "$PR_PASSED > $TARGET_PASSED" | bc -l) )); then + IMPROVEMENT=$(( $PR_PASSED - $TARGET_PASSED )) + echo " - Improvement: $IMPROVEMENT more passing tests than target branch" + fi + + if (( $(echo "$PR_PERCENTAGE > $TARGET_PERCENTAGE" | bc -l) )); then + PERCENTAGE_IMPROVEMENT=$(echo "$PR_PERCENTAGE - $TARGET_PERCENTAGE" | bc -l) + echo " - Percentage improvement: +${PERCENTAGE_IMPROVEMENT}% compared to target branch" + fi + exit 0 else echo "❌ PR branch has worse test results than target branch" + echo " - Passed tests: $PR_PASSED vs $TARGET_PASSED on target branch" + echo " - Pass percentage: $PR_PERCENTAGE% vs $TARGET_PERCENTAGE% on target branch" + + # Add to job summary for general comparison failure + echo "### :x: Test Comparison Failed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The PR branch has worse test results than the target branch:" >> $GITHUB_STEP_SUMMARY + echo "- Passed tests: $PR_PASSED (PR) vs $TARGET_PASSED (Target)" >> $GITHUB_STEP_SUMMARY + echo "- Pass percentage: $PR_PERCENTAGE% (PR) vs $TARGET_PERCENTAGE% (Target)" >> $GITHUB_STEP_SUMMARY + + # Calculate regression metrics + if (( $(echo "$PR_PASSED < $TARGET_PASSED" | bc -l) )); then + REGRESSION=$(( $TARGET_PASSED - $PR_PASSED )) + echo " - Regression: $REGRESSION fewer passing tests than target branch" + fi + + if (( $(echo "$PR_PERCENTAGE < $TARGET_PERCENTAGE" | bc -l) )); then + PERCENTAGE_REGRESSION=$(echo "$TARGET_PERCENTAGE - $PR_PERCENTAGE" | bc -l) + echo " - Percentage regression: -${PERCENTAGE_REGRESSION}% compared to target branch" + fi + exit 1 fi + - name: Upload comparison analysis logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: comparison_analysis_logs_${{ github.event.pull_request.number || github.run_id }} + path: debug_*_analysis.log + retention-days: 3 + if-no-files-found: ignore + perform-regression-analysis: needs: [test-source-branch, test-target-branch] uses: ./.github/workflows/meta-regression-analysis.yml @@ -710,6 +1116,7 @@ jobs: target_branch_artifact_name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} pr_branch_artifact_name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} + # Conditionally run notification job only if needed prepare-notification: name: Prepare Notification Data needs: @@ -719,6 +1126,7 @@ jobs: compare-results, perform-regression-analysis, ] + # Notify on collection errors, no tests found, compare result failure, or if regressions are detected if: | always() && ( @@ -740,23 +1148,24 @@ jobs: id: check_webhook_availability run: | if [ -z "${{ secrets.DISCORD_WEBHOOK_URL }}" ]; then + echo "::notice::DISCORD_WEBHOOK_URL secret is not set. Discord notifications will likely be skipped by the alert workflow if it relies on this secret." echo "webhook_available=false" >> $GITHUB_OUTPUT else echo "webhook_available=true" >> $GITHUB_OUTPUT fi - - name: Download regression details (if any) id: download_regressions if: needs.perform-regression-analysis.outputs.has_regressions == 'true' && needs.perform-regression-analysis.outputs.regression_count > 0 uses: actions/download-artifact@v4 with: name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests - path: . + path: . # Download to current directory continue-on-error: true - name: Construct Discord Notification id: construct_notification env: + LINT_RESULT: ${{ needs.lint.result }} SOURCE_TEST_RESULT: ${{ needs.test-source-branch.result }} TARGET_TEST_RESULT: ${{ needs.test-target-branch.result }} COMPARE_RESULT: ${{ needs.compare-results.result }} @@ -780,47 +1189,82 @@ jobs: REPO_URL: ${{ github.server_url }}/${{ github.repository }} ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_ASSIGNEES_JSON: ${{ toJson(github.event.pull_request.assignees) }} - USER_MAP_JSON: ${{ secrets.DISCORD_USER_MAP }} + USER_MAP_JSON: ${{ secrets.DISCORD_USER_MAP || '{}' }} REGRESSION_FILE_PATH: "regression_details.txt" DOWNLOAD_REGRESSIONS_OUTCOME: ${{ steps.download_regressions.outcome }} INPUT_PING_LATEST_COMMITTER: ${{ inputs.ping_latest_committer }} run: | - MESSAGE_LINES=() - PING_KEYS_OUTPUT="" + # Create debug file for detailed notification construction + exec 3>&1 4>&2 + exec 1> >(tee -a debug_notification_construction.log) 2>&1 + + MESSAGE_LINES=() # Use an array to build message lines + PING_KEYS_OUTPUT="" # Will be comma-separated GitHub logins ARTIFACT_PATH_OUTPUT="" + echo "Raw GH_ASSIGNEES_JSON value: [$GH_ASSIGNEES_JSON]" + echo "Raw USER_MAP_JSON value: [$USER_MAP_JSON]" + + # 1. Determine Pings - Collect GitHub Logins to pass to alert-discord.yml + # Initialize PING_KEYS_OUTPUT PING_KEYS_OUTPUT="" + # Add assignees to PING_KEYS_OUTPUT if [ -n "$USER_MAP_JSON" ] && [ "$USER_MAP_JSON" != "{}" ] && command -v jq &> /dev/null; then ASSIGNEE_LOGINS_ARRAY=($(echo "$GH_ASSIGNEES_JSON" | jq -r '.[].login // empty')) + echo "GH_ASSIGNEES_JSON received: $GH_ASSIGNEES_JSON" + echo "Extracted ASSIGNEE_LOGINS_ARRAY: (${ASSIGNEE_LOGINS_ARRAY[*]})" + echo "Count of assignees extracted: ${#ASSIGNEE_LOGINS_ARRAY[@]}" + MAPPED_ASSIGNEE_COUNT=0 TEMP_PING_KEYS=() for assignee_login in "${ASSIGNEE_LOGINS_ARRAY[@]}"; do if [ -z "$assignee_login" ]; then + echo "Skipping empty assignee login." continue fi + echo "Processing assignee for ping: '$assignee_login'" + # Check if this assignee_login exists as a key in USER_MAP_JSON if echo "$USER_MAP_JSON" | jq -e --arg K "$assignee_login" '.[$K]' > /dev/null; then + echo "Assignee '$assignee_login' FOUND in USER_MAP_JSON." TEMP_PING_KEYS+=("$assignee_login") MAPPED_ASSIGNEE_COUNT=$((MAPPED_ASSIGNEE_COUNT + 1)) + else + echo "Assignee '$assignee_login' NOT FOUND in USER_MAP_JSON." fi done + echo "Total assignees found in USER_MAP_JSON and added to pings: $MAPPED_ASSIGNEE_COUNT" + if [ ${#TEMP_PING_KEYS[@]} -gt 0 ]; then PING_KEYS_OUTPUT=$(IFS=,; echo "${TEMP_PING_KEYS[*]}") + echo "Initial PING_KEYS_OUTPUT from assignees: [$PING_KEYS_OUTPUT]" + else + echo "No assignees found or GH_ASSIGNEES_JSON was empty, or no assignees were found in USER_MAP_JSON." fi + elif [ -n "$USER_MAP_JSON" ] && [ "$USER_MAP_JSON" != "{}" ] && ! command -v jq &> /dev/null; then + echo "::warning::jq is not available. Cannot determine GitHub users (assignees) for pings." + else + echo "No user map JSON or jq not found. PING_KEYS_OUTPUT (from assignees) will be empty." fi + # Add latest committer if INPUT_PING_LATEST_COMMITTER is true if [[ "$INPUT_PING_LATEST_COMMITTER" == "true" ]]; then + echo "INPUT_PING_LATEST_COMMITTER is true. Attempting to fetch latest committer for PR #${PR_NUMBER}." if command -v gh &> /dev/null && [ -n "$PR_NUMBER" ]; then LATEST_COMMITTER_LOGIN_RAW=$(gh pr view "$PR_NUMBER" --json commits --jq '.commits[-1].author.login' 2>/dev/null || echo "") if [ -n "$LATEST_COMMITTER_LOGIN_RAW" ] && [ "$LATEST_COMMITTER_LOGIN_RAW" != "null" ]; then + # Apply bot filter (e.g., names ending in [bot] or -bot) LATEST_COMMITTER_LOGIN=$(echo "$LATEST_COMMITTER_LOGIN_RAW" | grep -v -E -i '(\[bot\]$|-bot$)' || echo "") if [ -n "$LATEST_COMMITTER_LOGIN" ]; then + echo "Latest committer identified: $LATEST_COMMITTER_LOGIN" + + # Check if this committer is already in PING_KEYS_OUTPUT ALREADY_IN_LIST=0 - if [ -n "$PING_KEYS_OUTPUT" ]; then + if [ -n "$PING_KEYS_OUTPUT" ]; then # Only check if PING_KEYS_OUTPUT is not empty IFS=',' read -ra PING_ARRAY <<< "$PING_KEYS_OUTPUT" for key in "${PING_ARRAY[@]}"; do if [[ "$key" == "$LATEST_COMMITTER_LOGIN" ]]; then @@ -836,22 +1280,48 @@ jobs: else PING_KEYS_OUTPUT="$PING_KEYS_OUTPUT,$LATEST_COMMITTER_LOGIN" fi + echo "Added latest committer '$LATEST_COMMITTER_LOGIN' to PING_KEYS_OUTPUT. New list: [$PING_KEYS_OUTPUT]" + else + echo "Latest committer '$LATEST_COMMITTER_LOGIN' is already in PING_KEYS_OUTPUT (likely an assignee)." fi + else + echo "Latest committer login '$LATEST_COMMITTER_LOGIN_RAW' was filtered out (likely a bot or pattern match) or empty after filter." fi + else + echo "No latest committer login found for PR #$PR_NUMBER from gh command, or login was null." + fi + else + if ! command -v gh &> /dev/null; then + echo "::warning::gh command not available. Cannot fetch latest committer." + fi + if [ -z "$PR_NUMBER" ]; then + echo "::warning::PR_NUMBER is not set (event might not be a pull_request). Cannot fetch latest committer." fi fi fi + # Restore stdout/stderr for GitHub Actions to show final summary + exec 1>&3 2>&4 + + # Make this a standard echo for better visibility of the final list + echo "Final Ping Keys Output (GitHub Logins from test-pytest.yml): [$PING_KEYS_OUTPUT]" echo "ping_user_ids_out=$PING_KEYS_OUTPUT" >> $GITHUB_OUTPUT + # Store branch names in variables with proper quoting PR_BRANCH="${PR_BRANCH_NAME:-unknown}" TARGET_BRANCH="${TARGET_BRANCH_NAME:-unknown}" + # 2. Construct Message Body MESSAGE_LINES+=("**Pytest Comparison & Regression Analysis for PR [#${PR_NUMBER}: ${PR_TITLE}](${PR_URL})**") MESSAGE_LINES+=("Branch: [\`${PR_BRANCH}\`](${REPO_URL}/tree/${PR_BRANCH}) against [\`${TARGET_BRANCH}\`](${REPO_URL}/tree/${TARGET_BRANCH})") MESSAGE_LINES+=("---") + # Job Status Summary MESSAGE_LINES+=("**Job Status:**") + LINT_STATUS="Success" + if [[ "$LINT_RESULT" == "failure" ]]; then LINT_STATUS="Failed"; elif [[ "$LINT_RESULT" == "skipped" ]]; then LINT_STATUS="Skipped"; fi + MESSAGE_LINES+=("- Linting: $LINT_STATUS") + SOURCE_TEST_STATUS="Success" if [[ "$SOURCE_TEST_RESULT" == "failure" ]]; then SOURCE_TEST_STATUS="Failed"; elif [[ "$SOURCE_TEST_RESULT" == "skipped" ]]; then SOURCE_TEST_STATUS="Skipped"; fi MESSAGE_LINES+=("- PR Branch Tests (\`${PR_BRANCH}\`): $SOURCE_TEST_STATUS") @@ -865,24 +1335,32 @@ jobs: MESSAGE_LINES+=("- Comparison & Regression: $COMPARE_STATUS") MESSAGE_LINES+=("---") + # Test Discovery Issues in PR Branch if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: ERROR: Test Discovery Failed in PR Branch (\`${PR_BRANCH}\`)**") MESSAGE_LINES+=(" - Type: \`${PR_ERROR_TYPE}\`") MESSAGE_LINES+=(" - Details: \`\`\`${PR_ERROR_DETAILS_TRUNCATED}\`\`\`") + MESSAGE_LINES+=(" - This usually indicates import errors or syntax issues preventing tests from being collected.") elif [[ "$PR_NO_TESTS_FOUND" == "true" ]]; then MESSAGE_LINES+=("**:warning: WARNING: No Tests Found in PR Branch (\`${PR_BRANCH}\`)**") + MESSAGE_LINES+=(" - Pytest did not discover any test files matching its patterns.") + MESSAGE_LINES+=(" - Ensure your test files are correctly named (e.g., \`test_*.py\` or \`*_test.py\`) and located.") fi + # Regression Analysis Summary if [[ "$HAS_REGRESSIONS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: REGRESSIONS DETECTED**") MESSAGE_LINES+=(" - **${REGRESSION_COUNT} test(s)** that were passing in \`${TARGET_BRANCH}\` are now **failing** in \`${PR_BRANCH}\`.") + # Calculate current message length CURRENT_MESSAGE=$(printf "%s\\n" "${MESSAGE_LINES[@]}") CURRENT_LENGTH=${#CURRENT_MESSAGE} if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then + # Read regression details REGRESSION_LIST=$(awk '/^[0-9]+\./ {sub(/^[0-9]+\. /, "- "); print}' "$REGRESSION_FILE_PATH") + # Calculate length with regression details TEMP_MESSAGE="$CURRENT_MESSAGE" TEMP_MESSAGE+="\`\`\`" TEMP_MESSAGE+="$REGRESSION_LIST" @@ -890,20 +1368,25 @@ jobs: TEMP_LENGTH=${#TEMP_MESSAGE} if [ $TEMP_LENGTH -le 2000 ]; then + # If total length would be under 2000 chars, include in message MESSAGE_LINES+=(" - **Failed Tests (Regressions):**") MESSAGE_LINES+=("\`\`\`") MESSAGE_LINES+=("$REGRESSION_LIST") MESSAGE_LINES+=("\`\`\`") - ARTIFACT_PATH_OUTPUT="" + ARTIFACT_PATH_OUTPUT="" # No artifact if details are inline else + # If would exceed 2000 chars, attach file instead MESSAGE_LINES+=(" - Details for the ${REGRESSION_COUNT} regressions are in the attached \`regression_details.txt\` file.") ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" fi else + MESSAGE_LINES+=(" (Regression details file not found or download failed; cannot list specific regressions here.)") ARTIFACT_PATH_OUTPUT="" fi elif [[ "$COMPARE_RESULT" == "failure" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then + # This case handles general comparison failures NOT due to specific regressions MESSAGE_LINES+=("**:warning: TEST RESULTS DECLINED**") + MESSAGE_LINES+=(" - The PR branch shows a decrease in test success compared to the target branch, but no specific regressions were identified by the \`meta-regression-analysis\` job.") MESSAGE_LINES+=(" - PR Branch (\`${PR_BRANCH}\`): **${PR_PASSED_TESTS}/${PR_TOTAL_TESTS} passed (${PR_PERCENTAGE}%)**") MESSAGE_LINES+=(" - Target Branch (\`${TARGET_BRANCH}\`): **${TARGET_PASSED_TESTS}/${TARGET_TOTAL_TESTS} passed (${TARGET_PERCENTAGE}%)**") elif [[ "$COMPARE_RESULT" == "success" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then @@ -915,27 +1398,33 @@ jobs: MESSAGE_LINES+=("---") MESSAGE_LINES+=("[View Workflow Run](${ACTION_RUN_URL})") + # Construct with actual newlines FINAL_MESSAGE_BODY=$(printf "%s\\n" "${MESSAGE_LINES[@]}") if [ ${#MESSAGE_LINES[@]} -gt 0 ]; then + # Remove the very last actual newline FINAL_MESSAGE_BODY="${FINAL_MESSAGE_BODY%\\n}" fi + echo "Final message body prepared in test-pytest.yml" + echo "message_body_out<> $GITHUB_OUTPUT echo "$FINAL_MESSAGE_BODY" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT echo "artifact_path_out=$ARTIFACT_PATH_OUTPUT" >> $GITHUB_OUTPUT + - name: Upload notification construction debug logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: notification_debug_logs_${{ github.event.pull_request.number || github.run_id }} + path: debug_notification_construction.log + retention-days: 3 + if-no-files-found: ignore + notify-discord: name: Send Discord Notification - needs: - [ - test-source-branch, - test-target-branch, - compare-results, - perform-regression-analysis, - prepare-notification, - ] + needs: [prepare-notification] if: | always() && needs.prepare-notification.outputs.should_notify == 'true' && From 4ce4de3b304b58f0697e3d22001935f757e92b6d Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 10:50:03 -0600 Subject: [PATCH 11/44] Update regression script; --- .github/workflows/test-py-pytest.yml | 71 ++++++++++++++++------------ 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index aa79b65..f44736e 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -98,16 +98,9 @@ jobs: - name: Check for test collection errors id: check-collection run: | - # Create verbose debug file for artifact - exec 3>&1 4>&2 - exec 1> >(tee -a debug_collection.log) 2>&1 - echo "Running pytest collection check..." python -m pytest --collect-only -v > collection_output.txt 2>&1 || true - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - # Set default values HAS_COLLECTION_ERRORS="false" NO_TESTS_FOUND="false" @@ -175,49 +168,37 @@ jobs: echo "has_errors=false" >> $GITHUB_OUTPUT fi - # Save full collection output to debug file for artifact - echo "=== FULL COLLECTION OUTPUT ===" >> debug_collection.log - cat collection_output.txt >> debug_collection.log + # Clean summary output + if [[ "$HAS_COLLECTION_ERRORS" == "true" ]]; then + echo "❌ Discovery Error: $ERROR_TYPE" + elif [[ "$NO_TESTS_FOUND" == "true" ]]; then + echo "⚠️ No Tests Found" + else + echo "✅ Discovery Success" + fi - name: Run tests on PR Branch if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - # Create verbose debug file for artifact - exec 3>&1 4>&2 - exec 1> >(tee -a debug_test_run.log) 2>&1 - echo "Running tests on PR branch..." - python -m pytest -vv --json-report --json-report-file=pr_results.json || true - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 + python -m pytest -vv --json-report --json-report-file=pr_results.json --tb=short > test_output.txt 2>&1 || true if [ -f pr_results.json ]; then - echo "Test results file successfully created for PR branch" - # Save first 200 chars to debug file only - echo "=== JSON RESULTS PREVIEW ===" >> debug_test_run.log - head -c 200 pr_results.json >> debug_test_run.log + echo "✅ Test execution completed" else - echo "::error::Failed to create test results file for PR branch" + echo "❌ Test execution failed" fi - name: Extract test results and create artifacts id: extract-results run: | echo "PR_BRANCH=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV - echo "Processing test results for PR branch: $PR_BRANCH" - - # Create debug file for detailed output - exec 3>&1 4>&2 - exec 1> >(tee -a debug_extract_results.log) 2>&1 python -c " import json import sys import os - print('Starting test results extraction script for PR branch') - # Default values in case file doesn't exist or is invalid pr_total = 0 pr_passed = 0 @@ -328,6 +309,33 @@ jobs: f.write(f'skipped_count={len(skipped_tests)}\\n') f.write(f'xfailed_count={len(xfailed_tests)}\\n') + # Extract warnings from test output + warnings_list = [] + try: + with open('test_output.txt', 'r') as f: + content = f.read() + # Extract warnings section + if '============================== warnings summary ===============================' in content: + warnings_section = content.split('============================== warnings summary ===============================')[1] + if '-- Docs:' in warnings_section: + warnings_section = warnings_section.split('-- Docs:')[0] + # Parse individual warnings + current_warning = '' + for line in warnings_section.split('\\n'): + line = line.strip() + if line and not line.startswith('='): + if line.endswith('.py') or line.endswith('.py:'): + if current_warning: + warnings_list.append(current_warning.strip()) + current_warning = line + else: + current_warning += ' ' + line + if current_warning: + warnings_list.append(current_warning.strip()) + print(f'Extracted {len(warnings_list)} warnings from test output') + except Exception as e: + print(f'Could not extract warnings: {e}') + # Save test lists to artifact files instead of job outputs test_data = { 'failing_tests': failing_tests, @@ -335,7 +343,8 @@ jobs: 'xfailed_tests': xfailed_tests, 'all_tests': all_tests, 'skipped_tests_with_reasons': skipped_tests_with_reasons, - 'xfailed_tests_with_reasons': xfailed_tests_with_reasons + 'xfailed_tests_with_reasons': xfailed_tests_with_reasons, + 'warnings': warnings_list } with open('pr_test_data.json', 'w') as f: From 97bb781f03609ca86c13fc7c9792b716893df9f7 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 11:45:48 -0600 Subject: [PATCH 12/44] Update regression script; --- .github/workflows/test-py-pytest.yml | 355 ++++++++++++++------------- 1 file changed, 181 insertions(+), 174 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index f44736e..452d840 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -319,20 +319,32 @@ jobs: warnings_section = content.split('============================== warnings summary ===============================')[1] if '-- Docs:' in warnings_section: warnings_section = warnings_section.split('-- Docs:')[0] - # Parse individual warnings - current_warning = '' - for line in warnings_section.split('\\n'): - line = line.strip() - if line and not line.startswith('='): - if line.endswith('.py') or line.endswith('.py:'): - if current_warning: - warnings_list.append(current_warning.strip()) - current_warning = line - else: - current_warning += ' ' + line - if current_warning: - warnings_list.append(current_warning.strip()) - print(f'Extracted {len(warnings_list)} warnings from test output') + + # Parse warnings - format is file path followed by indented warning details + lines = warnings_section.split('\\n') + current_warning_group = [] + + for line in lines: + line = line.rstrip() + if not line or line.startswith('='): + continue + + # Check if this is a file path (starts at column 0, ends with .py: or contains warning count) + if not line.startswith(' ') and ('.py:' in line or 'warnings' in line): + # Save previous warning group if exists + if current_warning_group: + warnings_list.append('\\n'.join(current_warning_group)) + # Start new warning group + current_warning_group = [line] + elif line.startswith(' ') and current_warning_group: + # This is part of the current warning (indented line) + current_warning_group.append(line) + + # Don't forget the last warning group + if current_warning_group: + warnings_list.append('\\n'.join(current_warning_group)) + + print(f'Extracted {len(warnings_list)} warning groups from test output') except Exception as e: print(f'Could not extract warnings: {e}') @@ -354,10 +366,7 @@ jobs: print('Results extraction completed') " - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "PR branch test results processed: ${{ steps.extract-results.outputs.passed }}/${{ steps.extract-results.outputs.total }} tests passed (${{ steps.extract-results.outputs.percentage }}%)" + echo "✅ Test results: ${{ steps.extract-results.outputs.passed }}/${{ steps.extract-results.outputs.total }} passed (${{ steps.extract-results.outputs.percentage }}%)" - name: Upload PR branch artifacts if: always() @@ -493,23 +502,13 @@ jobs: - name: Run tests on target branch if: steps.check-collection.outputs.has_collection_errors != 'true' run: | - # Create verbose debug file for artifact - exec 3>&1 4>&2 - exec 1> >(tee -a debug_target_test_run.log) 2>&1 - echo "Running tests on target branch..." - python -m pytest -vv --json-report --json-report-file=target_results.json || true - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 + python -m pytest -vv --json-report --json-report-file=target_results.json --tb=short > target_test_output.txt 2>&1 || true if [ -f target_results.json ]; then - echo "Test results file successfully created for target branch" - # Save first 200 chars to debug file only - echo "=== JSON RESULTS PREVIEW ===" >> debug_target_test_run.log - head -c 200 target_results.json >> debug_target_test_run.log + echo "✅ Test execution completed" else - echo "::warning::Failed to create test results file for target branch" + echo "❌ Test execution failed" fi - name: Extract test results and create artifacts @@ -603,10 +602,50 @@ jobs: f.write(f'percentage={target_percentage:.2f}\\n') f.write(f'passing_count={len(passing_tests)}\\n') + # Extract warnings from test output + warnings_list = [] + try: + with open('target_test_output.txt', 'r') as f: + content = f.read() + # Extract warnings section + if '============================== warnings summary ===============================' in content: + warnings_section = content.split('============================== warnings summary ===============================')[1] + if '-- Docs:' in warnings_section: + warnings_section = warnings_section.split('-- Docs:')[0] + + # Parse warnings - format is file path followed by indented warning details + lines = warnings_section.split('\\n') + current_warning_group = [] + + for line in lines: + line = line.rstrip() + if not line or line.startswith('='): + continue + + # Check if this is a file path (starts at column 0, ends with .py: or contains warning count) + if not line.startswith(' ') and ('.py:' in line or 'warnings' in line): + # Save previous warning group if exists + if current_warning_group: + warnings_list.append('\\n'.join(current_warning_group)) + # Start new warning group + current_warning_group = [line] + elif line.startswith(' ') and current_warning_group: + # This is part of the current warning (indented line) + current_warning_group.append(line) + + # Don't forget the last warning group + if current_warning_group: + warnings_list.append('\\n'.join(current_warning_group)) + + print(f'Extracted {len(warnings_list)} warning groups from target test output') + except Exception as e: + print(f'Could not extract warnings: {e}') + # Save test lists to artifact files instead of job outputs test_data = { 'passing_tests': passing_tests, - 'all_tests': all_tests + 'all_tests': all_tests, + 'warnings': warnings_list } with open('target_test_data.json', 'w') as f: @@ -668,55 +707,33 @@ jobs: - name: Check for collection errors run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_comparison_analysis.log) 2>&1 - - echo "Retrieving collection error status information" PR_COLLECTION_ERRORS="${{ needs.test-source-branch.outputs.collection_errors }}" PR_NO_TESTS="${{ needs.test-source-branch.outputs.no_tests_found }}" PR_ERROR_TYPE="${{ needs.test-source-branch.outputs.error_type }}" - PR_ERROR_DETAILS="${{ needs.test-source-branch.outputs.error_details }}" TARGET_COLLECTION_ERRORS="${{ needs.test-target-branch.outputs.collection_errors }}" - echo "PR branch collection errors: $PR_COLLECTION_ERRORS" - echo "PR branch no tests found: $PR_NO_TESTS" - echo "PR branch error type: $PR_ERROR_TYPE" - echo "Target branch collection errors: $TARGET_COLLECTION_ERRORS" - - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - # Distinct error handling for PR branch + # Clean discovery status reporting if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then - echo "::error::Test discovery errors in PR branch: $PR_ERROR_TYPE" - echo "::error::$PR_ERROR_DETAILS" - echo "❌ PR branch has test discovery errors. Python modules could not be imported correctly." + echo "❌ Discovery Error (PR): $PR_ERROR_TYPE" exit 1 - fi - - if [[ "$PR_NO_TESTS" == "true" ]]; then - echo "::error::No tests were found in the PR branch" - echo "❌ PR branch has no tests detected. Please add test files that match pytest's discovery pattern." + elif [[ "$PR_NO_TESTS" == "true" ]]; then + echo "❌ No Tests Found (PR)" exit 1 + else + echo "✅ Discovery Success (PR)" fi - # Warning for target branch issues (not a failure) if [[ "$TARGET_COLLECTION_ERRORS" == "true" ]]; then - echo "⚠️ Target branch has test discovery errors. Tests will still be compared but results may not be accurate." - fi - - if [[ "${{ needs.test-target-branch.outputs.no_tests_found }}" == "true" ]]; then - echo "⚠️ Target branch has no tests detected. PR branch tests will still be evaluated." + echo "⚠️ Discovery Error (Target)" + elif [[ "${{ needs.test-target-branch.outputs.no_tests_found }}" == "true" ]]; then + echo "⚠️ No Tests Found (Target)" + else + echo "✅ Discovery Success (Target)" fi - name: Run regression analysis from artifacts run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_regression_analysis.log) 2>&1 - - echo "Running regression analysis from artifact data..." + echo "Running regression analysis..." python3 - << 'EOF' import json @@ -777,10 +794,7 @@ jobs: print(traceback.format_exc()) EOF - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Regression analysis completed" + echo "✅ Regression analysis completed" - name: Check for regression details file id: check-regressions @@ -832,11 +846,7 @@ jobs: - name: Check for skip/xfail regressions from artifacts run: | - # Create analysis debug file - exec 3>&1 4>&2 - exec 1> >(tee -a debug_skip_xfail_analysis.log) 2>&1 - - echo "Checking for tests that were passing in target but are now skipped or xfailed in PR" + echo "Checking for skip/xfail status regressions..." python3 - << 'EOF' import json @@ -863,6 +873,7 @@ jobs: target_passing = target_data.get('passing_tests', []) pr_skipped = pr_data.get('skipped_tests', []) pr_xfailed = pr_data.get('xfailed_tests', []) + pr_failing = pr_data.get('failing_tests', []) pr_skipped_with_reasons = pr_data.get('skipped_tests_with_reasons', {}) pr_xfailed_with_reasons = pr_data.get('xfailed_tests_with_reasons', {}) @@ -877,46 +888,93 @@ jobs: pr_skipped_set = set(pr_skipped) pr_xfailed_set = set(pr_xfailed) + # Check different types of regressions passing_to_skipped = list(target_passing_set.intersection(pr_skipped_set)) passing_to_xfailed = list(target_passing_set.intersection(pr_xfailed_set)) + # TODO: Add failing→skipped and xfailed→skipped when we have target failing/xfailed data + total_skip_xfail_regressions = len(passing_to_skipped) + len(passing_to_xfailed) if total_skip_xfail_regressions > 0: - print(f"Found {total_skip_xfail_regressions} tests that were passing in target but are now skipped/xfailed in PR!") - - # Build comprehensive warning message - warning_parts = [f"Skip/XFail Analysis: {total_skip_xfail_regressions} test(s) that were passing in target branch are now being skipped or xfailed in PR branch."] + print(f"⚠️ Skip/XFail Regressions: {total_skip_xfail_regressions} test(s)") if passing_to_skipped: - warning_parts.append(f"Tests now SKIPPED ({len(passing_to_skipped)}):") - for idx, test in enumerate(sorted(passing_to_skipped), 1): - reason = pr_skipped_with_reasons.get(test, 'No reason provided') - warning_parts.append(f" {idx}. {test} - Reason: {reason}") + print(f"::warning::Passing→Skipped Regressions: {len(passing_to_skipped)} test(s) that were passing in target are now skipped in PR") if passing_to_xfailed: - warning_parts.append(f"Tests now XFAILED ({len(passing_to_xfailed)}):") - for idx, test in enumerate(sorted(passing_to_xfailed), 1): - reason = pr_xfailed_with_reasons.get(test, 'No reason provided') - warning_parts.append(f" {idx}. {test} - Reason: {reason}") - - warning_parts.append("While these changes don't fail the workflow, they indicate tests that were working before are now being bypassed. Please review these tests to ensure this is intentional.") - - # Print as single warning annotation - combined_warning = " ".join(warning_parts) - print(f"::warning::{combined_warning}") + print(f"::warning::Passing→XFailed Regressions: {len(passing_to_xfailed)} test(s) that were passing in target are now xfailed in PR") else: - print("No skip/xfail regressions found - all previously passing tests are still running.") + print("✅ No skip/xfail regressions detected") except Exception as e: print(f"Error in skip/xfail analysis: {e}") import traceback print(traceback.format_exc()) EOF - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 + echo "✅ Skip/xfail analysis completed" + + - name: Check for warning regressions from artifacts + run: | + echo "Checking for warning regressions between target and PR branches" + + python3 - << 'EOF' + import json + import os + import glob + + try: + # Load test data from artifacts + target_data = {} + pr_data = {} + + # Load target and PR data + target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') + if target_files: + with open(target_files[0], 'r') as f: + target_data = json.load(f) + + pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') + if pr_files: + with open(pr_files[0], 'r') as f: + pr_data = json.load(f) + + # Extract warnings + target_warnings = target_data.get('warnings', []) + pr_warnings = pr_data.get('warnings', []) + + print(f"Target branch has {len(target_warnings)} warning groups") + print(f"PR branch has {len(pr_warnings)} warning groups") + + # Check for new warnings (warnings in PR that aren't in target) + target_warning_set = set(target_warnings) + pr_warning_set = set(pr_warnings) + + new_warnings = list(pr_warning_set - target_warning_set) + resolved_warnings = list(target_warning_set - pr_warning_set) + + if new_warnings: + print(f"::warning::Warning Regressions: {len(new_warnings)} new warning group(s) detected in PR branch that weren't present in target branch.") + for i, warning in enumerate(new_warnings[:5], 1): # Limit to first 5 for readability + print(f"::warning::New Warning {i}: {warning[:200]}...") + if len(new_warnings) > 5: + print(f"::warning::... and {len(new_warnings) - 5} more new warnings") + else: + print("✅ No new warnings detected") + + if resolved_warnings: + print(f"::notice::Warning Improvements: {len(resolved_warnings)} warning group(s) from target branch have been resolved in PR branch.") + + if not new_warnings and not resolved_warnings: + print("✅ Warning status unchanged between branches") + + except Exception as e: + print(f"Error in warning regression analysis: {e}") + import traceback + print(traceback.format_exc()) + EOF - echo "Skip/xfail regression analysis completed" + echo "Warning regression analysis completed" - name: Check for test additions and removals from artifacts run: | @@ -1014,9 +1072,8 @@ jobs: - name: Compare test results run: | - echo "Test Results Summary:" - echo "Target branch (${{ inputs.target_branch_to_compare }}): ${{ needs.test-target-branch.outputs.passed }}/${{ needs.test-target-branch.outputs.total }} tests passed (${{ needs.test-target-branch.outputs.percentage }}%)" - echo "PR branch: ${{ needs.test-source-branch.outputs.passed }}/${{ needs.test-source-branch.outputs.total }} tests passed (${{ needs.test-source-branch.outputs.percentage }}%)" + echo "Target: ${{ needs.test-target-branch.outputs.passed }}/${{ needs.test-target-branch.outputs.total }} passed (${{ needs.test-target-branch.outputs.percentage }}%)" + echo "PR: ${{ needs.test-source-branch.outputs.passed }}/${{ needs.test-source-branch.outputs.total }} passed (${{ needs.test-source-branch.outputs.percentage }}%)" if [[ "${{ needs.test-source-branch.outputs.total }}" == "0" ]]; then echo "::error::No tests were found in the PR branch" @@ -1043,65 +1100,19 @@ jobs: fi fi - # Fail if any tests passed in target branch but now fail in PR branch + # Check for regressions if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then - echo "❌ PR branch has test regressions from target branch" REGRESSION_COUNT_VAL=${{ needs.perform-regression-analysis.outputs.regression_count }} - echo " - $REGRESSION_COUNT_VAL tests that were passing in target branch are now failing" - - echo "### :x: Test Regressions Detected!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**$REGRESSION_COUNT_VAL test(s) that were passing in the target branch are now failing in the PR branch.**" >> $GITHUB_STEP_SUMMARY - echo "This job (\`compare-results\`) has been marked as failed due to these regressions." >> $GITHUB_STEP_SUMMARY - - if [ -f regression_details.txt ]; then # Check if file exists (it should from previous step) - echo "Failing tests (regressions) list:" - cat regression_details.txt - else - echo "::warning::Regression details file (regression_details.txt) not found in Compare test results step. It might be available as an artifact from the 'perform-regression-analysis' job." - fi + echo "❌ Regressions Found: $REGRESSION_COUNT_VAL test(s)" exit 1 fi - # Continue with the original comparison if no regressions + # Compare overall results if (( $(echo "$PR_PASSED >= $TARGET_PASSED" | bc -l) )) && (( $(echo "$PR_PERCENTAGE >= $TARGET_PERCENTAGE" | bc -l) )); then - echo "✅ PR branch has equal or better test results than target branch" - - # Additional verbose information about improvement - if (( $(echo "$PR_PASSED > $TARGET_PASSED" | bc -l) )); then - IMPROVEMENT=$(( $PR_PASSED - $TARGET_PASSED )) - echo " - Improvement: $IMPROVEMENT more passing tests than target branch" - fi - - if (( $(echo "$PR_PERCENTAGE > $TARGET_PERCENTAGE" | bc -l) )); then - PERCENTAGE_IMPROVEMENT=$(echo "$PR_PERCENTAGE - $TARGET_PERCENTAGE" | bc -l) - echo " - Percentage improvement: +${PERCENTAGE_IMPROVEMENT}% compared to target branch" - fi - + echo "✅ Results: Equal or better than target" exit 0 else - echo "❌ PR branch has worse test results than target branch" - echo " - Passed tests: $PR_PASSED vs $TARGET_PASSED on target branch" - echo " - Pass percentage: $PR_PERCENTAGE% vs $TARGET_PERCENTAGE% on target branch" - - # Add to job summary for general comparison failure - echo "### :x: Test Comparison Failed" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The PR branch has worse test results than the target branch:" >> $GITHUB_STEP_SUMMARY - echo "- Passed tests: $PR_PASSED (PR) vs $TARGET_PASSED (Target)" >> $GITHUB_STEP_SUMMARY - echo "- Pass percentage: $PR_PERCENTAGE% (PR) vs $TARGET_PERCENTAGE% (Target)" >> $GITHUB_STEP_SUMMARY - - # Calculate regression metrics - if (( $(echo "$PR_PASSED < $TARGET_PASSED" | bc -l) )); then - REGRESSION=$(( $TARGET_PASSED - $PR_PASSED )) - echo " - Regression: $REGRESSION fewer passing tests than target branch" - fi - - if (( $(echo "$PR_PERCENTAGE < $TARGET_PERCENTAGE" | bc -l) )); then - PERCENTAGE_REGRESSION=$(echo "$TARGET_PERCENTAGE - $PR_PERCENTAGE" | bc -l) - echo " - Percentage regression: -${PERCENTAGE_REGRESSION}% compared to target branch" - fi - + echo "❌ Results: Worse than target" exit 1 fi @@ -1171,6 +1182,20 @@ jobs: path: . # Download to current directory continue-on-error: true + - name: Check downloaded regression file + if: needs.perform-regression-analysis.outputs.has_regressions == 'true' + run: | + echo "Checking for regression details file..." + if [ -f "regression_details.txt" ]; then + echo "✅ Regression details file found" + echo "File size: $(wc -c < regression_details.txt) bytes" + echo "First few lines:" + head -5 regression_details.txt + else + echo "❌ Regression details file not found" + ls -la + fi + - name: Construct Discord Notification id: construct_notification env: @@ -1366,30 +1391,12 @@ jobs: CURRENT_LENGTH=${#CURRENT_MESSAGE} if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then - # Read regression details - REGRESSION_LIST=$(awk '/^[0-9]+\./ {sub(/^[0-9]+\. /, "- "); print}' "$REGRESSION_FILE_PATH") - - # Calculate length with regression details - TEMP_MESSAGE="$CURRENT_MESSAGE" - TEMP_MESSAGE+="\`\`\`" - TEMP_MESSAGE+="$REGRESSION_LIST" - TEMP_MESSAGE+="\`\`\`" - TEMP_LENGTH=${#TEMP_MESSAGE} - - if [ $TEMP_LENGTH -le 2000 ]; then - # If total length would be under 2000 chars, include in message - MESSAGE_LINES+=(" - **Failed Tests (Regressions):**") - MESSAGE_LINES+=("\`\`\`") - MESSAGE_LINES+=("$REGRESSION_LIST") - MESSAGE_LINES+=("\`\`\`") - ARTIFACT_PATH_OUTPUT="" # No artifact if details are inline - else - # If would exceed 2000 chars, attach file instead - MESSAGE_LINES+=(" - Details for the ${REGRESSION_COUNT} regressions are in the attached \`regression_details.txt\` file.") - ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" - fi + echo "✅ Regression file found, attaching to Discord message" + MESSAGE_LINES+=(" - Detailed list of ${REGRESSION_COUNT} regressions attached as \`regression_details.txt\`") + ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" else - MESSAGE_LINES+=(" (Regression details file not found or download failed; cannot list specific regressions here.)") + echo "❌ Regression file not found for Discord attachment" + MESSAGE_LINES+=(" - Regression details file not available") ARTIFACT_PATH_OUTPUT="" fi elif [[ "$COMPARE_RESULT" == "failure" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then From e72ad0933fd3b0ef62b2fdcdb29a18d5a2152f32 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 12:17:51 -0600 Subject: [PATCH 13/44] Update regression script; --- .../workflows/meta-regression-analysis.yml | 39 +------- .github/workflows/test-py-pytest.yml | 92 ++++--------------- 2 files changed, 20 insertions(+), 111 deletions(-) diff --git a/.github/workflows/meta-regression-analysis.yml b/.github/workflows/meta-regression-analysis.yml index 9f3f66a..db06124 100644 --- a/.github/workflows/meta-regression-analysis.yml +++ b/.github/workflows/meta-regression-analysis.yml @@ -71,10 +71,6 @@ jobs: ITEM_TYPE_SINGULAR_ENV: "${{ inputs.item_type_singular }}" ITEM_TYPE_PLURAL_ENV: "${{ inputs.item_type_plural }}" run: | - # Create debug file for detailed output - exec 3>&1 4>&2 - exec 1> >(tee -a debug_regression_analysis.log) 2>&1 - echo "Running regression analysis..." python3 - <<'EOF' @@ -159,16 +155,15 @@ jobs: regression_items_list = sorted(list(target_passing_set.intersection(pr_failing_set))) if regression_items_list: - print(f"Found {len(regression_items_list)} regression(s)!") + print(f"❌ Regressions Found: {len(regression_items_list)} {item_type_p}") has_regressions_output = "true" regression_count_output = len(regression_items_list) with open("regression_details.txt", "w") as f: f.write(f"Found {len(regression_items_list)} {item_type_p} that were passing/clean in the target branch but are now failing/have issues in the PR branch:\n\n") for idx, item in enumerate(regression_items_list, 1): f.write(f"{idx}. {item}\n") - print("Regression details written to regression_details.txt") else: - print(f"No new regressions found for {item_type_p}.") + print(f"✅ No regressions detected") with open("regression_details.txt", "w") as f: f.write(f"No new regressions detected for {item_type_p} (items that were passing/clean in target and are now failing/with issues in PR).\n") @@ -190,39 +185,13 @@ jobs: print("::warning::GITHUB_OUTPUT environment variable not set.") EOF - # Restore stdout/stderr for GitHub Actions - exec 1>&3 2>&4 - - echo "Regression analysis script completed." - - # Only show regression summary in logs if regressions found - if [ -f "regression_details.txt" ]; then - REGRESSION_COUNT=$(grep -c "^[0-9]\+\." regression_details.txt || echo "0") - if [ "$REGRESSION_COUNT" -gt 0 ]; then - echo "::error::Found $REGRESSION_COUNT regression(s). Details available in artifact." - else - echo "No regressions detected." - fi - fi - - - name: Upload debug logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: regression_analysis_debug_logs_${{ inputs.pr_number || inputs.run_id }} - path: debug_regression_analysis.log - retention-days: 7 - if-no-files-found: ignore + echo "✅ Analysis completed" - name: Fail job if regressions are found if: steps.check-regressions-script.outputs.HAS_REGRESSIONS == 'true' run: | - echo "Regressions detected. Failing the job as per configuration." REGRESSION_COUNT_VAL=${{ steps.check-regressions-script.outputs.REGRESSION_COUNT }} - echo "### :x: Regressions Found!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY # Newline - echo "**$REGRESSION_COUNT_VAL regression(s) detected.** This job was intentionally failed." >> $GITHUB_STEP_SUMMARY - echo "The 'Upload regression details artifact' step will still attempt to upload details." >> $GITHUB_STEP_SUMMARY + echo "❌ Failing job due to $REGRESSION_COUNT_VAL regression(s)" exit 1 - name: Upload regression details artifact diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index 452d840..f90fa77 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -375,9 +375,7 @@ jobs: name: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | pr_test_data.json - debug_*.log - collection_output.txt - error_details.txt + test_output.txt pr_results.json retention-days: 3 if-no-files-found: ignore @@ -667,9 +665,7 @@ jobs: name: target_branch_data_${{ github.event.pull_request.number || github.run_id }} path: | target_test_data.json - debug_target_*.log - collection_output.txt - error_details.txt + target_test_output.txt target_results.json retention-days: 3 if-no-files-found: ignore @@ -900,10 +896,10 @@ jobs: print(f"⚠️ Skip/XFail Regressions: {total_skip_xfail_regressions} test(s)") if passing_to_skipped: - print(f"::warning::Passing→Skipped Regressions: {len(passing_to_skipped)} test(s) that were passing in target are now skipped in PR") + print(f"⚠️ Passing→Skipped: {len(passing_to_skipped)} test(s)") if passing_to_xfailed: - print(f"::warning::Passing→XFailed Regressions: {len(passing_to_xfailed)} test(s) that were passing in target are now xfailed in PR") + print(f"⚠️ Passing→XFailed: {len(passing_to_xfailed)} test(s)") else: print("✅ No skip/xfail regressions detected") except Exception as e: @@ -954,19 +950,11 @@ jobs: resolved_warnings = list(target_warning_set - pr_warning_set) if new_warnings: - print(f"::warning::Warning Regressions: {len(new_warnings)} new warning group(s) detected in PR branch that weren't present in target branch.") - for i, warning in enumerate(new_warnings[:5], 1): # Limit to first 5 for readability - print(f"::warning::New Warning {i}: {warning[:200]}...") - if len(new_warnings) > 5: - print(f"::warning::... and {len(new_warnings) - 5} more new warnings") + print(f"⚠️ Warning Regressions: {len(new_warnings)} new warning(s) detected") + elif resolved_warnings: + print(f"✅ Warning Improvements: {len(resolved_warnings)} warning(s) resolved") else: - print("✅ No new warnings detected") - - if resolved_warnings: - print(f"::notice::Warning Improvements: {len(resolved_warnings)} warning group(s) from target branch have been resolved in PR branch.") - - if not new_warnings and not resolved_warnings: - print("✅ Warning status unchanged between branches") + print("✅ No warning changes detected") except Exception as e: print(f"Error in warning regression analysis: {e}") @@ -1019,45 +1007,15 @@ jobs: removed_tests = list(target_all_set - pr_all_set) # In target but not in PR added_tests = list(pr_all_set - target_all_set) # In PR but not in target - # Report removed tests (warnings) - if removed_tests: - print(f"Found {len(removed_tests)} tests that were removed from target branch!") - - # Build comprehensive removal warning - removal_parts = [f"Test Removal Analysis: {len(removed_tests)} test(s) that existed in target branch are missing from PR branch."] - removal_parts.append(f"REMOVED Tests ({len(removed_tests)}):") - for idx, test in enumerate(sorted(removed_tests), 1): - removal_parts.append(f" {idx}. {test}") - removal_parts.append("These test removals should be reviewed to ensure they are intentional. If tests were renamed or moved, this may show as removal + addition.") - - # Print as single warning annotation - combined_removal_warning = " ".join(removal_parts) - print(f"::warning::{combined_removal_warning}") + # Report test changes + if removed_tests and added_tests: + print(f"📊 Test Changes: +{len(added_tests)} added, -{len(removed_tests)} removed") + elif added_tests: + print(f"✅ Test Additions: {len(added_tests)} new test(s) added") + elif removed_tests: + print(f"⚠️ Test Removals: {len(removed_tests)} test(s) removed") else: - print("No test removals detected.") - - # Report added tests (notifications/info) - if added_tests: - print(f"Found {len(added_tests)} new tests added in PR branch!") - - # Build comprehensive addition notice - addition_parts = [f"Test Addition Analysis: {len(added_tests)} new test(s) have been added in the PR branch."] - addition_parts.append(f"NEW Tests ({len(added_tests)}):") - for idx, test in enumerate(sorted(added_tests), 1): - addition_parts.append(f" {idx}. {test}") - addition_parts.append("New tests detected - this indicates expanded test coverage!") - - # Print as single notice annotation - combined_addition_notice = " ".join(addition_parts) - print(f"::notice::{combined_addition_notice}") - else: - print("No new tests detected in PR branch.") - - # Summary - if not removed_tests and not added_tests: - print("Test suite composition is unchanged between target and PR branches.") - else: - print(f"Test suite changes: {len(added_tests)} added, {len(removed_tests)} removed") + print("✅ Test suite unchanged") except Exception as e: print(f"Error in test addition/removal analysis: {e}") @@ -1116,15 +1074,6 @@ jobs: exit 1 fi - - name: Upload comparison analysis logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: comparison_analysis_logs_${{ github.event.pull_request.number || github.run_id }} - path: debug_*_analysis.log - retention-days: 3 - if-no-files-found: ignore - perform-regression-analysis: needs: [test-source-branch, test-target-branch] uses: ./.github/workflows/meta-regression-analysis.yml @@ -1429,15 +1378,6 @@ jobs: echo "artifact_path_out=$ARTIFACT_PATH_OUTPUT" >> $GITHUB_OUTPUT - - name: Upload notification construction debug logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: notification_debug_logs_${{ github.event.pull_request.number || github.run_id }} - path: debug_notification_construction.log - retention-days: 3 - if-no-files-found: ignore - notify-discord: name: Send Discord Notification needs: [prepare-notification] From 1234323c8f77a46e91898f54ad3a1f8ac3485e18 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 12:40:43 -0600 Subject: [PATCH 14/44] Update regression script; --- .../workflows/meta-regression-analysis.yml | 25 +++++++-- .github/workflows/test-py-pytest.yml | 56 +++++++++++++++++-- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/.github/workflows/meta-regression-analysis.yml b/.github/workflows/meta-regression-analysis.yml index db06124..06037e5 100644 --- a/.github/workflows/meta-regression-analysis.yml +++ b/.github/workflows/meta-regression-analysis.yml @@ -155,15 +155,18 @@ jobs: regression_items_list = sorted(list(target_passing_set.intersection(pr_failing_set))) if regression_items_list: - print(f"❌ Regressions Found: {len(regression_items_list)} {item_type_p}") has_regressions_output = "true" regression_count_output = len(regression_items_list) + print(f"Found {len(regression_items_list)} regression(s)!") + print("::error::Test Regressions Found!") + print(f"::error::{len(regression_items_list)} regression(s) detected. This job was intentionally failed.") with open("regression_details.txt", "w") as f: f.write(f"Found {len(regression_items_list)} {item_type_p} that were passing/clean in the target branch but are now failing/have issues in the PR branch:\n\n") for idx, item in enumerate(regression_items_list, 1): f.write(f"{idx}. {item}\n") + print("Regression details written to regression_details.txt") else: - print(f"✅ No regressions detected") + print(f"No new regressions found for {item_type_p}.") with open("regression_details.txt", "w") as f: f.write(f"No new regressions detected for {item_type_p} (items that were passing/clean in target and are now failing/with issues in PR).\n") @@ -185,13 +188,27 @@ jobs: print("::warning::GITHUB_OUTPUT environment variable not set.") EOF - echo "✅ Analysis completed" + echo "Regression analysis script completed." + + # Show regression summary in logs if regressions found + if [ -f "regression_details.txt" ]; then + REGRESSION_COUNT=$(grep -c "^[0-9]\+\." regression_details.txt || echo "0") + if [ "$REGRESSION_COUNT" -gt 0 ]; then + echo "::error::Found $REGRESSION_COUNT regression(s). Details available in artifact." + else + echo "No regressions detected." + fi + fi - name: Fail job if regressions are found if: steps.check-regressions-script.outputs.HAS_REGRESSIONS == 'true' run: | + echo "Regressions detected. Failing the job as per configuration." REGRESSION_COUNT_VAL=${{ steps.check-regressions-script.outputs.REGRESSION_COUNT }} - echo "❌ Failing job due to $REGRESSION_COUNT_VAL regression(s)" + echo "### :x: Regressions Found!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**$REGRESSION_COUNT_VAL regression(s) detected.** This job was intentionally failed." >> $GITHUB_STEP_SUMMARY + echo "The 'Upload regression details artifact' step will still attempt to upload details." >> $GITHUB_STEP_SUMMARY exit 1 - name: Upload regression details artifact diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index f90fa77..4f675a0 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -1058,19 +1058,65 @@ jobs: fi fi - # Check for regressions + # Fail if any tests passed in target branch but now fail in PR branch if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then + echo "❌ PR branch has test regressions from target branch" REGRESSION_COUNT_VAL=${{ needs.perform-regression-analysis.outputs.regression_count }} - echo "❌ Regressions Found: $REGRESSION_COUNT_VAL test(s)" + echo " - $REGRESSION_COUNT_VAL tests that were passing in target branch are now failing" + + echo "### :x: Test Regressions Detected!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**$REGRESSION_COUNT_VAL test(s) that were passing in the target branch are now failing in the PR branch.**" >> $GITHUB_STEP_SUMMARY + echo "This job (\`compare-results\`) has been marked as failed due to these regressions." >> $GITHUB_STEP_SUMMARY + + if [ -f regression_details.txt ]; then + echo "Failing tests (regressions) list:" + cat regression_details.txt + else + echo "::warning::Regression details file (regression_details.txt) not found in Compare test results step. It might be available as an artifact from the 'perform-regression-analysis' job." + fi exit 1 fi - # Compare overall results + # Continue with the original comparison if no regressions if (( $(echo "$PR_PASSED >= $TARGET_PASSED" | bc -l) )) && (( $(echo "$PR_PERCENTAGE >= $TARGET_PERCENTAGE" | bc -l) )); then - echo "✅ Results: Equal or better than target" + echo "✅ PR branch has equal or better test results than target branch" + + # Additional verbose information about improvement + if (( $(echo "$PR_PASSED > $TARGET_PASSED" | bc -l) )); then + IMPROVEMENT=$(( $PR_PASSED - $TARGET_PASSED )) + echo " - Improvement: $IMPROVEMENT more passing tests than target branch" + fi + + if (( $(echo "$PR_PERCENTAGE > $TARGET_PERCENTAGE" | bc -l) )); then + PERCENTAGE_IMPROVEMENT=$(echo "$PR_PERCENTAGE - $TARGET_PERCENTAGE" | bc -l) + echo " - Percentage improvement: +${PERCENTAGE_IMPROVEMENT}% compared to target branch" + fi + exit 0 else - echo "❌ Results: Worse than target" + echo "❌ PR branch has worse test results than target branch" + echo " - Passed tests: $PR_PASSED vs $TARGET_PASSED on target branch" + echo " - Pass percentage: $PR_PERCENTAGE% vs $TARGET_PERCENTAGE% on target branch" + + # Add to job summary for general comparison failure + echo "### :x: Test Comparison Failed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The PR branch has worse test results than the target branch:" >> $GITHUB_STEP_SUMMARY + echo "- Passed tests: $PR_PASSED (PR) vs $TARGET_PASSED (Target)" >> $GITHUB_STEP_SUMMARY + echo "- Pass percentage: $PR_PERCENTAGE% (PR) vs $TARGET_PERCENTAGE% (Target)" >> $GITHUB_STEP_SUMMARY + + # Calculate regression metrics + if (( $(echo "$PR_PASSED < $TARGET_PASSED" | bc -l) )); then + REGRESSION=$(( $TARGET_PASSED - $PR_PASSED )) + echo " - Regression: $REGRESSION fewer passing tests than target branch" + fi + + if (( $(echo "$PR_PERCENTAGE < $TARGET_PERCENTAGE" | bc -l) )); then + PERCENTAGE_REGRESSION=$(echo "$TARGET_PERCENTAGE - $PR_PERCENTAGE" | bc -l) + echo " - Percentage regression: -${PERCENTAGE_REGRESSION}% compared to target branch" + fi + exit 1 fi From eda5d63cb0aa469e30d8dcf702c27c0bc6bbcc62 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 13:38:21 -0600 Subject: [PATCH 15/44] Update regression script; --- .github/workflows/test-py-pytest.yml | 282 +++++++++++---------------- 1 file changed, 115 insertions(+), 167 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index 4f675a0..efa5e8c 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -554,9 +554,12 @@ jobs: target_passed = summary.get('passed', 0) print(f'Results extracted from summary - Total: {target_total}, Passed: {target_passed}') - # Extract passing tests and all tests + # Extract all test outcomes if 'tests' in target_results: - print('Extracting passing tests and all discovered tests') + print('Extracting all test outcomes from target') + failing_tests = [] + skipped_tests = [] + xfailed_tests = [] for test in target_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') @@ -564,8 +567,17 @@ jobs: all_tests.append(nodeid) # Track all tests regardless of outcome if outcome == 'passed': passing_tests.append(nodeid) + elif outcome in ['failed', 'error']: + failing_tests.append(nodeid) + elif outcome == 'skipped': + skipped_tests.append(nodeid) + elif outcome == 'xfailed': + xfailed_tests.append(nodeid) print(f'Found {len(passing_tests)} passing tests') + print(f'Found {len(failing_tests)} failing tests') + print(f'Found {len(skipped_tests)} skipped tests') + print(f'Found {len(xfailed_tests)} xfailed tests') print(f'Found {len(all_tests)} total discovered tests') else: print('No valid summary structure found') @@ -642,6 +654,9 @@ jobs: # Save test lists to artifact files instead of job outputs test_data = { 'passing_tests': passing_tests, + 'failing_tests': failing_tests if 'failing_tests' in locals() else [], + 'skipped_tests': skipped_tests if 'skipped_tests' in locals() else [], + 'xfailed_tests': xfailed_tests if 'xfailed_tests' in locals() else [], 'all_tests': all_tests, 'warnings': warnings_list } @@ -727,70 +742,114 @@ jobs: echo "✅ Discovery Success (Target)" fi - - name: Run regression analysis from artifacts + - name: Run comprehensive regression analysis run: | - echo "Running regression analysis..." + echo "Running comprehensive regression analysis..." python3 - << 'EOF' import json import os + import glob try: # Load test data from artifacts target_data = {} pr_data = {} - # Try to load target branch data - target_artifact_path = './artifacts/target_branch_data_*/target_test_data.json' - import glob - target_files = glob.glob(target_artifact_path) + # Load target and PR data + target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') if target_files: with open(target_files[0], 'r') as f: target_data = json.load(f) - print(f"Loaded target data from {target_files[0]}") - else: - print("No target branch test data found") - # Try to load PR branch data - pr_artifact_path = './artifacts/pr_branch_data_*/pr_test_data.json' - pr_files = glob.glob(pr_artifact_path) + pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') if pr_files: with open(pr_files[0], 'r') as f: pr_data = json.load(f) - print(f"Loaded PR data from {pr_files[0]}") - else: - print("No PR branch test data found") - # Extract test arrays - target_passing = target_data.get('passing_tests', []) - pr_failing = pr_data.get('failing_tests', []) + # Extract all test categories + target_passing = set(target_data.get('passing_tests', [])) + target_failing = set(target_data.get('failing_tests', [])) + target_skipped = set(target_data.get('skipped_tests', [])) + target_xfailed = set(target_data.get('xfailed_tests', [])) + target_warnings = set(target_data.get('warnings', [])) - print(f"Parsed {len(target_passing)} passing tests from target branch") - print(f"Parsed {len(pr_failing)} failing tests from PR branch") + pr_passing = set(pr_data.get('passing_tests', [])) + pr_failing = set(pr_data.get('failing_tests', [])) + pr_skipped = set(pr_data.get('skipped_tests', [])) + pr_xfailed = set(pr_data.get('xfailed_tests', [])) + pr_warnings = set(pr_data.get('warnings', [])) - # Find regressions using set operations - target_passing_set = set(target_passing) - pr_failing_set = set(pr_failing) - regression_tests = list(target_passing_set.intersection(pr_failing_set)) + # Analyze different regression types + pass_to_fail = list(target_passing.intersection(pr_failing)) + pass_to_skip = list(target_passing.intersection(pr_skipped.union(pr_xfailed))) + fail_to_skip = list(target_failing.intersection(pr_skipped)) + discovery_regressions = list(pr_warnings - target_warnings) - # Write results to file if there are regressions - if regression_tests: - print(f"Found {len(regression_tests)} regression(s)!") + # Create comprehensive regression report + has_any_regressions = bool(pass_to_fail or pass_to_skip or fail_to_skip or discovery_regressions) + + with open("comprehensive_regression_report.txt", "w") as f: + f.write("COMPREHENSIVE REGRESSION ANALYSIS\\n") + f.write("=" * 50 + "\\n\\n") + + if pass_to_fail: + f.write(f"PASS-TO-FAIL REGRESSIONS ({len(pass_to_fail)} tests)\\n") + f.write("Previously passing, now failing:\\n") + for i, test in enumerate(sorted(pass_to_fail), 1): + f.write(f" {i}. {test}\\n") + f.write("\\n") + + if pass_to_skip: + f.write(f"PASS-TO-SKIP REGRESSIONS ({len(pass_to_skip)} tests)\\n") + f.write("Previously passing, now skipped or xfailed:\\n") + for i, test in enumerate(sorted(pass_to_skip), 1): + f.write(f" {i}. {test}\\n") + f.write("\\n") + + if fail_to_skip: + f.write(f"FAIL-TO-SKIP REGRESSIONS ({len(fail_to_skip)} tests)\\n") + f.write("Previously failing, now skipped:\\n") + for i, test in enumerate(sorted(fail_to_skip), 1): + f.write(f" {i}. {test}\\n") + f.write("\\n") + + if discovery_regressions: + f.write(f"DISCOVERY REGRESSIONS ({len(discovery_regressions)} warnings)\\n") + f.write("New warnings not present in target branch:\\n") + for i, warning in enumerate(sorted(discovery_regressions), 1): + f.write(f" {i}. {warning[:200]}...\\n") + f.write("\\n") + if not has_any_regressions: + f.write("No regressions detected across all categories.\\n") + + # Also create the simple regression file for backward compatibility + if pass_to_fail: with open("regression_details.txt", "w") as f: - f.write(f"Found {len(regression_tests)} tests that were passing in target branch but now failing in PR branch:\\n\\n") - for idx, test in enumerate(sorted(regression_tests), 1): + f.write(f"Found {len(pass_to_fail)} tests that were passing in target branch but now failing in PR branch:\\n\\n") + for idx, test in enumerate(sorted(pass_to_fail), 1): f.write(f"{idx}. {test}\\n") - print("Regression details written to file") + + # Print summary + print(f"📊 Regression Analysis Results:") + print(f" Pass-to-Fail: {len(pass_to_fail)} tests") + print(f" Pass-to-Skip: {len(pass_to_skip)} tests") + print(f" Fail-to-Skip: {len(fail_to_skip)} tests") + print(f" Discovery: {len(discovery_regressions)} warnings") + + if has_any_regressions: + print(f"❌ Total regressions detected: {len(pass_to_fail) + len(pass_to_skip) + len(fail_to_skip) + len(discovery_regressions)}") else: - print("No regressions found") + print("✅ No regressions detected") + except Exception as e: print(f"Error in regression analysis: {e}") import traceback print(traceback.format_exc()) EOF - echo "✅ Regression analysis completed" + echo "✅ Comprehensive regression analysis completed" - name: Check for regression details file id: check-regressions @@ -837,133 +896,11 @@ jobs: uses: actions/upload-artifact@v4 with: name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests - path: regression_details.txt + path: | + regression_details.txt + comprehensive_regression_report.txt retention-days: 1 - - name: Check for skip/xfail regressions from artifacts - run: | - echo "Checking for skip/xfail status regressions..." - - python3 - << 'EOF' - import json - import os - import glob - - try: - # Load test data from artifacts - target_data = {} - pr_data = {} - - # Load target and PR data - target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') - if target_files: - with open(target_files[0], 'r') as f: - target_data = json.load(f) - - pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') - if pr_files: - with open(pr_files[0], 'r') as f: - pr_data = json.load(f) - - # Extract test arrays - target_passing = target_data.get('passing_tests', []) - pr_skipped = pr_data.get('skipped_tests', []) - pr_xfailed = pr_data.get('xfailed_tests', []) - pr_failing = pr_data.get('failing_tests', []) - pr_skipped_with_reasons = pr_data.get('skipped_tests_with_reasons', {}) - pr_xfailed_with_reasons = pr_data.get('xfailed_tests_with_reasons', {}) - - print(f"Parsed {len(target_passing)} passing tests from target branch") - print(f"Parsed {len(pr_skipped)} skipped tests from PR branch") - print(f"Parsed {len(pr_xfailed)} xfailed tests from PR branch") - print(f"Parsed {len(pr_skipped_with_reasons)} skipped tests with reasons") - print(f"Parsed {len(pr_xfailed_with_reasons)} xfailed tests with reasons") - - # Find tests that were passing in target but are now skipped or xfailed in PR - target_passing_set = set(target_passing) - pr_skipped_set = set(pr_skipped) - pr_xfailed_set = set(pr_xfailed) - - # Check different types of regressions - passing_to_skipped = list(target_passing_set.intersection(pr_skipped_set)) - passing_to_xfailed = list(target_passing_set.intersection(pr_xfailed_set)) - - # TODO: Add failing→skipped and xfailed→skipped when we have target failing/xfailed data - - total_skip_xfail_regressions = len(passing_to_skipped) + len(passing_to_xfailed) - - if total_skip_xfail_regressions > 0: - print(f"⚠️ Skip/XFail Regressions: {total_skip_xfail_regressions} test(s)") - - if passing_to_skipped: - print(f"⚠️ Passing→Skipped: {len(passing_to_skipped)} test(s)") - - if passing_to_xfailed: - print(f"⚠️ Passing→XFailed: {len(passing_to_xfailed)} test(s)") - else: - print("✅ No skip/xfail regressions detected") - except Exception as e: - print(f"Error in skip/xfail analysis: {e}") - import traceback - print(traceback.format_exc()) - EOF - - echo "✅ Skip/xfail analysis completed" - - - name: Check for warning regressions from artifacts - run: | - echo "Checking for warning regressions between target and PR branches" - - python3 - << 'EOF' - import json - import os - import glob - - try: - # Load test data from artifacts - target_data = {} - pr_data = {} - - # Load target and PR data - target_files = glob.glob('./artifacts/target_branch_data_*/target_test_data.json') - if target_files: - with open(target_files[0], 'r') as f: - target_data = json.load(f) - - pr_files = glob.glob('./artifacts/pr_branch_data_*/pr_test_data.json') - if pr_files: - with open(pr_files[0], 'r') as f: - pr_data = json.load(f) - - # Extract warnings - target_warnings = target_data.get('warnings', []) - pr_warnings = pr_data.get('warnings', []) - - print(f"Target branch has {len(target_warnings)} warning groups") - print(f"PR branch has {len(pr_warnings)} warning groups") - - # Check for new warnings (warnings in PR that aren't in target) - target_warning_set = set(target_warnings) - pr_warning_set = set(pr_warnings) - - new_warnings = list(pr_warning_set - target_warning_set) - resolved_warnings = list(target_warning_set - pr_warning_set) - - if new_warnings: - print(f"⚠️ Warning Regressions: {len(new_warnings)} new warning(s) detected") - elif resolved_warnings: - print(f"✅ Warning Improvements: {len(resolved_warnings)} warning(s) resolved") - else: - print("✅ No warning changes detected") - - except Exception as e: - print(f"Error in warning regression analysis: {e}") - import traceback - print(traceback.format_exc()) - EOF - - echo "Warning regression analysis completed" - - name: Check for test additions and removals from artifacts run: | # Create analysis debug file @@ -1058,23 +995,34 @@ jobs: fi fi - # Fail if any tests passed in target branch but now fail in PR branch + # Check for regressions and display comprehensive summary if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then - echo "❌ PR branch has test regressions from target branch" + echo "❌ Test regressions detected from target branch" REGRESSION_COUNT_VAL=${{ needs.perform-regression-analysis.outputs.regression_count }} - echo " - $REGRESSION_COUNT_VAL tests that were passing in target branch are now failing" echo "### :x: Test Regressions Detected!" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "**$REGRESSION_COUNT_VAL test(s) that were passing in the target branch are now failing in the PR branch.**" >> $GITHUB_STEP_SUMMARY - echo "This job (\`compare-results\`) has been marked as failed due to these regressions." >> $GITHUB_STEP_SUMMARY + echo "**$REGRESSION_COUNT_VAL test regression(s) detected.** See detailed breakdown below:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY - if [ -f regression_details.txt ]; then - echo "Failing tests (regressions) list:" + # Display comprehensive regression report if available + if [ -f comprehensive_regression_report.txt ]; then + echo "📋 **Comprehensive Regression Analysis:**" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + cat comprehensive_regression_report.txt >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Detailed regression analysis:" + cat comprehensive_regression_report.txt + elif [ -f regression_details.txt ]; then + echo "Regression details:" cat regression_details.txt else - echo "::warning::Regression details file (regression_details.txt) not found in Compare test results step. It might be available as an artifact from the 'perform-regression-analysis' job." + echo "::warning::Regression details files not found." fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "This job (\`compare-results\`) has been marked as failed due to these regressions." >> $GITHUB_STEP_SUMMARY exit 1 fi From 86bb9968979f79dd5238e84d729406ea448d6c2c Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 13:51:15 -0600 Subject: [PATCH 16/44] Update regression script; --- .github/workflows/test-py-pytest.yml | 62 ++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index efa5e8c..ab7f1b6 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -995,26 +995,74 @@ jobs: fi fi - # Check for regressions and display comprehensive summary - if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]]; then + # Check for regressions from meta-regression-analysis OR our comprehensive analysis + COMPREHENSIVE_REGRESSIONS="false" + if [ -f comprehensive_regression_report.txt ]; then + # Check if there are any actual regressions in our comprehensive report + if grep -q "REGRESSIONS.*([1-9]" comprehensive_regression_report.txt; then + COMPREHENSIVE_REGRESSIONS="true" + fi + fi + + if [[ "${{ needs.perform-regression-analysis.outputs.has_regressions }}" == "true" ]] || [[ "$COMPREHENSIVE_REGRESSIONS" == "true" ]]; then echo "❌ Test regressions detected from target branch" REGRESSION_COUNT_VAL=${{ needs.perform-regression-analysis.outputs.regression_count }} echo "### :x: Test Regressions Detected!" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "**$REGRESSION_COUNT_VAL test regression(s) detected.** See detailed breakdown below:" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY + + # Extract counts from comprehensive report if available + if [ -f comprehensive_regression_report.txt ]; then + PASS_FAIL_COUNT=$(grep -o "PASS-TO-FAIL REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + PASS_SKIP_COUNT=$(grep -o "PASS-TO-SKIP REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + FAIL_SKIP_COUNT=$(grep -o "FAIL-TO-SKIP REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + DISCOVERY_COUNT=$(grep -o "DISCOVERY REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + + TOTAL_REGRESSIONS=$((PASS_FAIL_COUNT + PASS_SKIP_COUNT + FAIL_SKIP_COUNT + DISCOVERY_COUNT)) + + echo "**$TOTAL_REGRESSIONS total regression(s) detected across multiple categories:**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Category | Count |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Pass → Fail | $PASS_FAIL_COUNT |" >> $GITHUB_STEP_SUMMARY + echo "| Pass → Skip/XFail | $PASS_SKIP_COUNT |" >> $GITHUB_STEP_SUMMARY + echo "| Fail → Skip | $FAIL_SKIP_COUNT |" >> $GITHUB_STEP_SUMMARY + echo "| Discovery Warnings | $DISCOVERY_COUNT |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + else + echo "**$REGRESSION_COUNT_VAL test regression(s) detected.** See detailed breakdown below:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi # Display comprehensive regression report if available if [ -f comprehensive_regression_report.txt ]; then echo "📋 **Comprehensive Regression Analysis:**" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - cat comprehensive_regression_report.txt >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Parse and format the comprehensive report for better GitHub display + while IFS= read -r line; do + if [[ "$line" =~ ^[A-Z-].*REGRESSIONS.*\([0-9]+ ]]; then + echo "### $line" >> $GITHUB_STEP_SUMMARY + elif [[ "$line" =~ ^Previously ]]; then + echo "*$line*" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + elif [[ "$line" =~ ^[[:space:]]*[0-9]+\. ]]; then + echo "- ${line#*. }" >> $GITHUB_STEP_SUMMARY + elif [[ ! "$line" =~ ^=.*=$ ]] && [[ -n "$line" ]]; then + echo "$line" >> $GITHUB_STEP_SUMMARY + fi + done < comprehensive_regression_report.txt + echo "" >> $GITHUB_STEP_SUMMARY echo "Detailed regression analysis:" cat comprehensive_regression_report.txt elif [ -f regression_details.txt ]; then + echo "### Pass-to-Fail Regressions" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + grep "^[0-9]\+\." regression_details.txt | while read -r line; do + echo "- ${line#*. }" >> $GITHUB_STEP_SUMMARY + done + echo "" >> $GITHUB_STEP_SUMMARY echo "Regression details:" cat regression_details.txt else From 3f35bb356e2bb5c95b0fdeb4972316408bdbc2c3 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 13:56:03 -0600 Subject: [PATCH 17/44] Update regression script with pass-to-gone and fail-to-gone; --- .github/workflows/test-py-pytest.yml | 32 +++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index ab7f1b6..a219ca3 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -780,14 +780,20 @@ jobs: pr_xfailed = set(pr_data.get('xfailed_tests', [])) pr_warnings = set(pr_data.get('warnings', [])) + # All tests in each branch (regardless of outcome) + target_all_tests = target_passing.union(target_failing, target_skipped, target_xfailed) + pr_all_tests = pr_passing.union(pr_failing, pr_skipped, pr_xfailed) + # Analyze different regression types pass_to_fail = list(target_passing.intersection(pr_failing)) pass_to_skip = list(target_passing.intersection(pr_skipped.union(pr_xfailed))) fail_to_skip = list(target_failing.intersection(pr_skipped)) + pass_to_gone = list(target_passing - pr_all_tests) # Passing tests that completely disappeared + fail_to_gone = list(target_failing - pr_all_tests) # Failing tests that completely disappeared discovery_regressions = list(pr_warnings - target_warnings) # Create comprehensive regression report - has_any_regressions = bool(pass_to_fail or pass_to_skip or fail_to_skip or discovery_regressions) + has_any_regressions = bool(pass_to_fail or pass_to_skip or fail_to_skip or pass_to_gone or fail_to_gone or discovery_regressions) with open("comprehensive_regression_report.txt", "w") as f: f.write("COMPREHENSIVE REGRESSION ANALYSIS\\n") @@ -814,6 +820,20 @@ jobs: f.write(f" {i}. {test}\\n") f.write("\\n") + if pass_to_gone: + f.write(f"PASS-TO-GONE REGRESSIONS ({len(pass_to_gone)} tests)\\n") + f.write("Previously passing, now completely missing:\\n") + for i, test in enumerate(sorted(pass_to_gone), 1): + f.write(f" {i}. {test}\\n") + f.write("\\n") + + if fail_to_gone: + f.write(f"FAIL-TO-GONE REGRESSIONS ({len(fail_to_gone)} tests)\\n") + f.write("Previously failing, now completely missing:\\n") + for i, test in enumerate(sorted(fail_to_gone), 1): + f.write(f" {i}. {test}\\n") + f.write("\\n") + if discovery_regressions: f.write(f"DISCOVERY REGRESSIONS ({len(discovery_regressions)} warnings)\\n") f.write("New warnings not present in target branch:\\n") @@ -836,10 +856,12 @@ jobs: print(f" Pass-to-Fail: {len(pass_to_fail)} tests") print(f" Pass-to-Skip: {len(pass_to_skip)} tests") print(f" Fail-to-Skip: {len(fail_to_skip)} tests") + print(f" Pass-to-Gone: {len(pass_to_gone)} tests") + print(f" Fail-to-Gone: {len(fail_to_gone)} tests") print(f" Discovery: {len(discovery_regressions)} warnings") if has_any_regressions: - print(f"❌ Total regressions detected: {len(pass_to_fail) + len(pass_to_skip) + len(fail_to_skip) + len(discovery_regressions)}") + print(f"❌ Total regressions detected: {len(pass_to_fail) + len(pass_to_skip) + len(fail_to_skip) + len(pass_to_gone) + len(fail_to_gone) + len(discovery_regressions)}") else: print("✅ No regressions detected") @@ -1016,9 +1038,11 @@ jobs: PASS_FAIL_COUNT=$(grep -o "PASS-TO-FAIL REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") PASS_SKIP_COUNT=$(grep -o "PASS-TO-SKIP REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") FAIL_SKIP_COUNT=$(grep -o "FAIL-TO-SKIP REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + PASS_GONE_COUNT=$(grep -o "PASS-TO-GONE REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + FAIL_GONE_COUNT=$(grep -o "FAIL-TO-GONE REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") DISCOVERY_COUNT=$(grep -o "DISCOVERY REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") - TOTAL_REGRESSIONS=$((PASS_FAIL_COUNT + PASS_SKIP_COUNT + FAIL_SKIP_COUNT + DISCOVERY_COUNT)) + TOTAL_REGRESSIONS=$((PASS_FAIL_COUNT + PASS_SKIP_COUNT + FAIL_SKIP_COUNT + PASS_GONE_COUNT + FAIL_GONE_COUNT + DISCOVERY_COUNT)) echo "**$TOTAL_REGRESSIONS total regression(s) detected across multiple categories:**" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -1027,6 +1051,8 @@ jobs: echo "| Pass → Fail | $PASS_FAIL_COUNT |" >> $GITHUB_STEP_SUMMARY echo "| Pass → Skip/XFail | $PASS_SKIP_COUNT |" >> $GITHUB_STEP_SUMMARY echo "| Fail → Skip | $FAIL_SKIP_COUNT |" >> $GITHUB_STEP_SUMMARY + echo "| Pass → Gone | $PASS_GONE_COUNT |" >> $GITHUB_STEP_SUMMARY + echo "| Fail → Gone | $FAIL_GONE_COUNT |" >> $GITHUB_STEP_SUMMARY echo "| Discovery Warnings | $DISCOVERY_COUNT |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY else From 051f154c1e5f3873efd707fe16d98ca462896abf Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 14:04:07 -0600 Subject: [PATCH 18/44] Update regression script with pass-to-gone and fail-to-gone; --- .github/workflows/test-py-pytest.yml | 62 +++++++++++++--------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index a219ca3..4232ed0 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -796,60 +796,60 @@ jobs: has_any_regressions = bool(pass_to_fail or pass_to_skip or fail_to_skip or pass_to_gone or fail_to_gone or discovery_regressions) with open("comprehensive_regression_report.txt", "w") as f: - f.write("COMPREHENSIVE REGRESSION ANALYSIS\\n") - f.write("=" * 50 + "\\n\\n") + f.write("COMPREHENSIVE REGRESSION ANALYSIS\n") + f.write("=" * 50 + "\n\n") if pass_to_fail: - f.write(f"PASS-TO-FAIL REGRESSIONS ({len(pass_to_fail)} tests)\\n") - f.write("Previously passing, now failing:\\n") + f.write(f"PASS-TO-FAIL REGRESSIONS ({len(pass_to_fail)} tests)\n") + f.write("Previously passing, now failing:\n") for i, test in enumerate(sorted(pass_to_fail), 1): - f.write(f" {i}. {test}\\n") - f.write("\\n") + f.write(f" {i}. {test}\n") + f.write("\n") if pass_to_skip: - f.write(f"PASS-TO-SKIP REGRESSIONS ({len(pass_to_skip)} tests)\\n") - f.write("Previously passing, now skipped or xfailed:\\n") + f.write(f"PASS-TO-SKIP REGRESSIONS ({len(pass_to_skip)} tests)\n") + f.write("Previously passing, now skipped or xfailed:\n") for i, test in enumerate(sorted(pass_to_skip), 1): - f.write(f" {i}. {test}\\n") - f.write("\\n") + f.write(f" {i}. {test}\n") + f.write("\n") if fail_to_skip: - f.write(f"FAIL-TO-SKIP REGRESSIONS ({len(fail_to_skip)} tests)\\n") - f.write("Previously failing, now skipped:\\n") + f.write(f"FAIL-TO-SKIP REGRESSIONS ({len(fail_to_skip)} tests)\n") + f.write("Previously failing, now skipped:\n") for i, test in enumerate(sorted(fail_to_skip), 1): - f.write(f" {i}. {test}\\n") - f.write("\\n") + f.write(f" {i}. {test}\n") + f.write("\n") if pass_to_gone: - f.write(f"PASS-TO-GONE REGRESSIONS ({len(pass_to_gone)} tests)\\n") - f.write("Previously passing, now completely missing:\\n") + f.write(f"PASS-TO-GONE REGRESSIONS ({len(pass_to_gone)} tests)\n") + f.write("Previously passing, now completely missing:\n") for i, test in enumerate(sorted(pass_to_gone), 1): - f.write(f" {i}. {test}\\n") - f.write("\\n") + f.write(f" {i}. {test}\n") + f.write("\n") if fail_to_gone: - f.write(f"FAIL-TO-GONE REGRESSIONS ({len(fail_to_gone)} tests)\\n") - f.write("Previously failing, now completely missing:\\n") + f.write(f"FAIL-TO-GONE REGRESSIONS ({len(fail_to_gone)} tests)\n") + f.write("Previously failing, now completely missing:\n") for i, test in enumerate(sorted(fail_to_gone), 1): - f.write(f" {i}. {test}\\n") - f.write("\\n") + f.write(f" {i}. {test}\n") + f.write("\n") if discovery_regressions: - f.write(f"DISCOVERY REGRESSIONS ({len(discovery_regressions)} warnings)\\n") - f.write("New warnings not present in target branch:\\n") + f.write(f"DISCOVERY REGRESSIONS ({len(discovery_regressions)} warnings)\n") + f.write("New warnings not present in target branch:\n") for i, warning in enumerate(sorted(discovery_regressions), 1): - f.write(f" {i}. {warning[:200]}...\\n") - f.write("\\n") + f.write(f" {i}. {warning[:200]}...\n") + f.write("\n") if not has_any_regressions: - f.write("No regressions detected across all categories.\\n") + f.write("No regressions detected across all categories.\n") # Also create the simple regression file for backward compatibility if pass_to_fail: with open("regression_details.txt", "w") as f: - f.write(f"Found {len(pass_to_fail)} tests that were passing in target branch but now failing in PR branch:\\n\\n") + f.write(f"Found {len(pass_to_fail)} tests that were passing in target branch but now failing in PR branch:\n\n") for idx, test in enumerate(sorted(pass_to_fail), 1): - f.write(f"{idx}. {test}\\n") + f.write(f"{idx}. {test}\n") # Print summary print(f"📊 Regression Analysis Results:") @@ -1080,8 +1080,6 @@ jobs: done < comprehensive_regression_report.txt echo "" >> $GITHUB_STEP_SUMMARY - echo "Detailed regression analysis:" - cat comprehensive_regression_report.txt elif [ -f regression_details.txt ]; then echo "### Pass-to-Fail Regressions" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -1089,8 +1087,6 @@ jobs: echo "- ${line#*. }" >> $GITHUB_STEP_SUMMARY done echo "" >> $GITHUB_STEP_SUMMARY - echo "Regression details:" - cat regression_details.txt else echo "::warning::Regression details files not found." fi From 32bbe0a5f64994d09e9154d342f9cd37b0d5f732 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 14:18:39 -0600 Subject: [PATCH 19/44] Fix dupe artifacts; --- .github/workflows/meta-regression-analysis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/meta-regression-analysis.yml b/.github/workflows/meta-regression-analysis.yml index 06037e5..348a555 100644 --- a/.github/workflows/meta-regression-analysis.yml +++ b/.github/workflows/meta-regression-analysis.yml @@ -205,10 +205,6 @@ jobs: run: | echo "Regressions detected. Failing the job as per configuration." REGRESSION_COUNT_VAL=${{ steps.check-regressions-script.outputs.REGRESSION_COUNT }} - echo "### :x: Regressions Found!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**$REGRESSION_COUNT_VAL regression(s) detected.** This job was intentionally failed." >> $GITHUB_STEP_SUMMARY - echo "The 'Upload regression details artifact' step will still attempt to upload details." >> $GITHUB_STEP_SUMMARY exit 1 - name: Upload regression details artifact From 2b4dd8058b2031cf4f536a7f729affb35b8d5add Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 14:19:30 -0600 Subject: [PATCH 20/44] Fix dupe artifacts; --- .github/workflows/meta-regression-analysis.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/meta-regression-analysis.yml b/.github/workflows/meta-regression-analysis.yml index 348a555..5533b6e 100644 --- a/.github/workflows/meta-regression-analysis.yml +++ b/.github/workflows/meta-regression-analysis.yml @@ -206,11 +206,3 @@ jobs: echo "Regressions detected. Failing the job as per configuration." REGRESSION_COUNT_VAL=${{ steps.check-regressions-script.outputs.REGRESSION_COUNT }} exit 1 - - - name: Upload regression details artifact - uses: actions/upload-artifact@v4 - if: always() - with: - name: regression_details_pr_${{ inputs.pr_number || inputs.run_id }}_${{ inputs.item_type_plural }} - path: regression_details.txt - retention-days: 7 From 3999586b663cf1fd9fcffdb7fe5d51eec67d2132 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 17:13:56 -0600 Subject: [PATCH 21/44] Fix Discord; --- .github/workflows/test-py-pytest.yml | 102 +++++++++++++++++++++++---- 1 file changed, 89 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index 4232ed0..fb78b42 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -1386,8 +1386,8 @@ jobs: if [[ "$PR_COLLECTION_ERRORS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: ERROR: Test Discovery Failed in PR Branch (\`${PR_BRANCH}\`)**") MESSAGE_LINES+=(" - Type: \`${PR_ERROR_TYPE}\`") - MESSAGE_LINES+=(" - Details: \`\`\`${PR_ERROR_DETAILS_TRUNCATED}\`\`\`") MESSAGE_LINES+=(" - This usually indicates import errors or syntax issues preventing tests from being collected.") + MESSAGE_LINES+=(" - See attached file for detailed error information.") elif [[ "$PR_NO_TESTS_FOUND" == "true" ]]; then MESSAGE_LINES+=("**:warning: WARNING: No Tests Found in PR Branch (\`${PR_BRANCH}\`)**") MESSAGE_LINES+=(" - Pytest did not discover any test files matching its patterns.") @@ -1397,20 +1397,96 @@ jobs: # Regression Analysis Summary if [[ "$HAS_REGRESSIONS" == "true" ]]; then MESSAGE_LINES+=("**:red_circle: REGRESSIONS DETECTED**") - MESSAGE_LINES+=(" - **${REGRESSION_COUNT} test(s)** that were passing in \`${TARGET_BRANCH}\` are now **failing** in \`${PR_BRANCH}\`.") - # Calculate current message length - CURRENT_MESSAGE=$(printf "%s\\n" "${MESSAGE_LINES[@]}") - CURRENT_LENGTH=${#CURRENT_MESSAGE} - - if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then - echo "✅ Regression file found, attaching to Discord message" - MESSAGE_LINES+=(" - Detailed list of ${REGRESSION_COUNT} regressions attached as \`regression_details.txt\`") - ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" + # Check if we have comprehensive regression file with categories + if [ -f "comprehensive_regression_report.txt" ]; then + # Extract counts from comprehensive report + PASS_FAIL_COUNT=$(grep -o "PASS-TO-FAIL REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + PASS_SKIP_COUNT=$(grep -o "PASS-TO-SKIP REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + FAIL_SKIP_COUNT=$(grep -o "FAIL-TO-SKIP REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + PASS_GONE_COUNT=$(grep -o "PASS-TO-GONE REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + FAIL_GONE_COUNT=$(grep -o "FAIL-TO-GONE REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + DISCOVERY_COUNT=$(grep -o "DISCOVERY REGRESSIONS (\([0-9]*\)" comprehensive_regression_report.txt | grep -o "[0-9]*" || echo "0") + + # Add category summaries (≤5 show paths, >5 show count + refer to file) + if [[ "$PASS_FAIL_COUNT" -gt 0 ]]; then + if [[ "$PASS_FAIL_COUNT" -le 5 ]]; then + MESSAGE_LINES+=("**Pass→Fail ($PASS_FAIL_COUNT):**") + readarray -t test_paths < <(grep -A 100 "PASS-TO-FAIL REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + for test_path in "${test_paths[@]}"; do + MESSAGE_LINES+=(" - \`$test_path\`") + done + else + MESSAGE_LINES+=("**Pass→Fail:** $PASS_FAIL_COUNT tests (see attached file)") + fi + fi + + if [[ "$PASS_SKIP_COUNT" -gt 0 ]]; then + if [[ "$PASS_SKIP_COUNT" -le 5 ]]; then + MESSAGE_LINES+=("**Pass→Skip ($PASS_SKIP_COUNT):**") + readarray -t test_paths < <(grep -A 100 "PASS-TO-SKIP REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + for test_path in "${test_paths[@]}"; do + MESSAGE_LINES+=(" - \`$test_path\`") + done + else + MESSAGE_LINES+=("**Pass→Skip:** $PASS_SKIP_COUNT tests (see attached file)") + fi + fi + + if [[ "$FAIL_SKIP_COUNT" -gt 0 ]]; then + if [[ "$FAIL_SKIP_COUNT" -le 5 ]]; then + MESSAGE_LINES+=("**Fail→Skip ($FAIL_SKIP_COUNT):**") + readarray -t test_paths < <(grep -A 100 "FAIL-TO-SKIP REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + for test_path in "${test_paths[@]}"; do + MESSAGE_LINES+=(" - \`$test_path\`") + done + else + MESSAGE_LINES+=("**Fail→Skip:** $FAIL_SKIP_COUNT tests (see attached file)") + fi + fi + + if [[ "$PASS_GONE_COUNT" -gt 0 ]]; then + if [[ "$PASS_GONE_COUNT" -le 5 ]]; then + MESSAGE_LINES+=("**Pass→Gone ($PASS_GONE_COUNT):**") + readarray -t test_paths < <(grep -A 100 "PASS-TO-GONE REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + for test_path in "${test_paths[@]}"; do + MESSAGE_LINES+=(" - \`$test_path\`") + done + else + MESSAGE_LINES+=("**Pass→Gone:** $PASS_GONE_COUNT tests (see attached file)") + fi + fi + + if [[ "$FAIL_GONE_COUNT" -gt 0 ]]; then + if [[ "$FAIL_GONE_COUNT" -le 5 ]]; then + MESSAGE_LINES+=("**Fail→Gone ($FAIL_GONE_COUNT):**") + readarray -t test_paths < <(grep -A 100 "FAIL-TO-GONE REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + for test_path in "${test_paths[@]}"; do + MESSAGE_LINES+=(" - \`$test_path\`") + done + else + MESSAGE_LINES+=("**Fail→Gone:** $FAIL_GONE_COUNT tests (see attached file)") + fi + fi + + if [[ "$DISCOVERY_COUNT" -gt 0 ]]; then + if [[ "$DISCOVERY_COUNT" -le 5 ]]; then + MESSAGE_LINES+=("**Discovery Warnings ($DISCOVERY_COUNT):**") + MESSAGE_LINES+=(" - $DISCOVERY_COUNT new warnings (see attached file)") + else + MESSAGE_LINES+=("**Discovery Warnings:** $DISCOVERY_COUNT warnings (see attached file)") + fi + fi + + ARTIFACT_PATH_OUTPUT="comprehensive_regression_report.txt" else - echo "❌ Regression file not found for Discord attachment" - MESSAGE_LINES+=(" - Regression details file not available") - ARTIFACT_PATH_OUTPUT="" + # Fallback to simple regression count + MESSAGE_LINES+=(" - **${REGRESSION_COUNT} test(s)** that were passing in \`${TARGET_BRANCH}\` are now **failing** in \`${PR_BRANCH}\`.") + if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then + ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" + else + ARTIFACT_PATH_OUTPUT="" + fi fi elif [[ "$COMPARE_RESULT" == "failure" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then # This case handles general comparison failures NOT due to specific regressions From 69eeca1bff8c8d2e235fb9eb38de1bdeaa724345 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Wed, 11 Jun 2025 20:53:36 -0600 Subject: [PATCH 22/44] Hopefully fix gone tests; --- .github/workflows/test-py-pytest.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index fb78b42..c110241 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -532,6 +532,9 @@ jobs: target_passed = 0 target_percentage = 0 passing_tests = [] + failing_tests = [] + skipped_tests = [] + xfailed_tests = [] all_tests = [] try: @@ -557,9 +560,6 @@ jobs: # Extract all test outcomes if 'tests' in target_results: print('Extracting all test outcomes from target') - failing_tests = [] - skipped_tests = [] - xfailed_tests = [] for test in target_results['tests']: outcome = test.get('outcome') nodeid = test.get('nodeid', '') @@ -654,9 +654,9 @@ jobs: # Save test lists to artifact files instead of job outputs test_data = { 'passing_tests': passing_tests, - 'failing_tests': failing_tests if 'failing_tests' in locals() else [], - 'skipped_tests': skipped_tests if 'skipped_tests' in locals() else [], - 'xfailed_tests': xfailed_tests if 'xfailed_tests' in locals() else [], + 'failing_tests': failing_tests, + 'skipped_tests': skipped_tests, + 'xfailed_tests': xfailed_tests, 'all_tests': all_tests, 'warnings': warnings_list } @@ -780,6 +780,18 @@ jobs: pr_xfailed = set(pr_data.get('xfailed_tests', [])) pr_warnings = set(pr_data.get('warnings', [])) + # Debug: Print data for troubleshooting + print(f"🔍 Debug - Target branch data:") + print(f" Passing: {len(target_passing)} tests") + print(f" Failing: {len(target_failing)} tests") + print(f" Skipped: {len(target_skipped)} tests") + print(f" XFailed: {len(target_xfailed)} tests") + print(f"🔍 Debug - PR branch data:") + print(f" Passing: {len(pr_passing)} tests") + print(f" Failing: {len(pr_failing)} tests") + print(f" Skipped: {len(pr_skipped)} tests") + print(f" XFailed: {len(pr_xfailed)} tests") + # All tests in each branch (regardless of outcome) target_all_tests = target_passing.union(target_failing, target_skipped, target_xfailed) pr_all_tests = pr_passing.union(pr_failing, pr_skipped, pr_xfailed) From 32c012408484283ad36f7471faa276e0f3db5945 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Thu, 12 Jun 2025 09:56:32 -0600 Subject: [PATCH 23/44] Fix passing tests; --- .github/workflows/test-py-pytest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index c110241..a80b78e 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -350,6 +350,7 @@ jobs: # Save test lists to artifact files instead of job outputs test_data = { + 'passing_tests': passing_tests, 'failing_tests': failing_tests, 'skipped_tests': skipped_tests, 'xfailed_tests': xfailed_tests, From a9c30f89c18f7d5f1890383aa5c397b87e94c098 Mon Sep 17 00:00:00 2001 From: James Grieve Date: Thu, 12 Jun 2025 10:16:52 -0600 Subject: [PATCH 24/44] Fix passing tests; --- .github/workflows/test-py-pytest.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index a80b78e..17db67c 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -207,6 +207,7 @@ jobs: skipped_tests = [] xfailed_tests = [] all_tests = [] + passing_tests = [] skipped_tests_with_reasons = {} xfailed_tests_with_reasons = {} @@ -220,7 +221,7 @@ jobs: if pr_results.get('exitcode', 0) > 1: print('Detected non-zero exitcode, likely a collection error') if 'collectors' in pr_results and pr_results['collectors']: - print(f'Collection errors found: {pr_results[\"collectors\"]}') + print(f'Collection errors found: {pr_results["collectors"]}') pr_total = 0 # Explicitly set to 0 - no tests run when collection fails pr_passed = 0 elif 'summary' in pr_results and isinstance(pr_results['summary'], dict): @@ -238,7 +239,9 @@ jobs: nodeid = test.get('nodeid', '') if nodeid: all_tests.append(nodeid) # Track all tests regardless of outcome - if outcome in ['failed', 'error']: + if outcome == 'passed': + passing_tests.append(nodeid) + elif outcome in ['failed', 'error']: failing_tests.append(nodeid) elif outcome == 'skipped': skipped_tests.append(nodeid) @@ -268,6 +271,7 @@ jobs: xfail_reason = str(test['call']['longrepr']) xfailed_tests_with_reasons[nodeid] = xfail_reason.strip() + print(f'Found {len(passing_tests)} passing tests') print(f'Found {len(failing_tests)} failing tests') print(f'Found {len(skipped_tests)} skipped tests') print(f'Found {len(xfailed_tests)} xfailed tests') @@ -286,7 +290,7 @@ jobs: if 'pr_results' in locals(): print(f'Available keys: {list(pr_results.keys())}') if 'summary' in pr_results: - print(f'Summary structure: {pr_results[\"summary\"]}') + print(f'Summary structure: {pr_results["summary"]}') except Exception as e: print(f'Error processing results: {e}') import traceback @@ -548,7 +552,7 @@ jobs: if target_results.get('exitcode', 0) > 1: print('Detected non-zero exitcode, likely a collection error') if 'collectors' in target_results and target_results['collectors']: - print(f'Collection errors found: {target_results[\"collectors\"]}') + print(f'Collection errors found: {target_results["collectors"]}') target_total = 0 # Explicitly set to 0 - no tests run when collection fails target_passed = 0 elif 'summary' in target_results and isinstance(target_results['summary'], dict): @@ -594,7 +598,7 @@ jobs: if 'target_results' in locals(): print(f'Available keys: {list(target_results.keys())}') if 'summary' in target_results: - print(f'Summary structure: {target_results[\"summary\"]}') + print(f'Summary structure: {target_results["summary"]}') except Exception as e: print(f'Error processing results: {e}') import traceback From 76d5153f3227ab66aa492e1bcb83e70455323eda Mon Sep 17 00:00:00 2001 From: James Grieve Date: Thu, 12 Jun 2025 10:31:34 -0600 Subject: [PATCH 25/44] Discord file upload - maybe; --- .github/workflows/meta-regression-analysis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/meta-regression-analysis.yml b/.github/workflows/meta-regression-analysis.yml index 5533b6e..026f527 100644 --- a/.github/workflows/meta-regression-analysis.yml +++ b/.github/workflows/meta-regression-analysis.yml @@ -200,6 +200,16 @@ jobs: fi fi + - name: Upload regression analysis results + if: always() + uses: actions/upload-artifact@v4 + with: + name: regression_analysis_results_${{ inputs.pr_number || inputs.run_id }} + path: | + regression_details.txt + retention-days: 3 + if-no-files-found: ignore + - name: Fail job if regressions are found if: steps.check-regressions-script.outputs.HAS_REGRESSIONS == 'true' run: | From 46e199adc24ea5537add5caf3d5a292dc38b3c3b Mon Sep 17 00:00:00 2001 From: James Grieve Date: Thu, 12 Jun 2025 10:48:07 -0600 Subject: [PATCH 26/44] Fix count; Maybe fix files; --- .github/workflows/test-py-pytest.yml | 52 +++++++++++++++++----------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/test-py-pytest.yml index 17db67c..ac7c8a1 100644 --- a/.github/workflows/test-py-pytest.yml +++ b/.github/workflows/test-py-pytest.yml @@ -931,7 +931,7 @@ jobs: fi - name: Upload regression details artifact - if: steps.check-regressions.outputs.HAS_REGRESSIONS == 'true' && steps.check-regressions.outputs.REGRESSION_COUNT > 0 + if: always() uses: actions/upload-artifact@v4 with: name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests @@ -939,6 +939,7 @@ jobs: regression_details.txt comprehensive_regression_report.txt retention-days: 1 + if-no-files-found: ignore - name: Check for test additions and removals from artifacts run: | @@ -1205,7 +1206,7 @@ jobs: fi - name: Download regression details (if any) id: download_regressions - if: needs.perform-regression-analysis.outputs.has_regressions == 'true' && needs.perform-regression-analysis.outputs.regression_count > 0 + if: always() uses: actions/download-artifact@v4 with: name: regression_details_pr_${{ github.event.pull_request.number || github.run_id }}_tests @@ -1213,7 +1214,7 @@ jobs: continue-on-error: true - name: Check downloaded regression file - if: needs.perform-regression-analysis.outputs.has_regressions == 'true' + if: always() run: | echo "Checking for regression details file..." if [ -f "regression_details.txt" ]; then @@ -1223,7 +1224,13 @@ jobs: head -5 regression_details.txt else echo "❌ Regression details file not found" - ls -la + fi + + if [ -f "comprehensive_regression_report.txt" ]; then + echo "✅ Comprehensive regression report found" + echo "File size: $(wc -c < comprehensive_regression_report.txt) bytes" + else + echo "❌ Comprehensive regression report not found" fi - name: Construct Discord Notification @@ -1429,9 +1436,9 @@ jobs: if [[ "$PASS_FAIL_COUNT" -gt 0 ]]; then if [[ "$PASS_FAIL_COUNT" -le 5 ]]; then MESSAGE_LINES+=("**Pass→Fail ($PASS_FAIL_COUNT):**") - readarray -t test_paths < <(grep -A 100 "PASS-TO-FAIL REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + readarray -t test_paths < <(grep -A 100 "PASS-TO-FAIL REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -$PASS_FAIL_COUNT | sed 's/^ [0-9]\+\. //') for test_path in "${test_paths[@]}"; do - MESSAGE_LINES+=(" - \`$test_path\`") + MESSAGE_LINES+=("• \`$test_path\`") done else MESSAGE_LINES+=("**Pass→Fail:** $PASS_FAIL_COUNT tests (see attached file)") @@ -1441,9 +1448,9 @@ jobs: if [[ "$PASS_SKIP_COUNT" -gt 0 ]]; then if [[ "$PASS_SKIP_COUNT" -le 5 ]]; then MESSAGE_LINES+=("**Pass→Skip ($PASS_SKIP_COUNT):**") - readarray -t test_paths < <(grep -A 100 "PASS-TO-SKIP REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + readarray -t test_paths < <(grep -A 100 "PASS-TO-SKIP REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -$PASS_SKIP_COUNT | sed 's/^ [0-9]\+\. //') for test_path in "${test_paths[@]}"; do - MESSAGE_LINES+=(" - \`$test_path\`") + MESSAGE_LINES+=("• \`$test_path\`") done else MESSAGE_LINES+=("**Pass→Skip:** $PASS_SKIP_COUNT tests (see attached file)") @@ -1453,9 +1460,9 @@ jobs: if [[ "$FAIL_SKIP_COUNT" -gt 0 ]]; then if [[ "$FAIL_SKIP_COUNT" -le 5 ]]; then MESSAGE_LINES+=("**Fail→Skip ($FAIL_SKIP_COUNT):**") - readarray -t test_paths < <(grep -A 100 "FAIL-TO-SKIP REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + readarray -t test_paths < <(grep -A 100 "FAIL-TO-SKIP REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -$FAIL_SKIP_COUNT | sed 's/^ [0-9]\+\. //') for test_path in "${test_paths[@]}"; do - MESSAGE_LINES+=(" - \`$test_path\`") + MESSAGE_LINES+=("• \`$test_path\`") done else MESSAGE_LINES+=("**Fail→Skip:** $FAIL_SKIP_COUNT tests (see attached file)") @@ -1465,9 +1472,9 @@ jobs: if [[ "$PASS_GONE_COUNT" -gt 0 ]]; then if [[ "$PASS_GONE_COUNT" -le 5 ]]; then MESSAGE_LINES+=("**Pass→Gone ($PASS_GONE_COUNT):**") - readarray -t test_paths < <(grep -A 100 "PASS-TO-GONE REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + readarray -t test_paths < <(grep -A 100 "PASS-TO-GONE REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -$PASS_GONE_COUNT | sed 's/^ [0-9]\+\. //') for test_path in "${test_paths[@]}"; do - MESSAGE_LINES+=(" - \`$test_path\`") + MESSAGE_LINES+=("• \`$test_path\`") done else MESSAGE_LINES+=("**Pass→Gone:** $PASS_GONE_COUNT tests (see attached file)") @@ -1477,9 +1484,9 @@ jobs: if [[ "$FAIL_GONE_COUNT" -gt 0 ]]; then if [[ "$FAIL_GONE_COUNT" -le 5 ]]; then MESSAGE_LINES+=("**Fail→Gone ($FAIL_GONE_COUNT):**") - readarray -t test_paths < <(grep -A 100 "FAIL-TO-GONE REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -5 | sed 's/^ [0-9]\+\. //') + readarray -t test_paths < <(grep -A 100 "FAIL-TO-GONE REGRESSIONS" comprehensive_regression_report.txt | grep "^ [0-9]\+\." | head -$FAIL_GONE_COUNT | sed 's/^ [0-9]\+\. //') for test_path in "${test_paths[@]}"; do - MESSAGE_LINES+=(" - \`$test_path\`") + MESSAGE_LINES+=("• \`$test_path\`") done else MESSAGE_LINES+=("**Fail→Gone:** $FAIL_GONE_COUNT tests (see attached file)") @@ -1489,21 +1496,15 @@ jobs: if [[ "$DISCOVERY_COUNT" -gt 0 ]]; then if [[ "$DISCOVERY_COUNT" -le 5 ]]; then MESSAGE_LINES+=("**Discovery Warnings ($DISCOVERY_COUNT):**") - MESSAGE_LINES+=(" - $DISCOVERY_COUNT new warnings (see attached file)") + MESSAGE_LINES+=("• $DISCOVERY_COUNT new warnings (see attached file)") else MESSAGE_LINES+=("**Discovery Warnings:** $DISCOVERY_COUNT warnings (see attached file)") fi fi - ARTIFACT_PATH_OUTPUT="comprehensive_regression_report.txt" else # Fallback to simple regression count MESSAGE_LINES+=(" - **${REGRESSION_COUNT} test(s)** that were passing in \`${TARGET_BRANCH}\` are now **failing** in \`${PR_BRANCH}\`.") - if [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then - ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" - else - ARTIFACT_PATH_OUTPUT="" - fi fi elif [[ "$COMPARE_RESULT" == "failure" ]] && [[ "$HAS_REGRESSIONS" != "true" ]]; then # This case handles general comparison failures NOT due to specific regressions @@ -1520,6 +1521,15 @@ jobs: MESSAGE_LINES+=("---") MESSAGE_LINES+=("[View Workflow Run](${ACTION_RUN_URL})") + # Set artifact path - always prefer comprehensive report if it exists + if [ -f "comprehensive_regression_report.txt" ]; then + ARTIFACT_PATH_OUTPUT="comprehensive_regression_report.txt" + elif [ -f "$REGRESSION_FILE_PATH" ] && [[ "$DOWNLOAD_REGRESSIONS_OUTCOME" == "success" ]]; then + ARTIFACT_PATH_OUTPUT="$REGRESSION_FILE_PATH" + else + ARTIFACT_PATH_OUTPUT="" + fi + # Construct with actual newlines FINAL_MESSAGE_BODY=$(printf "%s\\n" "${MESSAGE_LINES[@]}") if [ ${#MESSAGE_LINES[@]} -gt 0 ]; then From f2bd92b3588777b0a6b8b28ef61e3eda93285b30 Mon Sep 17 00:00:00 2001 From: Austin LeBlanc <79474577+austinleblanc@users.noreply.github.com> Date: Mon, 16 Jun 2025 14:01:10 -0400 Subject: [PATCH 27/44] Update build-next.yml Change test-lint-py.yml to test-py-lint.yml --- .github/workflows/build-next.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-next.yml b/.github/workflows/build-next.yml index 3887478..da21bc7 100644 --- a/.github/workflows/build-next.yml +++ b/.github/workflows/build-next.yml @@ -14,7 +14,7 @@ on: default: "ubuntu-latest" jobs: lint: - uses: ./.github/workflows/test-lint-py.yml + uses: ./.github/workflows/test-py-lint.yml permissions: contents: write From 40351c0778ed57aba2ea1e274b2fa28eced83b5a Mon Sep 17 00:00:00 2001 From: Austin LeBlanc <79474577+austinleblanc@users.noreply.github.com> Date: Mon, 16 Jun 2025 14:09:08 -0400 Subject: [PATCH 28/44] Update build-next.yml Swap `npm ci` to `npm i`, `ci` doesn't work in this context. --- .github/workflows/build-next.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-next.yml b/.github/workflows/build-next.yml index da21bc7..2bb4da2 100644 --- a/.github/workflows/build-next.yml +++ b/.github/workflows/build-next.yml @@ -51,6 +51,6 @@ jobs: node-version: ${{ inputs.node-version }} - run: | npm i lightningcss-linux-x64-gnu @tailwindcss/oxide-linux-x64-gnu sass-embedded-linux-x64 - npm ci + npm i npm list zod --depth=0 npm run build From 4f01d0e141db4d028ac6e8a360bce3cb492dbede Mon Sep 17 00:00:00 2001 From: parull249 Date: Mon, 30 Jun 2025 14:48:07 +0530 Subject: [PATCH 29/44] modified workflows --- .github/workflows/sync-meta.yml | 39 ++++++++++ .github/workflows/test-ts-lint.yml | 10 +++ scripts/sync_labels.py | 72 +++++++++++++++++++ triggers/typescript/nextjs/lint.yml | 4 +- .../typescript/nextjs/manage-submodules.yml | 2 +- .../typescript/nextjs/test-downstream.yml | 4 +- 6 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/sync-meta.yml create mode 100644 scripts/sync_labels.py diff --git a/.github/workflows/sync-meta.yml b/.github/workflows/sync-meta.yml new file mode 100644 index 0000000..b8f1abf --- /dev/null +++ b/.github/workflows/sync-meta.yml @@ -0,0 +1,39 @@ +name: Reusable Metadata Sync + +on: + workflow_call: + inputs: + sync_labels: + required: false + type: boolean + default: true + runs_on: + required: false + type: string + default: "ubuntu-latest" + +jobs: + sync: + runs-on: ${{ inputs.runs_on }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # ✅ Use GITHUB_TOKEN safely + + steps: + - name: Checkout self + uses: actions/checkout@v4 + with: + repository: your-org-name/Workflows + token: ${{ secrets.GITHUB_TOKEN }} + path: source + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install requests + + - name: Run Label Sync Script + if: ${{ inputs.sync_labels }} + run: python3 source/scripts/sync_labels.py diff --git a/.github/workflows/test-ts-lint.yml b/.github/workflows/test-ts-lint.yml index 2369565..f476956 100644 --- a/.github/workflows/test-ts-lint.yml +++ b/.github/workflows/test-ts-lint.yml @@ -30,6 +30,14 @@ on: required: false type: string default: "ubuntu-latest" + target_branch_artifact_name: + required: false + type: string + default: target_branch_data_${{ github.event.pull_request.number || github.run_id }} + pr_branch_artifact_name: + required: false + type: string + default: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} outputs: lint_errors_pr_branch: description: "JSON string of lint errors on the PR branch after attempting fixes." @@ -259,6 +267,8 @@ jobs: item_type_plural: "files with lint errors" pr_number: ${{ github.event.pull_request.number }} run_id: ${{ github.run_id }} + target_branch_artifact_name: ${{ inputs.target_branch_artifact_name }} + pr_branch_artifact_name: ${{ inputs.pr_branch_artifact_name }} # Removed outputs and steps from here, as they are defined by the reusable workflow # and accessed via `needs.analyze-regressions.outputs.*` in subsequent jobs. diff --git a/scripts/sync_labels.py b/scripts/sync_labels.py new file mode 100644 index 0000000..fbe1623 --- /dev/null +++ b/scripts/sync_labels.py @@ -0,0 +1,72 @@ +import os +import requests + +# Read the GitHub token from the environment +TOKEN = os.getenv("GH_TOKEN") +if not TOKEN: + raise EnvironmentError("GH_TOKEN is not set in environment") + +HEADERS = {"Authorization": f"token {TOKEN}"} + +# Source repo where truth is stored +SOURCE_REPO = "JamesonRGrieve/Workflows" + +# Target repos to sync to +TARGET_REPOS = [ + "JamesonRGrieve/AGInfrastructure", + "JamesonRGrieve/AGInteractive", + "JamesonRGrieve/AGInterface", + "JamesonRGrieve/AGInYourPC", + "JamesonRGrieve/AGIteration", + "JamesonRGrieve/nursegpt", + "JamesonRGrieve/nursegpt_web", + "JamesonRGrieve/ServerFramework", + "JamesonRGrieve/auth", + "JamesonRGrieve/zod2gql", + "JamesonRGrieve/dynamic-form", + "JamesonRGrieve/ClientFramework" +] + +def get_labels(repo): + url = f"https://api.github.com/repos/{repo}/labels" + res = requests.get(url, headers=HEADERS) + if res.status_code == 200: + print(f"✅ Fetched labels from {repo}") + return res.json() + else: + print(f"❌ Failed to fetch labels from {repo}") + print(f"Status Code: {res.status_code}") + print(f"Response: {res.text}") + return [] + +def create_or_update_label(repo, label): + url = f"https://api.github.com/repos/{repo}/labels" + data = { + "name": label["name"], + "color": label["color"], + "description": label.get("description", "") + } + res = requests.post(url, headers=HEADERS, json=data) + + if res.status_code == 201: + print(f"✅ Created label '{label['name']}' in {repo}") + elif res.status_code == 422 and "already_exists" in res.text: + print(f"⚠️ Label '{label['name']}' already exists in {repo}") + else: + print(f"❌ Failed to create label '{label['name']}' in {repo}") + print(f"Status Code: {res.status_code}") + print(f"Response: {res.text}") + +def sync_labels(): + labels = get_labels(SOURCE_REPO) + if not labels: + print("⚠️ No labels found or failed to fetch. Exiting.") + return + + for target_repo in TARGET_REPOS: + print(f"\n🔄 Syncing labels to {target_repo}") + for label in labels: + create_or_update_label(target_repo, label) + +if __name__ == "__main__": + sync_labels() diff --git a/triggers/typescript/nextjs/lint.yml b/triggers/typescript/nextjs/lint.yml index 1ea4902..96d5f47 100644 --- a/triggers/typescript/nextjs/lint.yml +++ b/triggers/typescript/nextjs/lint.yml @@ -9,8 +9,10 @@ on: jobs: call-reusable-lint: name: Call Reusable Lint Workflow - uses: JamesonRGrieve/Workflows/.github/workflows/test-lint-ts.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/test-ts-lint.yml@main # No specific inputs needed for the reusable lint workflow # Permissions are handled by the reusable workflow itself or should be inherited. # If secrets were needed, they would be passed here like: # secrets: inherit + lint_command: npm run lint + fix_command: npm run lint -- --fix diff --git a/triggers/typescript/nextjs/manage-submodules.yml b/triggers/typescript/nextjs/manage-submodules.yml index 5008bc0..cc1436c 100644 --- a/triggers/typescript/nextjs/manage-submodules.yml +++ b/triggers/typescript/nextjs/manage-submodules.yml @@ -11,7 +11,7 @@ jobs: name: Call Reusable Submodule Management # Make sure the reusable workflow is in the same repository at this path # or use //.github/workflows/manage-submodules.yml@ if it's in a central repo. - uses: JamesonRGrieve/Workflows/.github/workflows/manage-submodules.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/gh-submodules.yml@main with: # REQUIRED: Customize this list of comma-separated file paths # These are the files that will be synced from the main repository to the submodules diff --git a/triggers/typescript/nextjs/test-downstream.yml b/triggers/typescript/nextjs/test-downstream.yml index e9df331..5d69c14 100644 --- a/triggers/typescript/nextjs/test-downstream.yml +++ b/triggers/typescript/nextjs/test-downstream.yml @@ -30,7 +30,7 @@ jobs: name: Run Pytest Comparison # No explicit check_pr_source dependency, tests should run for any PR to main. # If tests pass and PR is from dev, then downstream will run. - uses: JamesonRGrieve/Workflows/.github/workflows/test-pytest.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/test-py-pytest.yml@main with: # PR is to main, so compare PR branch against main. target_branch_to_compare: ${{ github.base_ref }} # This will be 'main' @@ -41,7 +41,7 @@ jobs: name: Call Reusable Stage Downstream Workflow needs: [check-pr-source, run-tests] # Ensures tests pass and branch is correct before staging if: needs.check-pr-source.outputs.is_from_dev == 'true' - uses: JamesonRGrieve/Workflows/.github/workflows/test-downstream.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/gh-test-downstream.yml@main with: source_repo_ref: 'dev' # Staging happens from the 'dev' branch of the current repo secrets: From 668a8f935fec7a044682215a44788bb507c259f8 Mon Sep 17 00:00:00 2001 From: parull249 Date: Mon, 30 Jun 2025 16:47:26 +0530 Subject: [PATCH 30/44] added scripts --- .github/workflows/sync-meta.yml | 56 ++++++++++++++++++++++++++----- scripts/sync_branch_protection.py | 39 +++++++++++++++++++++ scripts/sync_labels.py | 41 ++++++---------------- scripts/sync_milestones.py | 28 ++++++++++++++++ scripts/sync_temlates.py | 35 +++++++++++++++++++ 5 files changed, 160 insertions(+), 39 deletions(-) create mode 100644 scripts/sync_branch_protection.py create mode 100644 scripts/sync_milestones.py create mode 100644 scripts/sync_temlates.py diff --git a/.github/workflows/sync-meta.yml b/.github/workflows/sync-meta.yml index b8f1abf..5c58ae7 100644 --- a/.github/workflows/sync-meta.yml +++ b/.github/workflows/sync-meta.yml @@ -1,12 +1,29 @@ +# Reusable Workflow: Workflows/.github/workflows/sync-meta.yml + name: Reusable Metadata Sync on: workflow_call: + secrets: + PAT_TOKEN: + required: true inputs: sync_labels: required: false type: boolean default: true + sync_templates: + required: false + type: boolean + default: true + sync_milestones: + required: false + type: boolean + default: true + sync_branch_rules: + required: false + type: boolean + default: true runs_on: required: false type: string @@ -15,15 +32,12 @@ on: jobs: sync: runs-on: ${{ inputs.runs_on }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # ✅ Use GITHUB_TOKEN safely - steps: - - name: Checkout self + - name: Checkout source repo (self) uses: actions/checkout@v4 with: - repository: your-org-name/Workflows - token: ${{ secrets.GITHUB_TOKEN }} + repository: JamesonRGrieve/Workflows + token: ${{ secrets.PAT_TOKEN }} path: source - name: Set up Python @@ -32,8 +46,32 @@ jobs: python-version: '3.10' - name: Install dependencies - run: pip install requests + run: pip install requests PyGithub - - name: Run Label Sync Script + - name: Sync Labels if: ${{ inputs.sync_labels }} - run: python3 source/scripts/sync_labels.py + run: | + python3 source/scripts/sync_labels.py + env: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} + + - name: Sync Templates + if: ${{ inputs.sync_templates }} + run: | + python3 source/scripts/sync_templates.py + env: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} + + - name: Sync Milestones + if: ${{ inputs.sync_milestones }} + run: | + python3 source/scripts/sync_milestones.py + env: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} + + - name: Sync Branch Protection Rules + if: ${{ inputs.sync_branch_rules }} + run: | + python3 source/scripts/sync_branch_protection.py + env: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} \ No newline at end of file diff --git a/scripts/sync_branch_protection.py b/scripts/sync_branch_protection.py new file mode 100644 index 0000000..97c9db9 --- /dev/null +++ b/scripts/sync_branch_protection.py @@ -0,0 +1,39 @@ +import os +from github import Github + +TOKEN = os.getenv("GH_TOKEN") +g = Github(TOKEN) + +SOURCE_REPO = g.get_repo("JamesonRGrieve/Workflows") +TARGET_REPOS = [ + "AGInfrastructure", "AGInteractive", "AGInterface", "AGInYourPC", "AGIteration", + "nursegpt", "nursegpt_web", "ServerFramework", "auth", "zod2gql", "dynamic-form", "ClientFramework" +] +ORG = "JamesonRGrieve" + +PROTECTED_BRANCHES = ["main", "dev", "legacy"] + +rules = { + "enforce_admins": True, + "required_status_checks": None, + "required_pull_request_reviews": { + "dismiss_stale_reviews": True, + "require_code_owner_reviews": True, + "required_approving_review_count": 1, + }, + "restrictions": None, + "allow_force_pushes": False, + "allow_deletions": False, + "required_linear_history": True, + "required_conversation_resolution": True +} + +for repo_name in TARGET_REPOS: + repo = g.get_repo(f"{ORG}/{repo_name}") + for branch in PROTECTED_BRANCHES: + try: + print(f"🔐 Protecting {branch} in {repo_name}") + b = repo.get_branch(branch) + b.edit_protection(**rules) + except Exception as e: + print(f"⚠️ Error protecting {branch} in {repo_name}: {e}") diff --git a/scripts/sync_labels.py b/scripts/sync_labels.py index fbe1623..97bc2bc 100644 --- a/scripts/sync_labels.py +++ b/scripts/sync_labels.py @@ -1,17 +1,10 @@ -import os import requests +import os -# Read the GitHub token from the environment -TOKEN = os.getenv("GH_TOKEN") -if not TOKEN: - raise EnvironmentError("GH_TOKEN is not set in environment") - +TOKEN = os.getenv("GITHUB_TOKEN") HEADERS = {"Authorization": f"token {TOKEN}"} -# Source repo where truth is stored SOURCE_REPO = "JamesonRGrieve/Workflows" - -# Target repos to sync to TARGET_REPOS = [ "JamesonRGrieve/AGInfrastructure", "JamesonRGrieve/AGInteractive", @@ -31,42 +24,30 @@ def get_labels(repo): url = f"https://api.github.com/repos/{repo}/labels" res = requests.get(url, headers=HEADERS) if res.status_code == 200: - print(f"✅ Fetched labels from {repo}") return res.json() - else: - print(f"❌ Failed to fetch labels from {repo}") - print(f"Status Code: {res.status_code}") - print(f"Response: {res.text}") - return [] + print(f"❌ Failed to fetch labels from {repo}") + return [] def create_or_update_label(repo, label): url = f"https://api.github.com/repos/{repo}/labels" - data = { + res = requests.post(url, headers=HEADERS, json={ "name": label["name"], "color": label["color"], "description": label.get("description", "") - } - res = requests.post(url, headers=HEADERS, json=data) - - if res.status_code == 201: - print(f"✅ Created label '{label['name']}' in {repo}") - elif res.status_code == 422 and "already_exists" in res.text: + }) + if res.status_code == 422 and "already_exists" in str(res.text): print(f"⚠️ Label '{label['name']}' already exists in {repo}") - else: + elif res.status_code != 201: print(f"❌ Failed to create label '{label['name']}' in {repo}") - print(f"Status Code: {res.status_code}") - print(f"Response: {res.text}") + else: + print(f"✅ Synced label '{label['name']}' to {repo}") def sync_labels(): labels = get_labels(SOURCE_REPO) - if not labels: - print("⚠️ No labels found or failed to fetch. Exiting.") - return - for target_repo in TARGET_REPOS: print(f"\n🔄 Syncing labels to {target_repo}") for label in labels: create_or_update_label(target_repo, label) if __name__ == "__main__": - sync_labels() + sync_labels() \ No newline at end of file diff --git a/scripts/sync_milestones.py b/scripts/sync_milestones.py new file mode 100644 index 0000000..d0e6d3d --- /dev/null +++ b/scripts/sync_milestones.py @@ -0,0 +1,28 @@ +import os +from github import Github + +TOKEN = os.getenv("GH_TOKEN") +g = Github(TOKEN) + +SOURCE_REPO = g.get_repo("JamesonRGrieve/Workflows") +TARGET_REPOS = [ + "AGInfrastructure", "AGInteractive", "AGInterface", "AGInYourPC", "AGIteration", + "nursegpt", "nursegpt_web", "ServerFramework", "auth", "zod2gql", "dynamic-form", "ClientFramework" +] +ORG = "JamesonRGrieve" + +source_milestones = SOURCE_REPO.get_milestones() + +for repo_name in TARGET_REPOS: + target_repo = g.get_repo(f"{ORG}/{repo_name}") + target_milestones = {m.title: m for m in target_repo.get_milestones(state="all")} + + for m in source_milestones: + if m.title not in target_milestones: + print(f"📅 Creating milestone {m.title} in {repo_name}") + target_repo.create_milestone( + title=m.title, + state=m.state, + description=m.description, + due_on=m.due_on + ) diff --git a/scripts/sync_temlates.py b/scripts/sync_temlates.py new file mode 100644 index 0000000..c773da0 --- /dev/null +++ b/scripts/sync_temlates.py @@ -0,0 +1,35 @@ +import os +import shutil +from github import Github + +TOKEN = os.getenv("GH_TOKEN") +g = Github(TOKEN) + +SOURCE_REPO = g.get_repo("JamesonRGrieve/Workflows") +TARGET_REPOS = [ + "AGInfrastructure", "AGInteractive", "AGInterface", "AGInYourPC", "AGIteration", + "nursegpt", "nursegpt_web", "ServerFramework", "auth", "zod2gql", "dynamic-form", "ClientFramework" +] +ORG = "JamesonRGrieve" + +TEMPLATE_DIRS = [".github/ISSUE_TEMPLATE", ".github/PULL_REQUEST_TEMPLATE"] + +for repo_name in TARGET_REPOS: + target_repo = g.get_repo(f"{ORG}/{repo_name}") + contents = target_repo.get_contents(".github") + existing_files = [c.path for c in contents] + + for dir_name in TEMPLATE_DIRS: + try: + source_files = SOURCE_REPO.get_contents(dir_name) + for file in source_files: + print(f"\n📁 Syncing {file.path} to {repo_name}") + content = file.decoded_content.decode("utf-8") + target_repo.create_file( + path=file.path, + message=f"sync: update template {file.name}", + content=content, + branch="main" + ) + except Exception as e: + print(f"⚠️ Skipping {dir_name} for {repo_name}: {e}") From b6c4be9ee224e62f28c9b488b3449c442aeec045 Mon Sep 17 00:00:00 2001 From: parull249 Date: Tue, 1 Jul 2025 18:48:16 +0530 Subject: [PATCH 31/44] added bandit reusable code and trigger --- .github/workflows/test-bandit.yml | 130 ++++++++++++++++++++++++++++++ triggers/python/on-pr-main.yml | 21 ++--- 2 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test-bandit.yml diff --git a/.github/workflows/test-bandit.yml b/.github/workflows/test-bandit.yml new file mode 100644 index 0000000..55e312d --- /dev/null +++ b/.github/workflows/test-bandit.yml @@ -0,0 +1,130 @@ +name: Reusable Bandit Security Check with Regression Detection + +# This reusable workflow is triggered by other workflows using 'workflow_call' +on: + workflow_call: + inputs: + target_branch_to_compare: + description: "Target branch to compare against (e.g., main)" + required: true + type: string + runs_on: + required: false + type: string + default: "ubuntu-latest" + outputs: + bandit_issues_json: + description: "JSON output of Bandit issues on PR branch" + value: ${{ jobs.run-bandit.outputs.bandit_issues_json }} + +jobs: + # Job 1: Run Bandit on the PR branch + run-bandit: + name: Run Bandit on PR Branch & Extract Results + runs-on: ${{ inputs.runs_on }} + outputs: + bandit_issues_json: ${{ steps.extract-pr.outputs.BANDIT_JSON }} + steps: + # Step 1: Checkout the current pull request code + - name: Checkout PR Branch + uses: actions/checkout@v4.1.1 + with: + persist-credentials: false + + # Step 2: Set up Python 3.10 environment + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + # Step 3: Install Bandit (Python security scanner) + - name: Install Bandit + run: pip install bandit + + # Step 4: Run Bandit and output results to a file + - name: Run Bandit on PR Branch + run: | + bandit -r . -f json -o pr_bandit_output.json || true + + # Step 5: Upload the results as a GitHub Actions artifact (for debugging or reporting) + - name: Upload PR Artifact + uses: actions/upload-artifact@v4 + with: + name: pr_bandit_output + path: pr_bandit_output.json + + # Step 6: Extract the raw issue list from the Bandit JSON output + - name: Extract PR Bandit JSON + id: extract-pr + run: | + CONTENT=$(cat pr_bandit_output.json | jq -c '.results') + echo "BANDIT_JSON=$CONTENT" >> $GITHUB_OUTPUT + + # Job 2: Run Bandit on the target branch for comparison + run-bandit-on-target: + name: Run Bandit on Target Branch + runs-on: ${{ inputs.runs_on }} + outputs: + bandit_target_json: ${{ steps.extract-target.outputs.TARGET_JSON }} + steps: + # Step 1: Checkout the base branch (e.g., main) + - name: Checkout Target Branch + uses: actions/checkout@v4 + with: + ref: ${{ inputs.target_branch_to_compare }} + persist-credentials: false + + # Step 2: Set up Python environment + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + # Step 3: Install Bandit + - name: Install Bandit + run: pip install bandit + + # Step 4: Run Bandit and save output + - name: Run Bandit on Target Branch + run: | + bandit -r . -f json -o target_bandit_output.json || true + + # Step 5: Upload results from the target branch + - name: Upload Target Artifact + uses: actions/upload-artifact@v4 + with: + name: target_bandit_output + path: target_bandit_output.json + + # Step 6: Extract raw issue list from the Bandit output + - name: Extract Target Bandit JSON + id: extract-target + run: | + CONTENT=$(cat target_bandit_output.json | jq -c '.results') + echo "TARGET_JSON=$CONTENT" >> $GITHUB_OUTPUT + + # Job 3: Compare the PR results against the target to detect regressions + compare-bandit: + name: Compare Bandit Issues (Regression Analysis) + runs-on: ${{ inputs.runs_on }} + needs: [run-bandit, run-bandit-on-target] + steps: + - name: Compare JSON + run: | + echo "Comparing Bandit results between PR and target branch..." + + echo "${{ needs.run-bandit.outputs.bandit_issues_json }}" > pr.json + echo "${{ needs.run-bandit-on-target.outputs.bandit_target_json }}" > target.json + + # Compare both JSON lists to find issues present in PR but not in target + NEW_ISSUES=$(jq -n --argfile pr pr.json --argfile base target.json ' + $pr - $base | length') + + echo "New security issues introduced: $NEW_ISSUES" + + if [ "$NEW_ISSUES" -gt 0 ]; then + echo "::error::New Bandit issues introduced in PR branch." + exit 1 + else + echo "No new security issues introduced." + fi diff --git a/triggers/python/on-pr-main.yml b/triggers/python/on-pr-main.yml index 05da9ee..4e0a57f 100644 --- a/triggers/python/on-pr-main.yml +++ b/triggers/python/on-pr-main.yml @@ -16,7 +16,6 @@ jobs: id: check_branch run: | IS_FROM_DEV="false" - # For pull requests, check if the source branch is dev if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.head_ref }}" == "dev" ]]; then IS_FROM_DEV="true" echo "This is a pull request coming from the dev branch into main." @@ -28,26 +27,30 @@ jobs: run-tests: name: Run Pytest Comparison - # No explicit check_pr_source dependency, tests should run for any PR to main. - # If tests pass and PR is from dev, then downstream will run. uses: JamesonRGrieve/Workflows/.github/workflows/test-pytest.yml@main with: - # PR is to main, so compare PR branch against main. - target_branch_to_compare: ${{ github.base_ref }} # This will be 'main' + target_branch_to_compare: ${{ github.base_ref }} permissions: - contents: write # For lint within pytest + contents: write + + run-bandit-check: + name: Run Bandit Regression Workflow + uses: JamesonRGrieve/Workflows/.github/workflows/test-bandit.yml@main + with: + target_branch_to_compare: ${{ github.base_ref }} # Usually 'main' in PRs to main + permissions: + contents: read call-reusable-stage-downstream: name: Call Reusable Stage Downstream Workflow - needs: [check-pr-source, run-tests] # Ensures tests pass and branch is correct before staging + needs: [check-pr-source, run-tests, run-bandit-check] if: needs.check-pr-source.outputs.is_from_dev == 'true' uses: JamesonRGrieve/Workflows/.github/workflows/test-downstream.yml@main with: - source_repo_ref: "dev" # Staging happens from the 'dev' branch of the current repo + source_repo_ref: "dev" secrets: PAT_TOKEN: ${{ secrets.PAT_TOKEN }} DOWNSTREAM_REPOSITORIES: ${{ secrets.DOWNSTREAM_REPOSITORIES }} permissions: contents: write pull-requests: write - # Permissions are handled by the reusable workflow itself. From bcb61756379583b598e5076beeb1f97f6777d38b Mon Sep 17 00:00:00 2001 From: parull249 Date: Tue, 1 Jul 2025 18:50:50 +0530 Subject: [PATCH 32/44] modified gh-projects-qa.yml --- .github/workflows/gh-projects-qa.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/gh-projects-qa.yml b/.github/workflows/gh-projects-qa.yml index 8333cfc..ac25117 100644 --- a/.github/workflows/gh-projects-qa.yml +++ b/.github/workflows/gh-projects-qa.yml @@ -2,9 +2,6 @@ name: Reusable - Set Q/A Status in Zephyrex Board on: workflow_call: - secrets: - GITHUB_TOKEN: - required: true jobs: qa-update-status: From a73f62ea13cdb78d672f176e7bbcdaf66401b58c Mon Sep 17 00:00:00 2001 From: parull249 Date: Wed, 2 Jul 2025 18:22:48 +0530 Subject: [PATCH 33/44] updated test-ts-lint.yml --- .github/workflows/test-ts-lint.yml | 2 -- scripts/{sync_temlates.py => sync_templates.py} | 0 2 files changed, 2 deletions(-) rename scripts/{sync_temlates.py => sync_templates.py} (100%) diff --git a/.github/workflows/test-ts-lint.yml b/.github/workflows/test-ts-lint.yml index f476956..3843e97 100644 --- a/.github/workflows/test-ts-lint.yml +++ b/.github/workflows/test-ts-lint.yml @@ -33,11 +33,9 @@ on: target_branch_artifact_name: required: false type: string - default: target_branch_data_${{ github.event.pull_request.number || github.run_id }} pr_branch_artifact_name: required: false type: string - default: pr_branch_data_${{ github.event.pull_request.number || github.run_id }} outputs: lint_errors_pr_branch: description: "JSON string of lint errors on the PR branch after attempting fixes." diff --git a/scripts/sync_temlates.py b/scripts/sync_templates.py similarity index 100% rename from scripts/sync_temlates.py rename to scripts/sync_templates.py From 0e9e0810e06837ec4dfcdb844103421a5c8fa580 Mon Sep 17 00:00:00 2001 From: parull249 Date: Wed, 2 Jul 2025 19:17:02 +0530 Subject: [PATCH 34/44] updated test-ts-lint.yml --- scripts/sync_branch_protection.py | 4 ++-- scripts/sync_labels.py | 8 ++++---- scripts/sync_milestones.py | 2 +- scripts/sync_templates.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/sync_branch_protection.py b/scripts/sync_branch_protection.py index 97c9db9..a1a19e2 100644 --- a/scripts/sync_branch_protection.py +++ b/scripts/sync_branch_protection.py @@ -32,8 +32,8 @@ repo = g.get_repo(f"{ORG}/{repo_name}") for branch in PROTECTED_BRANCHES: try: - print(f"🔐 Protecting {branch} in {repo_name}") + print(f"Protecting {branch} in {repo_name}") b = repo.get_branch(branch) b.edit_protection(**rules) except Exception as e: - print(f"⚠️ Error protecting {branch} in {repo_name}: {e}") + print(f"Error protecting {branch} in {repo_name}: {e}") diff --git a/scripts/sync_labels.py b/scripts/sync_labels.py index 97bc2bc..197dc24 100644 --- a/scripts/sync_labels.py +++ b/scripts/sync_labels.py @@ -36,16 +36,16 @@ def create_or_update_label(repo, label): "description": label.get("description", "") }) if res.status_code == 422 and "already_exists" in str(res.text): - print(f"⚠️ Label '{label['name']}' already exists in {repo}") + print(f"Label '{label['name']}' already exists in {repo}") elif res.status_code != 201: - print(f"❌ Failed to create label '{label['name']}' in {repo}") + print(f"Failed to create label '{label['name']}' in {repo}") else: - print(f"✅ Synced label '{label['name']}' to {repo}") + print(f"Synced label '{label['name']}' to {repo}") def sync_labels(): labels = get_labels(SOURCE_REPO) for target_repo in TARGET_REPOS: - print(f"\n🔄 Syncing labels to {target_repo}") + print(f"\nSyncing labels to {target_repo}") for label in labels: create_or_update_label(target_repo, label) diff --git a/scripts/sync_milestones.py b/scripts/sync_milestones.py index d0e6d3d..7ca97de 100644 --- a/scripts/sync_milestones.py +++ b/scripts/sync_milestones.py @@ -19,7 +19,7 @@ for m in source_milestones: if m.title not in target_milestones: - print(f"📅 Creating milestone {m.title} in {repo_name}") + print(f"Creating milestone {m.title} in {repo_name}") target_repo.create_milestone( title=m.title, state=m.state, diff --git a/scripts/sync_templates.py b/scripts/sync_templates.py index c773da0..815a8a2 100644 --- a/scripts/sync_templates.py +++ b/scripts/sync_templates.py @@ -23,7 +23,7 @@ try: source_files = SOURCE_REPO.get_contents(dir_name) for file in source_files: - print(f"\n📁 Syncing {file.path} to {repo_name}") + print(f"\nSyncing {file.path} to {repo_name}") content = file.decoded_content.decode("utf-8") target_repo.create_file( path=file.path, @@ -32,4 +32,4 @@ branch="main" ) except Exception as e: - print(f"⚠️ Skipping {dir_name} for {repo_name}: {e}") + print(f"Skipping {dir_name} for {repo_name}: {e}") From a9c2e0cf9dafbd36e8fae49de567aa949f055a06 Mon Sep 17 00:00:00 2001 From: parull249 Date: Wed, 2 Jul 2025 20:29:16 +0530 Subject: [PATCH 35/44] updated test-lint-py.yml --- triggers/python/on-push-other.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/triggers/python/on-push-other.yml b/triggers/python/on-push-other.yml index e956325..08e7d12 100644 --- a/triggers/python/on-push-other.yml +++ b/triggers/python/on-push-other.yml @@ -14,3 +14,5 @@ jobs: # Permissions are handled by the reusable workflow itself or should be inherited. # If secrets were needed, they would be passed here like: # secrets: inherit + permissions: + contents: write \ No newline at end of file From ff75c460258d176e112e5329f7a380c7f50197ca Mon Sep 17 00:00:00 2001 From: parull249 Date: Wed, 2 Jul 2025 20:43:00 +0530 Subject: [PATCH 36/44] updated test-lint-py.yml --- triggers/typescript/nextjs/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/triggers/typescript/nextjs/lint.yml b/triggers/typescript/nextjs/lint.yml index 96d5f47..160de32 100644 --- a/triggers/typescript/nextjs/lint.yml +++ b/triggers/typescript/nextjs/lint.yml @@ -15,4 +15,4 @@ jobs: # If secrets were needed, they would be passed here like: # secrets: inherit lint_command: npm run lint - fix_command: npm run lint -- --fix + fix_command: npm run lint --fix From e0bb1ff7f831e98df3d10d4d5d8c76062b56f61d Mon Sep 17 00:00:00 2001 From: parull249 Date: Wed, 2 Jul 2025 20:51:22 +0530 Subject: [PATCH 37/44] updated test-lint-py.yml --- triggers/typescript/nextjs/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/triggers/typescript/nextjs/lint.yml b/triggers/typescript/nextjs/lint.yml index 160de32..cd325f5 100644 --- a/triggers/typescript/nextjs/lint.yml +++ b/triggers/typescript/nextjs/lint.yml @@ -16,3 +16,4 @@ jobs: # secrets: inherit lint_command: npm run lint fix_command: npm run lint --fix + \ No newline at end of file From 623fa3c19010c45901fa21fb37ae316e41128b5c Mon Sep 17 00:00:00 2001 From: parull249 Date: Thu, 3 Jul 2025 13:53:55 +0530 Subject: [PATCH 38/44] modified permissions in on-pr-dev.yml --- triggers/python/on-pr-dev.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/triggers/python/on-pr-dev.yml b/triggers/python/on-pr-dev.yml index 7058ad7..78c8a89 100644 --- a/triggers/python/on-pr-dev.yml +++ b/triggers/python/on-pr-dev.yml @@ -7,6 +7,9 @@ on: branches: - dev +permissions: + contents: write + jobs: debug-pr-context: # New job for debugging name: Debug PR Context From 606ab19849d5c73acdfe5aef68994758755bb039 Mon Sep 17 00:00:00 2001 From: parull249 <103111355+parull249@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:10:16 +0530 Subject: [PATCH 39/44] Blocked Issues and Tracking Date (#2) Co-authored-by: shankar --- .github/workflows/gh-projects-qa.yml | 4 +- .github/workflows/issue-blocker-reusuable.yml | 87 +++++++++++++++++++ .github/workflows/tracking-date-reusuable.yml | 79 +++++++++++++++++ triggers/python/on-pr-main.yml | 2 +- triggers/python/on-push-other.yml | 2 + triggers/typescript/nextjs/lint.yml | 2 +- .../typescript/nextjs/manage-submodules.yml | 2 +- .../typescript/nextjs/test-downstream.yml | 4 +- 8 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/issue-blocker-reusuable.yml create mode 100644 .github/workflows/tracking-date-reusuable.yml diff --git a/.github/workflows/gh-projects-qa.yml b/.github/workflows/gh-projects-qa.yml index 8333cfc..98da393 100644 --- a/.github/workflows/gh-projects-qa.yml +++ b/.github/workflows/gh-projects-qa.yml @@ -2,9 +2,7 @@ name: Reusable - Set Q/A Status in Zephyrex Board on: workflow_call: - secrets: - GITHUB_TOKEN: - required: true + jobs: qa-update-status: diff --git a/.github/workflows/issue-blocker-reusuable.yml b/.github/workflows/issue-blocker-reusuable.yml new file mode 100644 index 0000000..2fab33c --- /dev/null +++ b/.github/workflows/issue-blocker-reusuable.yml @@ -0,0 +1,87 @@ +# .github/workflows/issue-blocker.yml + +name: "Reusable Issue Blocker Workflow" + +on: + workflow_dispatch: + workflow_call: + +permissions: + issues: write + +jobs: + clean-blocked-comments: + runs-on: ubuntu-latest + steps: + - name: Clean up 'Blocked by' references and manage issue status + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const regex = /#(\d+)/g; + + const issues = await github.rest.issues.listForRepo({ owner, repo, state: 'open' }); + + for (const issue of issues.data) { + if (issue.pull_request) continue; + + console.log(`\n🔍 Issue #${issue.number}: ${issue.title}`); + + const comments = await github.rest.issues.listComments({ owner, repo, issue_number: issue.number }); + let updated = false; + + for (const comment of comments.data) { + if (!comment.body.toLowerCase().includes('blocked by')) continue; + + console.log(`💬 Comment ID: ${comment.id}`); + console.log(`📄 Raw: ${JSON.stringify(comment.body)}`); + + const matches = [...comment.body.matchAll(regex)].map(m => parseInt(m[1])); + const stillOpen = []; + + for (const ref of matches) { + try { + const refIssue = await github.rest.issues.get({ owner, repo, issue_number: ref }); + if (refIssue.data.state === 'open') { + console.log(`🔗 Found reference to issue #${ref}\n⏳ Issue #${ref} is still open.`); + stillOpen.push(`#${ref}`); + } else { + console.log(`🔗 Found reference to issue #${ref}\n✅ Issue #${ref} is closed.`); + } + } catch (err) { + console.warn(`⚠️ Issue #${ref} could not be fetched.`); + } + } + + const newBody = stillOpen.length > 0 + ? `Blocked by ${stillOpen.join(' ')}` + : '[Auto-removed blocked-by reference: all blocking issues closed.]'; + + if (comment.body !== newBody) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: comment.id, + body: newBody + }); + updated = true; + console.log(`✏️ Comment updated to: "${newBody}"`); + } else { + console.log(`✅ Comment is already correct. No update needed.`); + } + } + + // Manage issue status using labels + const labels = issue.labels.map(l => l.name); + const hasBlocked = comments.data.some(c => c.body.includes('Blocked by #') && !c.body.includes('Auto-removed')); + + if (hasBlocked && !labels.includes('Blocked')) { + await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: ['Blocked'] }); + console.log(`🏷️ Label 'Blocked' added.`); + } else if (!hasBlocked && labels.includes('Blocked')) { + await github.rest.issues.setLabels({ owner, repo, issue_number: issue.number, labels: ['To-Do'] }); + console.log(`🚫 Label 'To-Do' added.`); + } + } \ No newline at end of file diff --git a/.github/workflows/tracking-date-reusuable.yml b/.github/workflows/tracking-date-reusuable.yml new file mode 100644 index 0000000..b4ed89e --- /dev/null +++ b/.github/workflows/tracking-date-reusuable.yml @@ -0,0 +1,79 @@ +name: Auto Date Tracking for All Issues + +on: + workflow_dispatch: + +permissions: + issues: write + contents: read + +jobs: + auto-date-tracking: + runs-on: ubuntu-latest + steps: + - name: Set Start and Completion Dates on All Issues + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const now = new Date().toISOString().split("T")[0]; + + const issues = await github.paginate( + github.rest.issues.listForRepo, + { + owner, + repo, + state: "all", + per_page: 100 + } + ); + + for (const issue of issues) { + if (issue.pull_request) continue; // skip PRs + + const issue_number = issue.number; + const labels = issue.labels.map(label => label.name); + const startDateComment = `🕒 **Start Date:** ${now}`; + const completeDateComment = `✅ **Completion Date:** ${now}`; + + const comments = await github.rest.issues.listComments({ + owner, + repo, + issue_number + }); + + const alreadyHasComment = (bodyText) => + comments.data.some(c => + c.body.trim() === bodyText.trim() && + c.user?.type === "Bot" && + c.user?.login === "github-actions[bot]" + ); + + // 🕒 Add Start Date if labeled "In Progress" + if (labels.includes("In Progress") && !alreadyHasComment(startDateComment)) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body: startDateComment + }); + console.log(`✅ Start Date set on issue #${issue_number}`); + } + + // ✅ Add Completion Date if labeled "Complete" and issue is closed + if ( + issue.state === "closed" && !alreadyHasComment(completeDateComment) + ) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body: completeDateComment + }); + console.log(`✅ Completion Date set on issue #${issue_number}`); + } + } + + console.log("🎉 Finished processing all issues."); \ No newline at end of file diff --git a/triggers/python/on-pr-main.yml b/triggers/python/on-pr-main.yml index 05da9ee..74ceae3 100644 --- a/triggers/python/on-pr-main.yml +++ b/triggers/python/on-pr-main.yml @@ -39,7 +39,7 @@ jobs: call-reusable-stage-downstream: name: Call Reusable Stage Downstream Workflow - needs: [check-pr-source, run-tests] # Ensures tests pass and branch is correct before staging + needs: [check-pr-source, run-tests, run-bandit-check] # Ensures tests pass and branch is correct before staging if: needs.check-pr-source.outputs.is_from_dev == 'true' uses: JamesonRGrieve/Workflows/.github/workflows/test-downstream.yml@main with: diff --git a/triggers/python/on-push-other.yml b/triggers/python/on-push-other.yml index e956325..7c97c3b 100644 --- a/triggers/python/on-push-other.yml +++ b/triggers/python/on-push-other.yml @@ -14,3 +14,5 @@ jobs: # Permissions are handled by the reusable workflow itself or should be inherited. # If secrets were needed, they would be passed here like: # secrets: inherit + permissions: + contents: write diff --git a/triggers/typescript/nextjs/lint.yml b/triggers/typescript/nextjs/lint.yml index 1ea4902..6267b90 100644 --- a/triggers/typescript/nextjs/lint.yml +++ b/triggers/typescript/nextjs/lint.yml @@ -9,7 +9,7 @@ on: jobs: call-reusable-lint: name: Call Reusable Lint Workflow - uses: JamesonRGrieve/Workflows/.github/workflows/test-lint-ts.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/test-ts-lint.yml@main # No specific inputs needed for the reusable lint workflow # Permissions are handled by the reusable workflow itself or should be inherited. # If secrets were needed, they would be passed here like: diff --git a/triggers/typescript/nextjs/manage-submodules.yml b/triggers/typescript/nextjs/manage-submodules.yml index 5008bc0..cc1436c 100644 --- a/triggers/typescript/nextjs/manage-submodules.yml +++ b/triggers/typescript/nextjs/manage-submodules.yml @@ -11,7 +11,7 @@ jobs: name: Call Reusable Submodule Management # Make sure the reusable workflow is in the same repository at this path # or use //.github/workflows/manage-submodules.yml@ if it's in a central repo. - uses: JamesonRGrieve/Workflows/.github/workflows/manage-submodules.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/gh-submodules.yml@main with: # REQUIRED: Customize this list of comma-separated file paths # These are the files that will be synced from the main repository to the submodules diff --git a/triggers/typescript/nextjs/test-downstream.yml b/triggers/typescript/nextjs/test-downstream.yml index e9df331..5d69c14 100644 --- a/triggers/typescript/nextjs/test-downstream.yml +++ b/triggers/typescript/nextjs/test-downstream.yml @@ -30,7 +30,7 @@ jobs: name: Run Pytest Comparison # No explicit check_pr_source dependency, tests should run for any PR to main. # If tests pass and PR is from dev, then downstream will run. - uses: JamesonRGrieve/Workflows/.github/workflows/test-pytest.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/test-py-pytest.yml@main with: # PR is to main, so compare PR branch against main. target_branch_to_compare: ${{ github.base_ref }} # This will be 'main' @@ -41,7 +41,7 @@ jobs: name: Call Reusable Stage Downstream Workflow needs: [check-pr-source, run-tests] # Ensures tests pass and branch is correct before staging if: needs.check-pr-source.outputs.is_from_dev == 'true' - uses: JamesonRGrieve/Workflows/.github/workflows/test-downstream.yml@main + uses: JamesonRGrieve/Workflows/.github/workflows/gh-test-downstream.yml@main with: source_repo_ref: 'dev' # Staging happens from the 'dev' branch of the current repo secrets: From 5f136f6710f35ccf4cf34423a46d24a81dadf842 Mon Sep 17 00:00:00 2001 From: parull249 Date: Thu, 7 Aug 2025 12:41:05 +0000 Subject: [PATCH 40/44] adjust severity filtering for Bandit to only include HIGH severity issues --- .github/workflows/test-bandit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bandit.yml b/.github/workflows/test-bandit.yml index 55e312d..e67951d 100644 --- a/.github/workflows/test-bandit.yml +++ b/.github/workflows/test-bandit.yml @@ -44,7 +44,7 @@ jobs: # Step 4: Run Bandit and output results to a file - name: Run Bandit on PR Branch run: | - bandit -r . -f json -o pr_bandit_output.json || true + bandit -r . -lll -f json -o pr_bandit_output.json || true # Step 5: Upload the results as a GitHub Actions artifact (for debugging or reporting) - name: Upload PR Artifact @@ -87,7 +87,7 @@ jobs: # Step 4: Run Bandit and save output - name: Run Bandit on Target Branch run: | - bandit -r . -f json -o target_bandit_output.json || true + bandit -r . -lll -f json -o target_bandit_output.json || true # Step 5: Upload results from the target branch - name: Upload Target Artifact From 16a1847aa50e7c8728e3477f9416258bcc07ff6f Mon Sep 17 00:00:00 2001 From: parull249 Date: Fri, 8 Aug 2025 09:01:45 +0000 Subject: [PATCH 41/44] resolved conflicts --- scripts/sync_branch_protection.py | 4 ++-- scripts/sync_labels.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/sync_branch_protection.py b/scripts/sync_branch_protection.py index a1a19e2..1bdf806 100644 --- a/scripts/sync_branch_protection.py +++ b/scripts/sync_branch_protection.py @@ -32,8 +32,8 @@ repo = g.get_repo(f"{ORG}/{repo_name}") for branch in PROTECTED_BRANCHES: try: - print(f"Protecting {branch} in {repo_name}") + print(f" Protecting {branch} in {repo_name}") b = repo.get_branch(branch) b.edit_protection(**rules) except Exception as e: - print(f"Error protecting {branch} in {repo_name}: {e}") + print(f" Error protecting {branch} in {repo_name}: {e}") diff --git a/scripts/sync_labels.py b/scripts/sync_labels.py index 197dc24..ce4a78a 100644 --- a/scripts/sync_labels.py +++ b/scripts/sync_labels.py @@ -25,7 +25,7 @@ def get_labels(repo): res = requests.get(url, headers=HEADERS) if res.status_code == 200: return res.json() - print(f"❌ Failed to fetch labels from {repo}") + print(f" Failed to fetch labels from {repo}") return [] def create_or_update_label(repo, label): @@ -36,11 +36,11 @@ def create_or_update_label(repo, label): "description": label.get("description", "") }) if res.status_code == 422 and "already_exists" in str(res.text): - print(f"Label '{label['name']}' already exists in {repo}") + print(f" Label '{label['name']}' already exists in {repo}") elif res.status_code != 201: - print(f"Failed to create label '{label['name']}' in {repo}") + print(f" Failed to create label '{label['name']}' in {repo}") else: - print(f"Synced label '{label['name']}' to {repo}") + print(f" Synced label '{label['name']}' to {repo}") def sync_labels(): labels = get_labels(SOURCE_REPO) From 29b0b104a03db22d7aa54365c42c592481b2dd39 Mon Sep 17 00:00:00 2001 From: parull249 Date: Fri, 8 Aug 2025 09:03:33 +0000 Subject: [PATCH 42/44] resolved conflicts --- .github/workflows/test-bandit.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-bandit.yml b/.github/workflows/test-bandit.yml index e67951d..23f2f2a 100644 --- a/.github/workflows/test-bandit.yml +++ b/.github/workflows/test-bandit.yml @@ -87,7 +87,11 @@ jobs: # Step 4: Run Bandit and save output - name: Run Bandit on Target Branch run: | +<<<<<<< HEAD bandit -r . -lll -f json -o target_bandit_output.json || true +======= + bandit -r . -f json -o target_bandit_output.json || true +>>>>>>> b6c4be9 (added bandit reusable code and trigger) # Step 5: Upload results from the target branch - name: Upload Target Artifact From c63ac30269ce56290a2f208be6b2f4f7ee6af375 Mon Sep 17 00:00:00 2001 From: parull249 Date: Fri, 8 Aug 2025 09:06:54 +0000 Subject: [PATCH 43/44] resolved conflicts --- triggers/typescript/nextjs/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/triggers/typescript/nextjs/lint.yml b/triggers/typescript/nextjs/lint.yml index cd325f5..488a780 100644 --- a/triggers/typescript/nextjs/lint.yml +++ b/triggers/typescript/nextjs/lint.yml @@ -16,4 +16,4 @@ jobs: # secrets: inherit lint_command: npm run lint fix_command: npm run lint --fix - \ No newline at end of file + From d5c91f0ff3060d40f725fe78eeb7818b785fb5db Mon Sep 17 00:00:00 2001 From: parull249 Date: Fri, 8 Aug 2025 09:09:00 +0000 Subject: [PATCH 44/44] resolved conflicts --- .github/workflows/test-bandit.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test-bandit.yml b/.github/workflows/test-bandit.yml index 23f2f2a..e67951d 100644 --- a/.github/workflows/test-bandit.yml +++ b/.github/workflows/test-bandit.yml @@ -87,11 +87,7 @@ jobs: # Step 4: Run Bandit and save output - name: Run Bandit on Target Branch run: | -<<<<<<< HEAD bandit -r . -lll -f json -o target_bandit_output.json || true -======= - bandit -r . -f json -o target_bandit_output.json || true ->>>>>>> b6c4be9 (added bandit reusable code and trigger) # Step 5: Upload results from the target branch - name: Upload Target Artifact