|
| 1 | +/* |
| 2 | + * Wrap the "composer run wp-tests-phpunit" command to process tests |
| 3 | + * that are expected to error and fail at the moment. |
| 4 | + * |
| 5 | + * This makes sure that the CI job passes, while explicitly tracking |
| 6 | + * the issues that need to be addressed. Ideally, over time this script |
| 7 | + * will become obsolete when all errors and failures are resolved. |
| 8 | + */ |
| 9 | +const { execSync } = require( 'child_process' ); |
| 10 | +const fs = require( 'fs' ); |
| 11 | +const path = require( 'path' ); |
| 12 | + |
| 13 | +const expectedErrors = [ |
| 14 | + 'Tests_DB_Charset::test_invalid_characters_in_query', |
| 15 | + 'Tests_DB_Charset::test_set_charset_changes_the_connection_collation', |
| 16 | + 'Tests_DB::test_get_col_info', |
| 17 | +]; |
| 18 | + |
| 19 | +const expectedFailures = [ |
| 20 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #2', |
| 21 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #3', |
| 22 | + 'Tests_Comment::test_wp_new_comment_respects_comment_field_lengths', |
| 23 | + 'Tests_Comment::test_wp_update_comment', |
| 24 | + 'Tests_DB_Charset::test_get_column_charset with data set #0', |
| 25 | + 'Tests_DB_Charset::test_get_column_charset with data set #1', |
| 26 | + 'Tests_DB_Charset::test_get_column_charset with data set #2', |
| 27 | + 'Tests_DB_Charset::test_get_column_charset with data set #3', |
| 28 | + 'Tests_DB_Charset::test_get_column_charset with data set #4', |
| 29 | + 'Tests_DB_Charset::test_get_column_charset with data set #5', |
| 30 | + 'Tests_DB_Charset::test_get_column_charset with data set #6', |
| 31 | + 'Tests_DB_Charset::test_get_column_charset with data set #7', |
| 32 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #0', |
| 33 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #1', |
| 34 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #2', |
| 35 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #3', |
| 36 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #4', |
| 37 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #5', |
| 38 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #6', |
| 39 | + 'Tests_DB_Charset::test_get_column_charset_is_mysql_undefined with data set #7', |
| 40 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #0', |
| 41 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #1', |
| 42 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #2', |
| 43 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #3', |
| 44 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #4', |
| 45 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #5', |
| 46 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #6', |
| 47 | + 'Tests_DB_Charset::test_get_column_charset_non_mysql with data set #7', |
| 48 | + 'Tests_DB_Charset::test_get_table_charset with data set #1', |
| 49 | + 'Tests_DB_Charset::test_get_table_charset with data set #4', |
| 50 | + 'Tests_DB_Charset::test_get_table_charset with data set #5', |
| 51 | + 'Tests_DB_Charset::test_get_table_charset with data set #6', |
| 52 | + 'Tests_DB_Charset::test_get_table_charset with data set #7', |
| 53 | + 'Tests_DB_Charset::test_process_field_charsets_on_nonexistent_table', |
| 54 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #21', |
| 55 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #22', |
| 56 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #23', |
| 57 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #24', |
| 58 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #25', |
| 59 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #26', |
| 60 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #27', |
| 61 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #28', |
| 62 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #30', |
| 63 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #31', |
| 64 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #32', |
| 65 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #33', |
| 66 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #34', |
| 67 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #35', |
| 68 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #36', |
| 69 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #37', |
| 70 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #39', |
| 71 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #40', |
| 72 | + 'Tests_DB_Charset::test_strip_invalid_text with data set #41', |
| 73 | + 'Tests_DB_Charset::test_strip_invalid_text_for_column_bails_if_ascii_input_too_long', |
| 74 | + 'Tests_DB_Charset::test_strip_invalid_text_from_query with data set "utf8 + utf8mb4"', |
| 75 | + 'Tests_DB_Charset::test_table_collation_check with data set "utf8_bin + big5_chinese_ci"', |
| 76 | + 'Tests_DB_Charset::test_table_collation_check with data set "utf8_unicode_ci"', |
| 77 | + 'Tests_DB_dbDelta::test_spatial_indices', |
| 78 | + 'Tests_DB::test_charset_switched_to_utf8mb4', |
| 79 | + 'Tests_DB::test_close', |
| 80 | + 'Tests_DB::test_delete_value_too_long_for_field with data set "too long"', |
| 81 | + 'Tests_DB::test_has_cap', |
| 82 | + 'Tests_DB::test_insert_value_too_long_for_field with data set "too long"', |
| 83 | + 'Tests_DB::test_mysqli_flush_sync', |
| 84 | + 'Tests_DB::test_non_unicode_collations', |
| 85 | + 'Tests_DB::test_pre_get_col_charset_filter', |
| 86 | + 'Tests_DB::test_process_fields_on_nonexistent_table', |
| 87 | + 'Tests_DB::test_process_fields_value_too_long_for_field with data set "too long"', |
| 88 | + 'Tests_DB::test_query_value_contains_invalid_chars', |
| 89 | + 'Tests_DB::test_replace_value_too_long_for_field with data set "too long"', |
| 90 | + 'Tests_DB::test_replace', |
| 91 | + 'Tests_DB::test_supports_collation', |
| 92 | + 'Tests_DB::test_update_value_too_long_for_field with data set "too long"', |
| 93 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #1', |
| 94 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #2', |
| 95 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #3', |
| 96 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #4', |
| 97 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #5', |
| 98 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #6', |
| 99 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #7', |
| 100 | + 'Tests_Menu_wpNavMenu::test_wp_nav_menu_should_not_have_has_children_class_with_custom_depth', |
| 101 | + 'WP_Test_REST_Posts_Controller::test_get_items_orderby_modified_query', |
| 102 | +]; |
| 103 | + |
| 104 | +console.log( 'Running WordPress PHPUnit tests with expected failures tracking...' ); |
| 105 | +console.log( 'Expected errors:', expectedErrors ); |
| 106 | +console.log( 'Expected failures:', expectedFailures ); |
| 107 | + |
| 108 | +try { |
| 109 | + try { |
| 110 | + execSync( |
| 111 | + `composer run wp-test-phpunit -- --log-junit=phpunit-results.xml --verbose`, |
| 112 | + { stdio: 'inherit' } |
| 113 | + ); |
| 114 | + console.log( '\n⚠️ All tests passed, checking if expected errors/failures occurred...' ); |
| 115 | + } catch ( error ) { |
| 116 | + console.log( '\n⚠️ Some tests errored/failed (expected). Analyzing results...' ); |
| 117 | + } |
| 118 | + |
| 119 | + // Read the JUnit XML test output: |
| 120 | + const junitOutputFile = path.join( __dirname, '..', '..', 'wordpress', 'phpunit-results.xml' ); |
| 121 | + if ( ! fs.existsSync( junitOutputFile ) ) { |
| 122 | + console.error( 'Error: JUnit output file not found!' ); |
| 123 | + process.exit( 1 ); |
| 124 | + } |
| 125 | + const junitXml = fs.readFileSync( junitOutputFile, 'utf8' ); |
| 126 | + |
| 127 | + // Extract test info from the XML: |
| 128 | + const actualErrors = []; |
| 129 | + const actualFailures = []; |
| 130 | + for ( const testcase of junitXml.matchAll( /<testcase([^>]*)\/>|<testcase([^>]*)>([\s\S]*?)<\/testcase>/g ) ) { |
| 131 | + const attributes = {}; |
| 132 | + const attributesString = testcase[2] ?? testcase[1]; |
| 133 | + for ( const attribute of attributesString.matchAll( /(\w+)="([^"]*)"/g ) ) { |
| 134 | + attributes[attribute[1]] = attribute[2]; |
| 135 | + } |
| 136 | + |
| 137 | + const content = testcase[3] ?? ''; |
| 138 | + const fqn = attributes.class ? `${attributes.class}::${attributes.name}` : attributes.name; |
| 139 | + const hasError = content.includes( '<error' ); |
| 140 | + const hasFailure = content.includes( '<failure' ); |
| 141 | + |
| 142 | + if ( hasError ) { |
| 143 | + actualErrors.push( fqn ); |
| 144 | + } |
| 145 | + |
| 146 | + if ( hasFailure ) { |
| 147 | + actualFailures.push( fqn ); |
| 148 | + } |
| 149 | + } |
| 150 | + |
| 151 | + let isSuccess = true; |
| 152 | + |
| 153 | + // Check if all expected errors actually errored |
| 154 | + const unexpectedNonErrors = expectedErrors.filter( test => ! actualErrors.includes( test ) ); |
| 155 | + if ( unexpectedNonErrors.length > 0 ) { |
| 156 | + console.error( '\n❌ The following tests were expected to error but did not:' ); |
| 157 | + unexpectedNonErrors.forEach( test => console.error( ` - ${test}` ) ); |
| 158 | + isSuccess = false; |
| 159 | + } |
| 160 | + |
| 161 | + // Check if all expected failures actually failed |
| 162 | + const unexpectedPasses = expectedFailures.filter( test => ! actualFailures.includes( test ) ); |
| 163 | + if ( unexpectedPasses.length > 0 ) { |
| 164 | + console.error( '\n❌ The following tests were expected to fail but passed:' ); |
| 165 | + unexpectedPasses.forEach( test => console.error( ` - ${test}` ) ); |
| 166 | + isSuccess = false; |
| 167 | + } |
| 168 | + |
| 169 | + // Check for unexpected errors |
| 170 | + const unexpectedErrors = actualErrors.filter( test => ! expectedErrors.includes( test ) ); |
| 171 | + if ( unexpectedErrors.length > 0 ) { |
| 172 | + console.error( '\n❌ The following tests errored unexpectedly:' ); |
| 173 | + unexpectedErrors.forEach( test => console.error( ` - ${test}` ) ); |
| 174 | + isSuccess = false; |
| 175 | + } |
| 176 | + |
| 177 | + // Check for unexpected failures |
| 178 | + const unexpectedFailures = actualFailures.filter( test => ! expectedFailures.includes( test ) ); |
| 179 | + if ( unexpectedFailures.length > 0 ) { |
| 180 | + console.error( '\n❌ The following tests failed unexpectedly:' ); |
| 181 | + unexpectedFailures.forEach( test => console.error( ` - ${test}` ) ); |
| 182 | + isSuccess = false; |
| 183 | + } |
| 184 | + |
| 185 | + if ( isSuccess ) { |
| 186 | + console.log( '\n✅ All tests behaved as expected!' ); |
| 187 | + process.exit( 0 ); |
| 188 | + } else { |
| 189 | + console.log( '\n❌ Some tests did not behave as expected!' ); |
| 190 | + process.exit( 1 ); |
| 191 | + } |
| 192 | +} catch ( error ) { |
| 193 | + console.error( '\n❌ Script execution error:', error.message ); |
| 194 | + process.exit( 1 ); |
| 195 | +} |
0 commit comments