From 11d28c35bdcab1dcf2db79d24f0e478fe6d99f2b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 17 Sep 2025 08:36:06 +0000 Subject: [PATCH 1/4] Initial plan From 14be414c939079c8603f21f648661cf05211b01b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 17 Sep 2025 08:41:07 +0000 Subject: [PATCH 2/4] Add noindex directive checking to prevent submission of noindex URLs Co-authored-by: pingal <18268276+pingal@users.noreply.github.com> --- .../class-indexnow-url-submission-admin.php | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/public/admin/class-indexnow-url-submission-admin.php b/public/admin/class-indexnow-url-submission-admin.php index d80281a..0af468d 100644 --- a/public/admin/class-indexnow-url-submission-admin.php +++ b/public/admin/class-indexnow-url-submission-admin.php @@ -139,6 +139,67 @@ public function add_action_links($links) return array_merge($settings_link, $links); } + /** + * Check if a post/page has noindex directive set. + * + * @param WP_Post $post The post object to check + * @param string $url The URL to check for noindex headers + * @return bool True if noindex is set, false otherwise + */ + private function has_noindex_directive($post, $url) { + // Check if WordPress site-wide search engine visibility is disabled + if (get_option('blog_public') == '0') { + if (true === WP_DEBUG && true === WP_DEBUG_LOG) { + error_log(__METHOD__ . " Site-wide search engine discouragement enabled"); + } + return true; + } + + // Check post meta for noindex (common SEO plugins use this) + $meta_robots = get_post_meta($post->ID, '_yoast_wpseo_meta-robots-noindex', true); + if ($meta_robots === '1') { + if (true === WP_DEBUG && true === WP_DEBUG_LOG) { + error_log(__METHOD__ . " Yoast noindex meta detected for post ID: " . $post->ID); + } + return true; + } + + // Check RankMath noindex meta + $rankmath_robots = get_post_meta($post->ID, 'rank_math_robots', true); + if (is_array($rankmath_robots) && in_array('noindex', $rankmath_robots)) { + if (true === WP_DEBUG && true === WP_DEBUG_LOG) { + error_log(__METHOD__ . " RankMath noindex meta detected for post ID: " . $post->ID); + } + return true; + } + + // Check All in One SEO meta + $aioseo_robots = get_post_meta($post->ID, '_aioseo_robots_noindex', true); + if ($aioseo_robots === '1') { + if (true === WP_DEBUG && true === WP_DEBUG_LOG) { + error_log(__METHOD__ . " All in One SEO noindex meta detected for post ID: " . $post->ID); + } + return true; + } + + // Check HTTP headers for X-Robots-Tag noindex + $response = wp_safe_remote_head($url); + if (!is_wp_error($response)) { + $headers = wp_remote_retrieve_headers($response); + if (isset($headers['x-robots-tag'])) { + $robots_header = strtolower($headers['x-robots-tag']); + if (strpos($robots_header, 'noindex') !== false) { + if (true === WP_DEBUG && true === WP_DEBUG_LOG) { + error_log(__METHOD__ . " X-Robots-Tag noindex header detected for URL: " . $url); + } + return true; + } + } + } + + return false; + } + // This function checks the type of update on a page/post and accordingly calls the submit api if enabled public function on_post_published($new_status, $old_status, $post) { @@ -193,6 +254,14 @@ public function on_post_published($new_status, $old_status, $post) } } + // Check for noindex directive - don't submit URLs that should not be indexed + if ($type != 'delete' && $this->has_noindex_directive($post, $link)) { + if (true === WP_DEBUG && true === WP_DEBUG_LOG) { + error_log(__METHOD__ . " Skipping URL submission due to noindex directive: " . $link); + } + return; + } + $siteUrl = get_home_url(); // check if same url was submitted recently(within a minute) From 13e9b01b49273fe721ac341ccc2fe7cce9e36fdf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 17 Sep 2025 08:44:24 +0000 Subject: [PATCH 3/4] Improve noindex detection with better error handling and timeout Co-authored-by: pingal <18268276+pingal@users.noreply.github.com> --- .../class-indexnow-url-submission-admin.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/public/admin/class-indexnow-url-submission-admin.php b/public/admin/class-indexnow-url-submission-admin.php index 0af468d..110f81c 100644 --- a/public/admin/class-indexnow-url-submission-admin.php +++ b/public/admin/class-indexnow-url-submission-admin.php @@ -183,11 +183,21 @@ private function has_noindex_directive($post, $url) { } // Check HTTP headers for X-Robots-Tag noindex - $response = wp_safe_remote_head($url); + // Use a short timeout to avoid delays in post publishing + $response = wp_safe_remote_head($url, array('timeout' => 5)); if (!is_wp_error($response)) { $headers = wp_remote_retrieve_headers($response); + // Check both 'x-robots-tag' and 'X-Robots-Tag' (case insensitive) + $robots_header = ''; if (isset($headers['x-robots-tag'])) { - $robots_header = strtolower($headers['x-robots-tag']); + $robots_header = $headers['x-robots-tag']; + } elseif (isset($headers['X-Robots-Tag'])) { + $robots_header = $headers['X-Robots-Tag']; + } + + if (!empty($robots_header)) { + $robots_header = strtolower($robots_header); + // Check for noindex in various formats if (strpos($robots_header, 'noindex') !== false) { if (true === WP_DEBUG && true === WP_DEBUG_LOG) { error_log(__METHOD__ . " X-Robots-Tag noindex header detected for URL: " . $url); @@ -195,6 +205,8 @@ private function has_noindex_directive($post, $url) { return true; } } + } elseif (true === WP_DEBUG && true === WP_DEBUG_LOG) { + error_log(__METHOD__ . " Could not check X-Robots-Tag header for URL: " . $url . " - " . $response->get_error_message()); } return false; From 07e140bcc842418929a440fc7aafe3ee70d41ecc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 17 Sep 2025 08:46:38 +0000 Subject: [PATCH 4/4] Update plugin version and documentation for noindex feature Co-authored-by: pingal <18268276+pingal@users.noreply.github.com> --- public/indexnow-url-submission.php | 4 ++-- public/readme.txt | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/public/indexnow-url-submission.php b/public/indexnow-url-submission.php index 218b3cc..310538f 100644 --- a/public/indexnow-url-submission.php +++ b/public/indexnow-url-submission.php @@ -9,7 +9,7 @@ * Plugin Name: IndexNow * Plugin URI: https://www.bing.com/webmaster * Description: A small plugin to allow Url submissions to IndexNow. - * Version: 1.0.1 + * Version: 1.0.2 * Author: Microsoft Bing * Author URI: https://www.bing.com/indexnow * License: GPL-2.0+ @@ -27,7 +27,7 @@ * Currently plugin version. */ -define( 'BWT_INDEXNOW_PLUGIN_VERSION', '1.0.1' ); +define( 'BWT_INDEXNOW_PLUGIN_VERSION', '1.0.2' ); /** * Plugin name. diff --git a/public/readme.txt b/public/readme.txt index 7acda15..8bb0370 100644 --- a/public/readme.txt +++ b/public/readme.txt @@ -17,6 +17,8 @@ pages to supporting search engines. IndexNow Plugin for WordPress enables automated submission of URLs from WordPress sites to the multiple search engines without the need to register and verify your site with them. Once installed, the plugin will automatically generate and host the API key on your site. It detects page creation/update/ deletion in WordPress and automatically submits the URLs in the background. This ensures that search engines will always have the latest updates about your site. +The plugin automatically respects noindex directives to prevent submission of URLs that should not be indexed, including WordPress site-wide search engine visibility settings, SEO plugin noindex settings (Yoast, RankMath, All in One SEO), and X-Robots-Tag headers. + Some other handy features included in the plugin: * Toggle the automatic submission feature. @@ -61,6 +63,14 @@ Indexing of URLs is specific and dependent on each search engine’s rules, plea == Changelog == = 1.0.2 = +* Add noindex directive detection to prevent submission of URLs that should not be indexed. +* Support for WordPress site-wide search engine visibility settings. +* Support for popular SEO plugins (Yoast, RankMath, All in One SEO) noindex meta tags. +* Support for X-Robots-Tag HTTP headers. +* Improved logging for debugging noindex detection. +* Allow plugin owners of WordPress site to access Indexing Insights in Bing Webmaster tools. + += 1.0.1 = * Allow plugin owners of WordPress site to access Indexing Insights in Bing Webmaster tools. = 1.0.1 =