Add noindex directive detection to prevent submission of URLs that should not be indexed

public/admin/class-indexnow-url-submission-admin.php

@@ -139,6 +139,79 @@ public function add_action_links($links)
 		return array_merge($settings_link, $links);
 	}
 
+	/**
+	 * Check if a post/page has noindex directive set.
+	 * 
+	 * @param WP_Post $post The post object to check
+	 * @param string $url The URL to check for noindex headers
+	 * @return bool True if noindex is set, false otherwise
+	 */
+	private function has_noindex_directive($post, $url) {
+		// Check if WordPress site-wide search engine visibility is disabled
+		if (get_option('blog_public') == '0') {
+			if (true === WP_DEBUG && true === WP_DEBUG_LOG) {
+				error_log(__METHOD__ . " Site-wide search engine discouragement enabled");
+			}
+			return true;
+		}
+
+		// Check post meta for noindex (common SEO plugins use this)
+		$meta_robots = get_post_meta($post->ID, '_yoast_wpseo_meta-robots-noindex', true);
+		if ($meta_robots === '1') {
+			if (true === WP_DEBUG && true === WP_DEBUG_LOG) {
+				error_log(__METHOD__ . " Yoast noindex meta detected for post ID: " . $post->ID);
+			}
+			return true;
+		}
+
+		// Check RankMath noindex meta
+		$rankmath_robots = get_post_meta($post->ID, 'rank_math_robots', true);
+		if (is_array($rankmath_robots) && in_array('noindex', $rankmath_robots)) {
+			if (true === WP_DEBUG && true === WP_DEBUG_LOG) {
+				error_log(__METHOD__ . " RankMath noindex meta detected for post ID: " . $post->ID);
+			}
+			return true;
+		}
+
+		// Check All in One SEO meta
+		$aioseo_robots = get_post_meta($post->ID, '_aioseo_robots_noindex', true);
+		if ($aioseo_robots === '1') {
+			if (true === WP_DEBUG && true === WP_DEBUG_LOG) {
+				error_log(__METHOD__ . " All in One SEO noindex meta detected for post ID: " . $post->ID);
+			}
+			return true;
+		}
+
+		// Check HTTP headers for X-Robots-Tag noindex
+		// Use a short timeout to avoid delays in post publishing
+		$response = wp_safe_remote_head($url, array('timeout' => 5));
+		if (!is_wp_error($response)) {
+			$headers = wp_remote_retrieve_headers($response);
+			// Check both 'x-robots-tag' and 'X-Robots-Tag' (case insensitive)
+			$robots_header = '';
+			if (isset($headers['x-robots-tag'])) {
+				$robots_header = $headers['x-robots-tag'];
+			} elseif (isset($headers['X-Robots-Tag'])) {
+				$robots_header = $headers['X-Robots-Tag'];
+			}
+
+			if (!empty($robots_header)) {
+				$robots_header = strtolower($robots_header);
+				// Check for noindex in various formats
+				if (strpos($robots_header, 'noindex') !== false) {
+					if (true === WP_DEBUG && true === WP_DEBUG_LOG) {
+						error_log(__METHOD__ . " X-Robots-Tag noindex header detected for URL: " . $url);
+					}
+					return true;
+				}
+			}
+		} elseif (true === WP_DEBUG && true === WP_DEBUG_LOG) {
+			error_log(__METHOD__ . " Could not check X-Robots-Tag header for URL: " . $url . " - " . $response->get_error_message());
+		}
+
+		return false;
+	}
+
 	// This function checks the type of update on a page/post and accordingly calls the submit api if enabled
 	public function on_post_published($new_status, $old_status, $post)
 	{
@@ -193,6 +266,14 @@ public function on_post_published($new_status, $old_status, $post)
 					}
 				}
 
+				// Check for noindex directive - don't submit URLs that should not be indexed
+				if ($type != 'delete' && $this->has_noindex_directive($post, $link)) {
+					if (true === WP_DEBUG && true === WP_DEBUG_LOG) {
+						error_log(__METHOD__ . " Skipping URL submission due to noindex directive: " . $link);
+					}
+					return;
+				}
+
 				$siteUrl = get_home_url();
 
 				// check if same url was submitted recently(within a minute)

public/indexnow-url-submission.php

@@ -9,7 +9,7 @@
  * Plugin Name:       IndexNow
  * Plugin URI:        https://www.bing.com/webmaster
  * Description:       A small plugin to allow Url submissions to IndexNow.
- * Version:           1.0.1
+ * Version:           1.0.2
  * Author:            Microsoft Bing
  * Author URI:        https://www.bing.com/indexnow
  * License:           GPL-2.0+
@@ -27,7 +27,7 @@
  * Currently plugin version.
  */
 
-define( 'BWT_INDEXNOW_PLUGIN_VERSION', '1.0.1' );
+define( 'BWT_INDEXNOW_PLUGIN_VERSION', '1.0.2' );
 
 /**
  * Plugin name.

public/readme.txt

@@ -17,6 +17,8 @@ pages to supporting search engines.
 
 IndexNow Plugin for WordPress enables automated submission of URLs from WordPress sites to the multiple search engines without the need to register and verify your site with them. Once installed, the plugin will automatically generate and host the API key on your site. It detects page creation/update/ deletion in WordPress and automatically submits the URLs in the background. This ensures that search engines will always have the latest updates about your site.
 
+The plugin automatically respects noindex directives to prevent submission of URLs that should not be indexed, including WordPress site-wide search engine visibility settings, SEO plugin noindex settings (Yoast, RankMath, All in One SEO), and X-Robots-Tag headers.
+
 Some other handy features included in the plugin:
 
 * Toggle the automatic submission feature.
@@ -61,6 +63,14 @@ Indexing of URLs is specific and dependent on each search engine’s rules, plea
 == Changelog ==
 
 = 1.0.2 =
+* Add noindex directive detection to prevent submission of URLs that should not be indexed.
+* Support for WordPress site-wide search engine visibility settings.
+* Support for popular SEO plugins (Yoast, RankMath, All in One SEO) noindex meta tags.
+* Support for X-Robots-Tag HTTP headers.
+* Improved logging for debugging noindex detection.
+* Allow plugin owners of WordPress site to access Indexing Insights in Bing Webmaster tools.
+
+= 1.0.1 =
 * Allow plugin owners of WordPress site to access Indexing Insights in Bing Webmaster tools.
 
 = 1.0.1 =