fix: handle ElasticsearchStatusException (429) in RoutedBulkIndexer

The ES coordinating memory limit causes HTTP 429 rejections which throw
ElasticsearchStatusException (RuntimeException). submitWithRetry only
caught IOException, so 429s killed BP threads silently. Now catches
ElasticsearchStatusException with sleep+backoff and requeue instead
of thread death.

Author: oblodgett

agr_variant_indexer/src/main/java/org/alliancegenome/indexer/variant/es/managers/RoutedBulkIndexer.java

@@ -12,6 +12,7 @@
 import org.alliancegenome.es.util.EsClientFactory;
 import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -177,6 +178,15 @@ private void submitWithRetry(List<byte[]> docs, String routing) {
 				}
 			}
 
+		} catch (ElasticsearchStatusException e) {
+			totalRetries++;
+			log.warn(label + " Bulk request rejected (HTTP " + e.status().getStatus() + "): " + e.getMessage() + ", sleeping and splitting " + docs.size() + " items and requeueing");
+			try {
+				Thread.sleep(1000 + ThreadLocalRandom.current().nextInt(2000));
+			} catch (InterruptedException ie) {
+				Thread.currentThread().interrupt();
+			}
+			requeueSplit(docs);
 		} catch (IOException e) {
 			totalRetries++;
 			log.warn(label + " Bulk request failed: " + e.getMessage() + ", reconnecting and splitting " + docs.size() + " items and requeueing");