fix: move speciesOrder logic to base Indexer and fix non-self key values

The buildSpeciesOrder method was setting non-self species keys to each
species' own phylogenetic order from the lookup map. This meant all
non-self documents shared the same speciesOrder value for a given focus
taxon key, causing species to interleave on gene page disease/expression
tables.

Now all non-self keys are set to the subject species' phylogenetic order,
giving each species a unique value when sorted by speciesOrder.<taxonId>.

Also consolidates the duplicated SpeciesInterface proxy, species lookup,
and buildSpeciesOrder from both DiseaseAnnotationCurationIndexer and
GeneExpressionAnnotationIndexer into the shared Indexer base class with
lazy initialization on first use.

Author: oblodgett

agr_indexer/src/main/java/org/alliancegenome/indexer/TestSpeciesOrder.java

@@ -57,8 +57,9 @@ public static void main(String[] args) throws Exception {
 	private static HashMap<String, Integer> buildSpeciesOrder(Map<String, Integer> speciesOrderLookup, String taxonCurie) {
 		HashMap<String, Integer> order = new HashMap<>();
 		String subjectTaxonIdPart = taxonCurie.replace("NCBITaxon:", "");
-		for (Map.Entry<String, Integer> entry : speciesOrderLookup.entrySet()) {
-			order.put(entry.getKey(), entry.getValue());
+		Integer subjectOrder = speciesOrderLookup.getOrDefault(subjectTaxonIdPart, 0);
+		for (String key : speciesOrderLookup.keySet()) {
+			order.put(key, subjectOrder);
 		}
 		order.put(subjectTaxonIdPart, 0);
 		return order;

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/Indexer.java

@@ -11,12 +11,18 @@
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.TimeUnit;
 
+import org.alliancegenome.core.config.ConfigHelper;
 import org.alliancegenome.core.util.StatsCollector;
 import org.alliancegenome.curation_api.model.document.es.ESDocument;
+import org.alliancegenome.curation_api.model.entities.Species;
+import org.alliancegenome.es.rest.RestConfig;
 import org.alliancegenome.es.util.EsClientFactory;
 import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
+import org.alliancegenome.indexer.indexers.curation.interfaces.SpeciesInterface;
+
+import si.mazi.rescu.RestProxyFactory;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkProcessor;
@@ -43,6 +49,10 @@ public abstract class Indexer extends Thread {
 	public static String indexName;
 	protected IndexerConfig indexerConfig;
 	private RestHighLevelClient searchClient;
+
+	private final SpeciesInterface speciesApi = RestProxyFactory.createProxy(SpeciesInterface.class, ConfigHelper.getCurationApiUrl(), RestConfig.config);
+	// taxonIdPart (e.g. "9606") -> phylogeneticOrder
+	protected Map<String, Integer> speciesOrderLookup;
 	protected Runtime runtime = Runtime.getRuntime();
 	protected DecimalFormat df = new DecimalFormat("#");
 	protected ObjectMapper om = new ObjectMapper();
@@ -230,6 +240,32 @@ protected <T> List<List<T>> partition(List<T> list, int size) {
 		return parts;
 	}
 
+	protected void loadSpeciesOrderLookup() {
+		speciesOrderLookup = new HashMap<>();
+		List<Species> allSpecies = speciesApi.findForPublic(0, 100, "FieldsOnly", new HashMap<>()).getResults();
+		for (Species species : allSpecies) {
+			if (species.getTaxon() != null && species.getPhylogeneticOrder() != null) {
+				String taxonIdPart = species.getTaxon().getCurie().replace("NCBITaxon:", "");
+				speciesOrderLookup.put(taxonIdPart, species.getPhylogeneticOrder());
+			}
+		}
+		log.info("Loaded " + speciesOrderLookup.size() + " species for speciesOrder lookup");
+	}
+
+	protected HashMap<String, Integer> buildSpeciesOrder(String taxonCurie) {
+		if (speciesOrderLookup == null) {
+			loadSpeciesOrderLookup();
+		}
+		HashMap<String, Integer> order = new HashMap<>();
+		String subjectTaxonIdPart = taxonCurie.replace("NCBITaxon:", "");
+		Integer subjectOrder = speciesOrderLookup.getOrDefault(subjectTaxonIdPart, 0);
+		for (String key : speciesOrderLookup.keySet()) {
+			order.put(key, subjectOrder);
+		}
+		order.put(subjectTaxonIdPart, 0);
+		return order;
+	}
+
 	protected abstract void index(ProcessDisplayHelper display);
 
 	protected abstract void startSingleThread(LinkedBlockingDeque<String> queue);

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/DiseaseAnnotationCurationIndexer.java

@@ -20,7 +20,6 @@
 import org.alliancegenome.api.entity.DiseaseAnnotationDocument;
 import org.alliancegenome.api.entity.GeneDiseaseAnnotationDocument;
 import org.alliancegenome.core.helpers.DiseaseAnnotationHelper;
-import org.alliancegenome.core.config.ConfigHelper;
 import org.alliancegenome.curation_api.model.entities.AGMDiseaseAnnotation;
 import org.alliancegenome.curation_api.model.entities.AffectedGenomicModel;
 import org.alliancegenome.curation_api.model.entities.Allele;
@@ -32,7 +31,6 @@
 import org.alliancegenome.curation_api.model.entities.Gene;
 import org.alliancegenome.curation_api.model.entities.GeneDiseaseAnnotation;
 import org.alliancegenome.curation_api.model.entities.Reference;
-import org.alliancegenome.curation_api.model.entities.Species;
 import org.alliancegenome.curation_api.model.entities.VocabularyTerm;
 import org.alliancegenome.curation_api.model.entities.base.SubmittedObject;
 import org.alliancegenome.curation_api.model.entities.ontology.DOTerm;
@@ -42,13 +40,11 @@
 import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.indexer.indexers.Indexer;
-import org.alliancegenome.indexer.indexers.curation.interfaces.SpeciesInterface;
 import org.alliancegenome.indexer.indexers.curation.service.AGMDiseaseAnnotationService;
 import org.alliancegenome.indexer.indexers.curation.service.AlleleDiseaseAnnotationService;
 import org.alliancegenome.indexer.indexers.curation.service.GeneDiseaseAnnotationService;
 import org.alliancegenome.indexer.indexers.curation.service.VocabularyTermService;
 
-import si.mazi.rescu.RestProxyFactory;
 import org.alliancegenome.neo4j.repository.DiseaseRepository;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -60,16 +56,11 @@
 @Slf4j
 public class DiseaseAnnotationCurationIndexer extends Indexer {
 
-	private final SpeciesInterface speciesApi = RestProxyFactory.createProxy(SpeciesInterface.class, ConfigHelper.getCurationApiUrl(), RestConfig.config);
-
 	private GeneDiseaseAnnotationService geneService;
 	private AlleleDiseaseAnnotationService alleleService;
 	private AGMDiseaseAnnotationService agmService;
 	private VocabularyTermService vocabTermService;
 
-	// taxonIdPart (e.g. "9606") -> phylogeneticOrder
-	private Map<String, Integer> speciesOrderLookup;
-
 	private Map<String, Set<String>> closureMap;
 	private Map<String, Pair<Gene, ArrayList<DiseaseAnnotation>>> geneMap = new HashMap<>();
 	private Map<String, Pair<Gene, ArrayList<DiseaseAnnotation>>> generatedImplicatedGeneMap = new HashMap<>();
@@ -104,16 +95,6 @@ protected void index(ProcessDisplayHelper display) {
 		DiseaseRepository diseaseRepository = new DiseaseRepository();
 		closureMap = diseaseRepository.getDOClosureChildMapping();
 
-		speciesOrderLookup = new HashMap<>();
-		List<Species> allSpecies = speciesApi.findForPublic(0, 100, "FieldsOnly", new HashMap<>()).getResults();
-		for (Species species : allSpecies) {
-			if (species.getTaxon() != null && species.getPhylogeneticOrder() != null) {
-				String taxonIdPart = species.getTaxon().getCurie().replace("NCBITaxon:", "");
-				speciesOrderLookup.put(taxonIdPart, species.getPhylogeneticOrder());
-			}
-		}
-		log.info("Loaded " + speciesOrderLookup.size() + " species for speciesOrder lookup");
-
 		indexGenes();
 		indexAlleles();
 		indexAGMs();
@@ -390,16 +371,6 @@ private List<AGMDiseaseAnnotationDocument> createAGMDiseaseAnnotationDocuments()
 		return ret;
 	}
 
-	private HashMap<String, Integer> buildSpeciesOrder(String taxonCurie) {
-		HashMap<String, Integer> order = new HashMap<>();
-		String subjectTaxonIdPart = taxonCurie.replace("NCBITaxon:", "");
-		for (Map.Entry<String, Integer> entry : speciesOrderLookup.entrySet()) {
-			order.put(entry.getKey(), entry.getValue());
-		}
-		order.put(subjectTaxonIdPart, 0);
-		return order;
-	}
-
 	private void populateBaseDiseaseAnnotationDocument(BiologicalEntity biologicalEntity, DiseaseAnnotation da, DiseaseAnnotationDocument dad) {
 		if (dad.getCountId() == null) {
 			dad.setCountId(uniqueAnnotationCounter.getAndIncrement());

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/GeneExpressionAnnotationIndexer.java

@@ -1,21 +1,17 @@
 package org.alliancegenome.indexer.indexers.curation;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.LinkedBlockingDeque;
 
 import org.alliancegenome.core.config.ConfigHelper;
 import org.alliancegenome.curation_api.interfaces.document.GeneExpressionDocumentInterface;
 import org.alliancegenome.curation_api.model.document.es.GeneExpressionDocument;
-import org.alliancegenome.curation_api.model.entities.Species;
 import org.alliancegenome.curation_api.response.SearchResponse;
 import org.alliancegenome.es.rest.RestConfig;
 import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.indexer.indexers.Indexer;
-import org.alliancegenome.indexer.indexers.curation.interfaces.SpeciesInterface;
 import org.apache.commons.collections.CollectionUtils;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -27,30 +23,16 @@
 public class GeneExpressionAnnotationIndexer extends Indexer {
 
 	private final GeneExpressionDocumentInterface geneExpressionApi = RestProxyFactory.createProxy(GeneExpressionDocumentInterface.class, ConfigHelper.getCurationApiUrl(), RestConfig.config);
-	private final SpeciesInterface speciesApi = RestProxyFactory.createProxy(SpeciesInterface.class, ConfigHelper.getCurationApiUrl(), RestConfig.config);
 
 	private List<List<String>> idBatches;
 
-	// taxonIdPart (e.g. "9606") -> phylogeneticOrder
-	private Map<String, Integer> speciesOrderLookup;
-
 	public GeneExpressionAnnotationIndexer(IndexerConfig indexerConfig) {
 		super(indexerConfig);
 	}
 
 	@Override
 	protected void index(ProcessDisplayHelper display) {
 		try {
-			speciesOrderLookup = new HashMap<>();
-			List<Species> allSpecies = speciesApi.findForPublic(0, 100, "FieldsOnly", new HashMap<>()).getResults();
-			for (Species species : allSpecies) {
-				if (species.getTaxon() != null && species.getPhylogeneticOrder() != null) {
-					String taxonIdPart = species.getTaxon().getCurie().replace("NCBITaxon:", "");
-					speciesOrderLookup.put(taxonIdPart, species.getPhylogeneticOrder());
-				}
-			}
-			log.info("Loaded {} species for speciesOrder lookup", speciesOrderLookup.size());
-
 			log.info("Fetching all gene IDs...");
 			SearchResponse<String> idsResponse = geneExpressionApi.getGeneIds();
 
@@ -103,16 +85,6 @@ protected void startSingleThread(LinkedBlockingDeque<String> queue) {
 		}
 	}
 
-	private HashMap<String, Integer> buildSpeciesOrder(String taxonCurie) {
-		HashMap<String, Integer> order = new HashMap<>();
-		String subjectTaxonIdPart = taxonCurie.replace("NCBITaxon:", "");
-		for (Map.Entry<String, Integer> entry : speciesOrderLookup.entrySet()) {
-			order.put(entry.getKey(), entry.getValue());
-		}
-		order.put(subjectTaxonIdPart, 0);
-		return order;
-	}
-
 	@Override
 	protected ObjectMapper customizeObjectMapper(ObjectMapper objectMapper) {
 		return RestConfig.config.getJacksonObjectMapperFactory().createObjectMapper();