Added dynamic views to the indexer

agr_indexer/pom.xml

@@ -32,6 +32,19 @@
 			<artifactId>process-display</artifactId>
 			<version>1.0.12</version>
 		</dependency>
+
+
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-to-slf4j</artifactId>
+			<version>2.20.0</version>
+		</dependency>
+
+		<!-- Required for Log4j2 API calls -->
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-api</artifactId>
+		</dependency>
 	</dependencies>
 	<build>
 		<finalName>${project.artifactId}</finalName>

agr_indexer/src/main/java/org/alliancegenome/indexer/config/IndexerConfig.java

@@ -30,7 +30,7 @@ public enum IndexerConfig {
 	GeneGeneticInteractionIndexers("geneGeneticInteraction", GeneGeneticInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1),
 	GeneMolecularInteractionIndexers("geneMolecularInteraction", GeneMolecularInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1),
 	ParalogyIndexer("paralogy", GeneToGeneParalogyIndexer.class, 4, 5000, 5000, 8, 1),
-	PhenotypeAnnotationIndexer("phenotypeAnnotation", PhenotypeAnnotationCurationIndexer.class, 1, 1500, 1500, 2, 1),
+	PhenotypeAnnotationIndexer("phenotypeAnnotation", PhenotypeAnnotationCurationIndexer.class, 4, 1500, 1500, 2, 1),
 	ReleaseInfoIndexer("release", ReleaseInfoIndexer.class, 1, 1, 1, 1, 1),
 	;
 

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/GoIndexer.java

@@ -13,15 +13,18 @@
 @Slf4j
 public class GoIndexer extends Indexer {
 
-	private final GoRepository goRepo = new GoRepository();
-	private final GoTranslator goTrans = new GoTranslator();
+	private GoRepository goRepo;
+	private GoTranslator goTrans;
 
 	public GoIndexer(IndexerConfig config) {
 		super(config);
 	}
 
 	@Override
 	public void index() {
+
+		goRepo = new GoRepository();
+		goTrans = new GoTranslator();
 
 		log.info("Pulling All Terms");
 

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/DiseaseAnnotationCurationIndexer.java

@@ -1,13 +1,36 @@
 package org.alliancegenome.indexer.indexers.curation;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.extern.slf4j.Slf4j;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.stream.Collectors;
+
 import org.alliancegenome.api.entity.AGMDiseaseAnnotationDocument;
 import org.alliancegenome.api.entity.AlleleDiseaseAnnotationDocument;
 import org.alliancegenome.api.entity.DiseaseAnnotationDocument;
 import org.alliancegenome.api.entity.GeneDiseaseAnnotationDocument;
 import org.alliancegenome.core.helpers.DiseaseAnnotationHelper;
-import org.alliancegenome.curation_api.model.entities.*;
+import org.alliancegenome.curation_api.model.entities.AGMDiseaseAnnotation;
+import org.alliancegenome.curation_api.model.entities.AffectedGenomicModel;
+import org.alliancegenome.curation_api.model.entities.Allele;
+import org.alliancegenome.curation_api.model.entities.AlleleDiseaseAnnotation;
+import org.alliancegenome.curation_api.model.entities.BiologicalEntity;
+import org.alliancegenome.curation_api.model.entities.ConditionRelation;
+import org.alliancegenome.curation_api.model.entities.DiseaseAnnotation;
+import org.alliancegenome.curation_api.model.entities.ExperimentalCondition;
+import org.alliancegenome.curation_api.model.entities.Gene;
+import org.alliancegenome.curation_api.model.entities.GeneDiseaseAnnotation;
+import org.alliancegenome.curation_api.model.entities.Reference;
+import org.alliancegenome.curation_api.model.entities.VocabularyTerm;
 import org.alliancegenome.curation_api.model.entities.base.SubmittedObject;
 import org.alliancegenome.curation_api.model.entities.ontology.DOTerm;
 import org.alliancegenome.curation_api.model.entities.ontology.ECOTerm;
@@ -19,31 +42,28 @@
 import org.alliancegenome.indexer.indexers.curation.service.AGMDiseaseAnnotationService;
 import org.alliancegenome.indexer.indexers.curation.service.AlleleDiseaseAnnotationService;
 import org.alliancegenome.indexer.indexers.curation.service.GeneDiseaseAnnotationService;
-import org.alliancegenome.indexer.indexers.curation.service.VocabularyService;
+import org.alliancegenome.indexer.indexers.curation.service.VocabularyTermService;
 import org.alliancegenome.neo4j.entity.SpeciesType;
 import org.alliancegenome.neo4j.repository.DiseaseRepository;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.tuple.Pair;
 
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.stream.Collectors;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
-import static java.util.stream.Collectors.groupingBy;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class DiseaseAnnotationCurationIndexer extends Indexer {
 
-	private GeneDiseaseAnnotationService geneService = new GeneDiseaseAnnotationService();
-	private AlleleDiseaseAnnotationService alleleService = new AlleleDiseaseAnnotationService();
-	private AGMDiseaseAnnotationService agmService = new AGMDiseaseAnnotationService();
-	private VocabularyService vocabService = new VocabularyService();
+	private GeneDiseaseAnnotationService geneService;
+	private AlleleDiseaseAnnotationService alleleService;
+	private AGMDiseaseAnnotationService agmService;
+	private VocabularyTermService vocabTermService;
 	private DiseaseRepository diseaseRepository;
 
 	private Map<String, Set<String>> closureMap;
 	private Map<String, Pair<Gene, ArrayList<DiseaseAnnotation>>> geneMap = new HashMap<>();
-	Map<String, Pair<Gene, ArrayList<DiseaseAnnotation>>> generatedImplicatedGeneMap = new HashMap<>();
+	private Map<String, Pair<Gene, ArrayList<DiseaseAnnotation>>> generatedImplicatedGeneMap = new HashMap<>();
 
 	private Map<String, Pair<Allele, ArrayList<DiseaseAnnotation>>> alleleMap = new HashMap<>();
 	private Map<String, Pair<AffectedGenomicModel, ArrayList<DiseaseAnnotation>>> agmMap = new HashMap<>();
@@ -66,8 +86,12 @@ protected void startSingleThread(LinkedBlockingDeque<String> queue) {
 
 	@Override
 	protected void index() {
-
+		geneService = new GeneDiseaseAnnotationService();
+		alleleService = new AlleleDiseaseAnnotationService();
+		agmService = new AGMDiseaseAnnotationService();
+		vocabTermService = new VocabularyTermService();
 		diseaseRepository = new DiseaseRepository();
+
 		closureMap = diseaseRepository.getDOClosureChildMapping();
 
 		indexGenes();
@@ -185,7 +209,7 @@ private List<GeneDiseaseAnnotationDocument> createGeneDiseaseAnnotationDocuments
 		ProcessDisplayHelper ph = new ProcessDisplayHelper(10000);
 		ph.startProcess("Creating Gene Disease Annotations", geneMap.size());
 
-		final VocabularyTerm relationIsImplicatedIn = vocabService.getDiseaseRelationTerms().get("is_implicated_in");
+		final VocabularyTerm relationIsImplicatedIn = vocabTermService.getDiseaseRelationTerms().get("is_implicated_in");
 
 		for (Entry<String, Pair<Gene, ArrayList<DiseaseAnnotation>>> entry : geneMap.entrySet()) {
 			HashMap<String, GeneDiseaseAnnotationDocument> lookup = new HashMap<>();
@@ -287,7 +311,7 @@ private List<AlleleDiseaseAnnotationDocument> createAlleleDiseaseAnnotationDocum
 		ProcessDisplayHelper ph = new ProcessDisplayHelper(10000);
 		ph.startProcess("Creating Allele Disease Annotations", alleleMap.size());
 
-		VocabularyTerm relation = vocabService.getDiseaseRelationTerms().get("is_implicated_in");
+		VocabularyTerm relation = vocabTermService.getDiseaseRelationTerms().get("is_implicated_in");
 
 		for (Entry<String, Pair<Allele, ArrayList<DiseaseAnnotation>>> entry : alleleMap.entrySet()) {
 			HashMap<String, AlleleDiseaseAnnotationDocument> lookup = new HashMap<>();

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/GeneExpressionAnnotationIndexer.java

@@ -1,7 +1,9 @@
 package org.alliancegenome.indexer.indexers.curation;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.extern.slf4j.Slf4j;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.LinkedBlockingDeque;
+
 import org.alliancegenome.api.entity.GeneExpressionAnnotationDocument;
 import org.alliancegenome.curation_api.model.entities.GeneExpressionAnnotation;
 import org.alliancegenome.curation_api.response.SearchResponse;
@@ -11,14 +13,14 @@
 import org.alliancegenome.indexer.indexers.curation.service.GeneExpressionAnnotationService;
 import org.apache.commons.collections.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.LinkedBlockingDeque;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class GeneExpressionAnnotationIndexer extends Indexer {
 
-	GeneExpressionAnnotationService geneExpressionAnnotationService = new GeneExpressionAnnotationService();
+	GeneExpressionAnnotationService geneExpressionAnnotationService;
 
 	public GeneExpressionAnnotationIndexer(IndexerConfig indexerConfig) {
 		super(indexerConfig);
@@ -27,6 +29,7 @@ public GeneExpressionAnnotationIndexer(IndexerConfig indexerConfig) {
 	@Override
 	protected void index() {
 		try {
+			geneExpressionAnnotationService = new GeneExpressionAnnotationService();
 			SearchResponse<GeneExpressionAnnotation> response = geneExpressionAnnotationService.getGeneExpressionAnnotations(0, 0);
 			log.info("GeneExpressionAnnotation count: " + response.getTotalResults());
 			int totalPages = (int) (response.getTotalResults() / indexerConfig.getBufferSize());

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/GeneGeneticInteractionCurationIndexer.java

@@ -20,15 +20,16 @@
 @Slf4j
 public class GeneGeneticInteractionCurationIndexer extends Indexer {
 
-	private GeneGeneticInteractionService geneGeneticInteractionService = new GeneGeneticInteractionService();
+
+	private GeneGeneticInteractionService geneGeneticInteractionService;
 
 	public GeneGeneticInteractionCurationIndexer(IndexerConfig config) {
 		super(config);
 	}
 
 	@Override
 	protected void index() {
-
+		geneGeneticInteractionService = new GeneGeneticInteractionService();
 		try {
 			SearchResponse<GeneGeneticInteraction> interactionResponse = geneGeneticInteractionService.getGeneGeneticInteractions(0, 0);
 			log.info("GeneGeneticInteraction count: " + interactionResponse.getTotalResults());

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/GeneMolecularInteractionCurationIndexer.java

@@ -20,15 +20,15 @@
 @Slf4j
 public class GeneMolecularInteractionCurationIndexer extends Indexer {
 
-	private GeneMolecularInteractionService geneMolecularInteractionService = new GeneMolecularInteractionService();
+	private GeneMolecularInteractionService geneMolecularInteractionService;
 
 	public GeneMolecularInteractionCurationIndexer(IndexerConfig config) {
 		super(config);
 	}
 
 	@Override
 	protected void index() {
-
+		geneMolecularInteractionService = new GeneMolecularInteractionService();
 		try {
 			SearchResponse<GeneMolecularInteraction> interactionResponse = geneMolecularInteractionService.getGeneMolecularInteractions(0, 0);
 			log.info("GeneMolecularInteraction count: " + interactionResponse.getTotalResults());

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/GeneToGeneParalogyIndexer.java

@@ -11,20 +11,23 @@
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.indexer.indexers.Indexer;
 import org.alliancegenome.indexer.indexers.curation.service.GeneToGeneParalogyService;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
+
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class GeneToGeneParalogyIndexer extends Indexer {
 
-	GeneToGeneParalogyService service = new GeneToGeneParalogyService();
+	GeneToGeneParalogyService service;
 
 	public GeneToGeneParalogyIndexer(IndexerConfig indexerConfig) {
 		super(indexerConfig);
 	}
 
 	@Override
 	protected void index() {
+		service = new GeneToGeneParalogyService();
 		try {
 			SearchResponse<GeneToGeneParalogy> paralogyResponse = service.getGeneToGeneParalogy(0, 0);
 			//log.info("GeneToGeneParalogy count: " + paralogyResponse.getTotalResults());

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/PhenotypeAnnotationCurationIndexer.java

@@ -1,33 +1,51 @@
 package org.alliancegenome.indexer.indexers.curation;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.extern.slf4j.Slf4j;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.concurrent.LinkedBlockingDeque;
+
 import org.alliancegenome.api.entity.AllelePhenotypeAnnotationDocument;
 import org.alliancegenome.api.entity.GenePhenotypeAnnotationDocument;
 import org.alliancegenome.api.entity.PhenotypeAnnotationDocument;
-import org.alliancegenome.curation_api.model.entities.*;
-import org.alliancegenome.es.util.ProcessDisplayHelper;
+import org.alliancegenome.curation_api.model.entities.AGMPhenotypeAnnotation;
+import org.alliancegenome.curation_api.model.entities.AffectedGenomicModel;
+import org.alliancegenome.curation_api.model.entities.Allele;
+import org.alliancegenome.curation_api.model.entities.AllelePhenotypeAnnotation;
+import org.alliancegenome.curation_api.model.entities.BiologicalEntity;
+import org.alliancegenome.curation_api.model.entities.CrossReference;
+import org.alliancegenome.curation_api.model.entities.Gene;
+import org.alliancegenome.curation_api.model.entities.GenePhenotypeAnnotation;
+import org.alliancegenome.curation_api.model.entities.PhenotypeAnnotation;
+import org.alliancegenome.curation_api.model.entities.Reference;
+import org.alliancegenome.curation_api.model.entities.VocabularyTerm;
+import org.alliancegenome.curation_api.util.ProcessDisplayHelper;
 import org.alliancegenome.indexer.RestConfig;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.indexer.indexers.Indexer;
 import org.alliancegenome.indexer.indexers.curation.service.AGMPhenotypeAnnotationService;
 import org.alliancegenome.indexer.indexers.curation.service.AllelePhenotypeAnnotationService;
 import org.alliancegenome.indexer.indexers.curation.service.GenePhenotypeAnnotationService;
-import org.alliancegenome.indexer.indexers.curation.service.VocabularyService;
+import org.alliancegenome.indexer.indexers.curation.service.VocabularyTermService;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.tuple.Pair;
 
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.LinkedBlockingDeque;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.extern.slf4j.Slf4j;
+
 
 @Slf4j
 public class PhenotypeAnnotationCurationIndexer extends Indexer {
 
-	private GenePhenotypeAnnotationService geneService = new GenePhenotypeAnnotationService();
-	private AllelePhenotypeAnnotationService alleleService = new AllelePhenotypeAnnotationService();
-	private AGMPhenotypeAnnotationService agmService = new AGMPhenotypeAnnotationService();
-	private VocabularyService vocabService = new VocabularyService();
+	private GenePhenotypeAnnotationService geneService;
+	private AllelePhenotypeAnnotationService alleleService;
+	private AGMPhenotypeAnnotationService agmService;
+	private VocabularyTermService vocabTermService;
+
 	private Map<String, Pair<Gene, ArrayList<PhenotypeAnnotation>>> geneMap = new HashMap<>();
 	private Map<String, Pair<Allele, ArrayList<PhenotypeAnnotation>>> alleleMap = new HashMap<>();
 	private Map<String, Pair<AffectedGenomicModel, ArrayList<PhenotypeAnnotation>>> agmMap = new HashMap<>();
@@ -49,6 +67,11 @@ protected void startSingleThread(LinkedBlockingDeque<String> queue) {
 	@Override
 	protected void index() {
 
+		geneService = new GenePhenotypeAnnotationService();
+		alleleService = new AllelePhenotypeAnnotationService();
+		agmService = new AGMPhenotypeAnnotationService();
+		vocabTermService = new VocabularyTermService();
+
 		indexGenes();
 		indexAlleles();
 		indexAGMs();
@@ -70,7 +93,7 @@ private List<GenePhenotypeAnnotationDocument> createGenePhenotypeAnnotationDocum
 		ProcessDisplayHelper ph = new ProcessDisplayHelper(10000);
 		ph.startProcess("Creating Gene Phenotype Annotations", geneMap.size());
 
-		final VocabularyTerm relationIsImplicatedIn = vocabService.getDiseaseRelationTerms().get("is_implicated_in");
+		VocabularyTerm relationIsImplicatedIn = vocabTermService.getDiseaseRelationTerms().get("is_implicated_in");
 
 		for (Entry<String, Pair<Gene, ArrayList<PhenotypeAnnotation>>> pairMap : geneMap.entrySet()) {
 			HashMap<String, GenePhenotypeAnnotationDocument> lookup = new HashMap<>();
@@ -130,7 +153,7 @@ private static String getConsolidationKey(PhenotypeAnnotation da) {
 	}
 
 	private void indexGenes() {
-		List<GenePhenotypeAnnotation> genePhenotypeAnnotations = geneService.getFiltered();
+		List<GenePhenotypeAnnotation> genePhenotypeAnnotations = geneService.getFiltered(indexerConfig.getThreadCount(), indexerConfig.getBufferSize());
 		addPhenotypeAnnotationsToLGlobalMap(genePhenotypeAnnotations);
 	}
 
@@ -145,8 +168,8 @@ private void addPhenotypeAnnotationsToLGlobalMap(List<GenePhenotypeAnnotation> g
 
 	private void indexAlleles() {
 
-		List<AllelePhenotypeAnnotation> allelePhenotypeAnnotations = alleleService.getFiltered();
-		log.info("Filtered Alleles: " + allelePhenotypeAnnotations.size());
+		List<AllelePhenotypeAnnotation> allelePhenotypeAnnotations = alleleService.getFiltered(indexerConfig.getThreadCount(), indexerConfig.getBufferSize());
+		log.info("Filtered Alleles: " + String.format("%,d", allelePhenotypeAnnotations.size()));
 		for (AllelePhenotypeAnnotation da : allelePhenotypeAnnotations) {
 			Allele allele = da.getPhenotypeAnnotationSubject();
 			Pair<Allele, ArrayList<PhenotypeAnnotation>> allelePair = alleleMap.computeIfAbsent(allele.getIdentifier(), alleleCurie -> Pair.of(allele, new ArrayList<>()));
@@ -172,7 +195,7 @@ private void extractGenePhenotypeAnnotations(PhenotypeAnnotation da, Gene inferr
 
 	private void indexAGMs() {
 
-		List<AGMPhenotypeAnnotation> agmDiseaseAnnotations = agmService.getFiltered();
+		List<AGMPhenotypeAnnotation> agmDiseaseAnnotations = agmService.getFiltered(indexerConfig.getThreadCount(), indexerConfig.getBufferSize());
 		log.info("Filtered AGM PAs: " + String.format("%,d", agmDiseaseAnnotations.size()));
 
 		for (AGMPhenotypeAnnotation da : agmDiseaseAnnotations) {
@@ -209,7 +232,7 @@ private List<AllelePhenotypeAnnotationDocument> createAllelePhenotypeAnnotationD
 		ProcessDisplayHelper ph = new ProcessDisplayHelper(10000);
 		ph.startProcess("Creating Allele Disease Annotations", alleleMap.size());
 
-		VocabularyTerm relation = vocabService.getDiseaseRelationTerms().get("is_implicated_in");
+		VocabularyTerm relation = vocabTermService.getDiseaseRelationTerms().get("is_implicated_in");
 
 		for (Entry<String, Pair<Allele, ArrayList<PhenotypeAnnotation>>> pairMap : alleleMap.entrySet()) {
 			HashMap<String, AllelePhenotypeAnnotationDocument> lookup = new HashMap<>();