Merge pull request #1539 from alliance-genome/better_reporting

Added better reporting for counts on all the indexers

Author: oblodgett

agr_indexer/src/main/java/org/alliancegenome/indexer/config/IndexerConfig.java

@@ -28,39 +28,41 @@
 public enum IndexerConfig {
 
 	// Neo Indexers
-	ModelIndexer("model", ModelIndexer.class, 4, 1500, 1426, 4, 1, false),
+	ModelIndexer("model", ModelIndexer.class, 4, 1500, 1426, 4, 1, true),
 
 	// Curation Indexers
 
 	// Run Parallelly
 	ParalogyIndexer("paralogy", GeneToGeneParalogyIndexer.class, 4, 5000, 5000, 8, 1, true),
 	GOSearchResultCurationIndexer("goSearchResult", GOSearchResultCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
-	HTPDatasetSearchResultCurationIndexer("htpDatasetSearchResult", HTPDatasetSearchResultCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
-	LiteratureIndexer("literature", LiteratureIndexer.class, 4, 5000, 5000, 1, 1, true),
-	GeneGeneticInteractionIndexers("geneGeneticInteraction", GeneGeneticInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1, true),
 	AffectedGenomicModelIndexer("affectedGenomicModels", AffectedGenomicModelCurationIndexer.class, 4, 1500, 1500, 8, 1, true),
-	DiseaseSummaryIndexer("diseaseSummary", DiseaseSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
-	DiseaseSearchResultIndexer("diseaseSearchResult", DiseaseSearchResultCurationIndexer.class, 8, 50, 50, 4, 1, true),
 
+	GeneSearchResultCurationIndexer("geneSearchResult", GeneSearchResultCurationIndexer.class, 4, 1000, 1500, 4, 1, true),
+	GeneSummaryIndexer("geneSummary", GeneSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
+	
+	DiseaseSearchResultIndexer("diseaseSearchResult", DiseaseSearchResultCurationIndexer.class, 8, 50, 50, 4, 1, true),
+	VariantSummaryIndexer("variantSummary", VariantSummaryCurationIndexer.class, 1, 3000, 1500, 8, 1, false),
+	GeneGeneticInteractionIndexers("geneGeneticInteraction", GeneGeneticInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1, true),
+	HTPDatasetSearchResultCurationIndexer("htpDatasetSearchResult", HTPDatasetSearchResultCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
+	LiteratureIndexer("literature", LiteratureIndexer.class, 4, 5000, 5000, 1, 1, true),
+	GeneExpressionAnnotationIndexer("geneExpressionAnnotation", GeneExpressionAnnotationIndexer.class, 8, 125, 2000, 4, 1, true),
 	SiteMapAccessionCurationIndexer("sitemap", SiteMapAccessionCurationIndexer.class, 4, 1500, 1500, 8, 1, true),
+	DiseaseSummaryIndexer("diseaseSummary", DiseaseSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
+	// neo model indexer
+	TransgenicAlleleIndexer("transgenicAlleles", TransgenicAlleleCurationIndexer.class, 1, 3000, 1500, 8, 1, true),
 	GeneExpressionRibbonSummaryIndexer("geneExpressionRibbonSummary", GeneExpressionRibbonSummaryIndexer.class, 1, 1, 1, 1, 1, true),
 	ReleaseInfoIndexer("release", ReleaseInfoIndexer.class, 1, 1, 1, 1, 1, true),
 
-	// Run Sequentially
+	// Run Sequentially -- typically take a lot of RAM or do consolidation in memory
 	AlleleSummaryIndexer("alleleSummary", AlleleSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, false),
-	GeneSummaryIndexer("geneSummary", GeneSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, false),
 	GeneToGeneOrthologyIndexer("geneToGeneOrthology", GeneToGeneOrthologyIndexer.class, 4, 2500, 2500, 8, 1, false),
 
-	GeneExpressionAnnotationIndexer("geneExpressionAnnotation", GeneExpressionAnnotationIndexer.class, 8, 125, 2000, 4, 1, false),
-
-	PhenotypeAnnotationIndexer("phenotypeAnnotation", PhenotypeAnnotationCurationIndexer.class, 4, 1500, 1500, 2, 1, false),
-	TransgenicAlleleIndexer("transgenicAlleles", TransgenicAlleleCurationIndexer.class, 1, 3000, 1500, 8, 1, false),
 	GeneMolecularInteractionIndexers("geneMolecularInteraction", GeneMolecularInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1, false),
+	
+	PhenotypeAnnotationIndexer("phenotypeAnnotation", PhenotypeAnnotationCurationIndexer.class, 4, 1500, 1500, 2, 1, false),
 	DiseaseAnnotationIndexer("diseaseAnnotation", DiseaseAnnotationCurationIndexer.class, 1, 1500, 1500, 2, 1, false),
-	VariantSummaryIndexer("variantSummary", VariantSummaryCurationIndexer.class, 1, 3000, 1500, 8, 1, false),
-
-	GeneSearchResultCurationIndexer("geneSearchResult", GeneSearchResultCurationIndexer.class, 4, 1000, 1500, 4, 1, false),
-
+	
+	
 	;
 
 

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/DatasetIndexer.java

@@ -6,6 +6,7 @@
 import org.alliancegenome.core.translators.document.HTPDatasetTranslator;
 import org.alliancegenome.es.index.site.cache.DatasetDocumentCache;
 import org.alliancegenome.es.index.site.document.SearchableItemDocument;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.neo4j.entity.node.HTPDataset;
@@ -24,7 +25,7 @@ public DatasetIndexer(IndexerConfig config) {
 	}
 
 	@Override
-	public void index() {
+	public void index(ProcessDisplayHelper display) {
 		try {
 			repo = new DatasetIndexerRepository();
 			cache = repo.getCache();

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/DiseaseIndexer.java

@@ -7,6 +7,7 @@
 import org.alliancegenome.core.translators.document.DiseaseTranslator;
 import org.alliancegenome.es.index.site.cache.DiseaseDocumentCache;
 import org.alliancegenome.es.index.site.document.SearchableItemDocument;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.neo4j.entity.node.DOTerm;
@@ -25,7 +26,7 @@ public DiseaseIndexer(IndexerConfig config) {
 	}
 
 	@Override
-	public void index() {
+	public void index(ProcessDisplayHelper display) {
 		try {
 			DiseaseIndexerRepository diseaseIndexerRepository = new DiseaseIndexerRepository();
 			diseaseDocumentCache = diseaseIndexerRepository.getDiseaseDocumentCache();

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/GeneIndexer.java

@@ -8,6 +8,7 @@
 import org.alliancegenome.core.translators.document.GeneTranslator;
 import org.alliancegenome.es.index.site.cache.GeneDocumentCache;
 import org.alliancegenome.es.index.site.document.SearchableItemDocument;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.neo4j.entity.node.Gene;
@@ -25,7 +26,7 @@ public GeneIndexer(IndexerConfig config) {
 	}
 
 	@Override
-	public void index() {
+	public void index(ProcessDisplayHelper display) {
 
 		try {
 			LinkedBlockingDeque<String> queue = new LinkedBlockingDeque<>();

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/GoIndexer.java

@@ -4,6 +4,7 @@
 
 import org.alliancegenome.core.translators.document.GoTranslator;
 import org.alliancegenome.es.index.site.document.SearchableItemDocument;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.neo4j.entity.node.GOTerm;
 import org.alliancegenome.neo4j.repository.GoRepository;
@@ -21,7 +22,7 @@ public GoIndexer(IndexerConfig config) {
 	}
 
 	@Override
-	public void index() {
+	public void index(ProcessDisplayHelper display) {
 
 		goRepo = new GoRepository();
 		goTrans = new GoTranslator();

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/Indexer.java

@@ -104,7 +104,7 @@ public void runIndex() {
 		try {
 			Instant start = Instant.now();
 			display.startProcess(getClass().getSimpleName());
-			index();
+			index(display);
 			log.info("Waiting for bulkProcessor to finish");
 			bulkProcessor.flush();
 			bulkProcessor.awaitClose(30L, TimeUnit.DAYS);
@@ -126,7 +126,7 @@ public void run() {
 		try {
 			Instant start = Instant.now();
 			display.startProcess(getClass().getSimpleName());
-			index();
+			index(display);
 			log.info("Waiting for bulkProcessor to finish");
 			bulkProcessor.flush();
 			bulkProcessor.awaitClose(30L, TimeUnit.DAYS);
@@ -213,7 +213,7 @@ protected <T> List<List<T>> partition(List<T> list, int size) {
 		return parts;
 	}
 
-	protected abstract void index();
+	protected abstract void index(ProcessDisplayHelper display);
 
 	protected abstract void startSingleThread(LinkedBlockingDeque<String> queue);
 

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/LiteratureIndexer.java

@@ -7,6 +7,7 @@
 
 import org.alliancegenome.api.entity.LiteratureSummaryDocument;
 import org.alliancegenome.core.config.ConfigHelper;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.indexer.indexers.curation.interfaces.ElasticSearchInterface;
@@ -27,7 +28,7 @@ public LiteratureIndexer(IndexerConfig indexerConfig) {
 
 	// with multiple thread
 	@Override
-	protected void index() {
+	protected void index(ProcessDisplayHelper display) {
 		log.info("IndexName: " + indexName);
 		Map<String, Object> countObject = literatureESApi.count(indexName);
 		try {

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/ModelIndexer.java

@@ -6,6 +6,7 @@
 import org.alliancegenome.core.translators.document.ModelTranslator;
 import org.alliancegenome.es.index.site.cache.ModelDocumentCache;
 import org.alliancegenome.es.index.site.document.SearchableItemDocument;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.neo4j.entity.node.AffectedGenomicModel;
@@ -24,13 +25,13 @@ public ModelIndexer(IndexerConfig config) {
 	}
 
 	@Override
-	protected void index() {
+	protected void index(ProcessDisplayHelper display) {
 		try {
 			repo = new ModelIndexerRepository();
 			cache = repo.getModelDocumentCache();
 
 			LinkedBlockingDeque<String> queue = new LinkedBlockingDeque<>(cache.getModelMap().keySet());
-
+			display.startProcess(queue.size());
 			initiateThreading(queue);
 			repo.close();
 		} catch (Exception e) {

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/VariantIndexer.java

@@ -7,6 +7,7 @@
 import org.alliancegenome.core.translators.document.VariantTranslator;
 import org.alliancegenome.es.index.site.cache.IndexerCache;
 import org.alliancegenome.es.index.site.document.SearchableItemDocument;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.neo4j.entity.node.Variant;
@@ -25,7 +26,7 @@ public VariantIndexer(IndexerConfig config) {
 	}
 
 	@Override
-	protected void index() {
+	protected void index(ProcessDisplayHelper display) {
 
 		// for now, variants only get indexed if an additional flag is set
 		if (!ConfigHelper.getIndexVariants()) {

agr_indexer/src/main/java/org/alliancegenome/indexer/indexers/curation/AffectedGenomicModelCurationIndexer.java

@@ -8,6 +8,7 @@
 import org.alliancegenome.curation_api.model.document.es.AffectedGenomicModelDocument;
 import org.alliancegenome.curation_api.response.SearchResponse;
 import org.alliancegenome.es.rest.RestConfig;
+import org.alliancegenome.es.util.ProcessDisplayHelper;
 import org.alliancegenome.exceptional.client.ExceptionCatcher;
 import org.alliancegenome.indexer.config.IndexerConfig;
 import org.alliancegenome.indexer.indexers.Indexer;
@@ -30,13 +31,13 @@ public AffectedGenomicModelCurationIndexer(IndexerConfig config) {
 	}
 
 	@Override
-	protected void index() {
+	protected void index(ProcessDisplayHelper display) {
 		try {
 			log.info("Fetching all AGM IDs...");
 			SearchResponse<Long> idsResponse = modelApi.getAllIds();
 			List<Long> allIds = idsResponse.getResults();
 			log.info("Fetched {} AGM IDs", allIds.size());
-
+			display.startProcess(allIds.size());
 			idBatches = partition(allIds, indexerConfig.getBufferSize());
 			log.info("Partitioned into {} batches of up to {}", idBatches.size(), indexerConfig.getBufferSize());