Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,39 +28,41 @@
public enum IndexerConfig {

// Neo Indexers
ModelIndexer("model", ModelIndexer.class, 4, 1500, 1426, 4, 1, false),
ModelIndexer("model", ModelIndexer.class, 4, 1500, 1426, 4, 1, true),

// Curation Indexers

// Run Parallelly
ParalogyIndexer("paralogy", GeneToGeneParalogyIndexer.class, 4, 5000, 5000, 8, 1, true),
GOSearchResultCurationIndexer("goSearchResult", GOSearchResultCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
HTPDatasetSearchResultCurationIndexer("htpDatasetSearchResult", HTPDatasetSearchResultCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
LiteratureIndexer("literature", LiteratureIndexer.class, 4, 5000, 5000, 1, 1, true),
GeneGeneticInteractionIndexers("geneGeneticInteraction", GeneGeneticInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1, true),
AffectedGenomicModelIndexer("affectedGenomicModels", AffectedGenomicModelCurationIndexer.class, 4, 1500, 1500, 8, 1, true),
DiseaseSummaryIndexer("diseaseSummary", DiseaseSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
DiseaseSearchResultIndexer("diseaseSearchResult", DiseaseSearchResultCurationIndexer.class, 8, 50, 50, 4, 1, true),

GeneSearchResultCurationIndexer("geneSearchResult", GeneSearchResultCurationIndexer.class, 4, 1000, 1500, 4, 1, true),
GeneSummaryIndexer("geneSummary", GeneSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, true),

DiseaseSearchResultIndexer("diseaseSearchResult", DiseaseSearchResultCurationIndexer.class, 8, 50, 50, 4, 1, true),
VariantSummaryIndexer("variantSummary", VariantSummaryCurationIndexer.class, 1, 3000, 1500, 8, 1, false),
GeneGeneticInteractionIndexers("geneGeneticInteraction", GeneGeneticInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1, true),
HTPDatasetSearchResultCurationIndexer("htpDatasetSearchResult", HTPDatasetSearchResultCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
LiteratureIndexer("literature", LiteratureIndexer.class, 4, 5000, 5000, 1, 1, true),
GeneExpressionAnnotationIndexer("geneExpressionAnnotation", GeneExpressionAnnotationIndexer.class, 8, 125, 2000, 4, 1, true),
SiteMapAccessionCurationIndexer("sitemap", SiteMapAccessionCurationIndexer.class, 4, 1500, 1500, 8, 1, true),
DiseaseSummaryIndexer("diseaseSummary", DiseaseSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, true),
// neo model indexer
TransgenicAlleleIndexer("transgenicAlleles", TransgenicAlleleCurationIndexer.class, 1, 3000, 1500, 8, 1, true),
GeneExpressionRibbonSummaryIndexer("geneExpressionRibbonSummary", GeneExpressionRibbonSummaryIndexer.class, 1, 1, 1, 1, 1, true),
ReleaseInfoIndexer("release", ReleaseInfoIndexer.class, 1, 1, 1, 1, 1, true),

// Run Sequentially
// Run Sequentially -- typically take a lot of RAM or do consolidation in memory
AlleleSummaryIndexer("alleleSummary", AlleleSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, false),
GeneSummaryIndexer("geneSummary", GeneSummaryCurationIndexer.class, 4, 1500, 1500, 4, 1, false),
GeneToGeneOrthologyIndexer("geneToGeneOrthology", GeneToGeneOrthologyIndexer.class, 4, 2500, 2500, 8, 1, false),

GeneExpressionAnnotationIndexer("geneExpressionAnnotation", GeneExpressionAnnotationIndexer.class, 8, 125, 2000, 4, 1, false),

PhenotypeAnnotationIndexer("phenotypeAnnotation", PhenotypeAnnotationCurationIndexer.class, 4, 1500, 1500, 2, 1, false),
TransgenicAlleleIndexer("transgenicAlleles", TransgenicAlleleCurationIndexer.class, 1, 3000, 1500, 8, 1, false),
GeneMolecularInteractionIndexers("geneMolecularInteraction", GeneMolecularInteractionCurationIndexer.class, 4, 1500, 1500, 2, 1, false),

PhenotypeAnnotationIndexer("phenotypeAnnotation", PhenotypeAnnotationCurationIndexer.class, 4, 1500, 1500, 2, 1, false),
DiseaseAnnotationIndexer("diseaseAnnotation", DiseaseAnnotationCurationIndexer.class, 1, 1500, 1500, 2, 1, false),
VariantSummaryIndexer("variantSummary", VariantSummaryCurationIndexer.class, 1, 3000, 1500, 8, 1, false),

GeneSearchResultCurationIndexer("geneSearchResult", GeneSearchResultCurationIndexer.class, 4, 1000, 1500, 4, 1, false),



;


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.alliancegenome.core.translators.document.HTPDatasetTranslator;
import org.alliancegenome.es.index.site.cache.DatasetDocumentCache;
import org.alliancegenome.es.index.site.document.SearchableItemDocument;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.neo4j.entity.node.HTPDataset;
Expand All @@ -24,7 +25,7 @@ public DatasetIndexer(IndexerConfig config) {
}

@Override
public void index() {
public void index(ProcessDisplayHelper display) {
try {
repo = new DatasetIndexerRepository();
cache = repo.getCache();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.alliancegenome.core.translators.document.DiseaseTranslator;
import org.alliancegenome.es.index.site.cache.DiseaseDocumentCache;
import org.alliancegenome.es.index.site.document.SearchableItemDocument;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.neo4j.entity.node.DOTerm;
Expand All @@ -25,7 +26,7 @@ public DiseaseIndexer(IndexerConfig config) {
}

@Override
public void index() {
public void index(ProcessDisplayHelper display) {
try {
DiseaseIndexerRepository diseaseIndexerRepository = new DiseaseIndexerRepository();
diseaseDocumentCache = diseaseIndexerRepository.getDiseaseDocumentCache();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.alliancegenome.core.translators.document.GeneTranslator;
import org.alliancegenome.es.index.site.cache.GeneDocumentCache;
import org.alliancegenome.es.index.site.document.SearchableItemDocument;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.neo4j.entity.node.Gene;
Expand All @@ -25,7 +26,7 @@ public GeneIndexer(IndexerConfig config) {
}

@Override
public void index() {
public void index(ProcessDisplayHelper display) {

try {
LinkedBlockingDeque<String> queue = new LinkedBlockingDeque<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import org.alliancegenome.core.translators.document.GoTranslator;
import org.alliancegenome.es.index.site.document.SearchableItemDocument;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.neo4j.entity.node.GOTerm;
import org.alliancegenome.neo4j.repository.GoRepository;
Expand All @@ -21,7 +22,7 @@ public GoIndexer(IndexerConfig config) {
}

@Override
public void index() {
public void index(ProcessDisplayHelper display) {

goRepo = new GoRepository();
goTrans = new GoTranslator();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public void runIndex() {
try {
Instant start = Instant.now();
display.startProcess(getClass().getSimpleName());
index();
index(display);
log.info("Waiting for bulkProcessor to finish");
bulkProcessor.flush();
bulkProcessor.awaitClose(30L, TimeUnit.DAYS);
Expand All @@ -126,7 +126,7 @@ public void run() {
try {
Instant start = Instant.now();
display.startProcess(getClass().getSimpleName());
index();
index(display);
log.info("Waiting for bulkProcessor to finish");
bulkProcessor.flush();
bulkProcessor.awaitClose(30L, TimeUnit.DAYS);
Expand Down Expand Up @@ -213,7 +213,7 @@ protected <T> List<List<T>> partition(List<T> list, int size) {
return parts;
}

protected abstract void index();
protected abstract void index(ProcessDisplayHelper display);

protected abstract void startSingleThread(LinkedBlockingDeque<String> queue);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import org.alliancegenome.api.entity.LiteratureSummaryDocument;
import org.alliancegenome.core.config.ConfigHelper;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.curation.interfaces.ElasticSearchInterface;
Expand All @@ -27,7 +28,7 @@ public LiteratureIndexer(IndexerConfig indexerConfig) {

// with multiple thread
@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
log.info("IndexName: " + indexName);
Map<String, Object> countObject = literatureESApi.count(indexName);
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.alliancegenome.core.translators.document.ModelTranslator;
import org.alliancegenome.es.index.site.cache.ModelDocumentCache;
import org.alliancegenome.es.index.site.document.SearchableItemDocument;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.neo4j.entity.node.AffectedGenomicModel;
Expand All @@ -24,13 +25,13 @@ public ModelIndexer(IndexerConfig config) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
repo = new ModelIndexerRepository();
cache = repo.getModelDocumentCache();

LinkedBlockingDeque<String> queue = new LinkedBlockingDeque<>(cache.getModelMap().keySet());

display.startProcess(queue.size());
initiateThreading(queue);
repo.close();
} catch (Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.alliancegenome.core.translators.document.VariantTranslator;
import org.alliancegenome.es.index.site.cache.IndexerCache;
import org.alliancegenome.es.index.site.document.SearchableItemDocument;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.neo4j.entity.node.Variant;
Expand All @@ -25,7 +26,7 @@ public VariantIndexer(IndexerConfig config) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {

// for now, variants only get indexed if an additional flag is set
if (!ConfigHelper.getIndexVariants()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.alliancegenome.curation_api.model.document.es.AffectedGenomicModelDocument;
import org.alliancegenome.curation_api.response.SearchResponse;
import org.alliancegenome.es.rest.RestConfig;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
Expand All @@ -30,13 +31,13 @@ public AffectedGenomicModelCurationIndexer(IndexerConfig config) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
log.info("Fetching all AGM IDs...");
SearchResponse<Long> idsResponse = modelApi.getAllIds();
List<Long> allIds = idsResponse.getResults();
log.info("Fetched {} AGM IDs", allIds.size());

display.startProcess(allIds.size());
idBatches = partition(allIds, indexerConfig.getBufferSize());
log.info("Partitioned into {} batches of up to {}", idBatches.size(), indexerConfig.getBufferSize());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.alliancegenome.curation_api.response.SearchResponse;
import org.alliancegenome.curation_api.view.CurationView;
import org.alliancegenome.es.rest.RestConfig;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
Expand All @@ -37,13 +38,13 @@ public AlleleSummaryCurationIndexer(IndexerConfig indexerConfig) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
log.info("Fetching all allele IDs...");
SearchResponse<Long> idsResponse = alleleApi.getAllIds();
List<Long> allIds = idsResponse.getResults();
log.info("Fetched {} allele IDs", allIds.size());

display.startProcess(allIds.size());
idBatches = partition(allIds, indexerConfig.getBufferSize());
log.info("Partitioned into {} batches of up to {}", idBatches.size(), indexerConfig.getBufferSize());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ protected void startSingleThread(LinkedBlockingDeque<String> queue) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
geneService = new GeneDiseaseAnnotationService();
alleleService = new AlleleDiseaseAnnotationService();
agmService = new AGMDiseaseAnnotationService();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.alliancegenome.curation_api.model.document.es.DiseaseSearchResultDocument;
import org.alliancegenome.curation_api.response.SearchResponse;
import org.alliancegenome.es.rest.RestConfig;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
Expand All @@ -26,12 +27,12 @@ public DiseaseSearchResultCurationIndexer(IndexerConfig indexerConfig) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
log.info("Fetching all disease search result documents...");
SearchResponse<DiseaseSearchResultDocument> response = diseaseApi.findAll();
log.info("Fetched {} disease search result documents", response.getResults().size());

display.startProcess(response.getResults().size());
indexDocuments(response.getResults());
} catch (Exception e) {
log.error("Error while indexing...", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.alliancegenome.curation_api.model.document.es.DiseaseSummaryDocument;
import org.alliancegenome.curation_api.response.SearchResponse;
import org.alliancegenome.es.rest.RestConfig;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
Expand All @@ -33,9 +34,10 @@ public DiseaseSummaryCurationIndexer(IndexerConfig indexerConfig) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
SearchResponse<DiseaseSummaryDocument> diseaseSummaryResponse = diseaseApi.findSummary(0, 0, params);
display.startProcess(diseaseSummaryResponse.getTotalResults());
int totalPages = (int) (diseaseSummaryResponse.getTotalResults() / indexerConfig.getBufferSize());
LinkedBlockingDeque<String> queue = new LinkedBlockingDeque<>();
for (int i = 0; i <= totalPages; i++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.alliancegenome.curation_api.model.document.es.GOSearchResultDocument;
import org.alliancegenome.curation_api.response.SearchResponse;
import org.alliancegenome.es.rest.RestConfig;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
Expand All @@ -30,13 +31,13 @@ public GOSearchResultCurationIndexer(IndexerConfig indexerConfig) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
log.info("Fetching all GO search result IDs...");
SearchResponse<Long> idsResponse = goApi.getAllIds();
List<Long> allIds = idsResponse.getResults();
log.info("Fetched {} GO search result IDs", allIds.size());

display.startProcess(allIds.size());
idBatches = partition(allIds, indexerConfig.getBufferSize());
log.info("Partitioned into {} batches of up to {}", idBatches.size(), indexerConfig.getBufferSize());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.alliancegenome.curation_api.model.entities.Species;
import org.alliancegenome.curation_api.response.SearchResponse;
import org.alliancegenome.es.rest.RestConfig;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
Expand Down Expand Up @@ -38,7 +39,7 @@ public GeneExpressionAnnotationIndexer(IndexerConfig indexerConfig) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
speciesOrderLookup = new HashMap<>();
List<Species> allSpecies = speciesApi.findForPublic(0, 100, "FieldsOnly", new HashMap<>()).getResults();
Expand All @@ -55,7 +56,7 @@ protected void index() {

List<String> primaryExternalIds = idsResponse.getResults();
log.info("Fetched {} gene IDs", primaryExternalIds.size());

display.startProcess(primaryExternalIds.size());
idBatches = partition(primaryExternalIds, indexerConfig.getBufferSize());
log.info("Partitioned into {} batches of up to {}", idBatches.size(), indexerConfig.getBufferSize());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.alliancegenome.curation_api.interfaces.document.GeneExpressionRibbonDocumentInterface;
import org.alliancegenome.curation_api.model.document.es.GeneExpressionRibbonSummaryDocument;
import org.alliancegenome.es.rest.RestConfig;
import org.alliancegenome.es.util.ProcessDisplayHelper;
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
Expand All @@ -24,7 +25,7 @@ public GeneExpressionRibbonSummaryIndexer(IndexerConfig indexerConfig) {
}

@Override
protected void index() {
protected void index(ProcessDisplayHelper display) {
try {
GeneExpressionRibbonSummaryDocument response = api.findDocument();
if (response == null) {
Expand Down
Loading
Loading