Skip to content

Remove Neo4j ID set filtering from curation indexers #1563

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
oblodgett merged 1 commit into from
Apr 10, 2026
Merged

Remove Neo4j ID set filtering from curation indexers #1563

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 1 addition & 17 deletions ...rc/main/java/org/alliancegenome/indexer/indexers/curation/GeneToGeneOrthologyIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.alliancegenome.indexer.indexers.curation;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand All @@ -18,7 +17,6 @@
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
import org.alliancegenome.indexer.indexers.curation.service.BaseService;
import org.apache.commons.collections4.CollectionUtils;

import com.fasterxml.jackson.databind.ObjectMapper;
Expand All @@ -33,7 +31,6 @@ public class GeneToGeneOrthologyIndexer extends Indexer {
private final GeneExpressionAnnotationCrudInterface geneExpressionApi = RestProxyFactory.createProxy(GeneExpressionAnnotationCrudInterface.class, ConfigHelper.getCurationApiUrl(), RestConfig.config);
private final GeneDiseaseAnnotationCrudInterface geneDiseaseApi = RestProxyFactory.createProxy(GeneDiseaseAnnotationCrudInterface.class, ConfigHelper.getCurationApiUrl(), RestConfig.config);

private Set<String> allNeoGeneIDs;
private Set<String> geneExpressionSet;
private Set<String> geneAnnotationSet;

Expand All @@ -45,8 +42,6 @@ public GeneToGeneOrthologyIndexer(IndexerConfig config) {

@Override
public void index(ProcessDisplayHelper display) {
BaseService baseService = new BaseService();
allNeoGeneIDs = baseService.getAllNeoGeneIDs();
geneExpressionSet = new HashSet<>(geneExpressionApi.annotatedGeneList().getEntities());
geneAnnotationSet = new HashSet<>(geneDiseaseApi.annotatedGeneList().getEntities());

Expand Down Expand Up @@ -99,8 +94,7 @@ protected void startSingleThread(LinkedBlockingDeque<String> queue) {
}
}

List<GeneToGeneOrthologyDocument> filteredResults = filterValidResults(results);
indexDocuments(filteredResults);
indexDocuments(results);
} catch (Exception e) {
log.error("Error while indexing...", e);
ExceptionCatcher.report(e);
Expand All @@ -115,14 +109,4 @@ protected ObjectMapper customizeObjectMapper(ObjectMapper objectMapper) {
return RestConfig.config.getJacksonObjectMapperFactory().createObjectMapper();
}

private List<GeneToGeneOrthologyDocument> filterValidResults(List<GeneToGeneOrthologyDocument> docs) {
List<GeneToGeneOrthologyDocument> result = new ArrayList<>();
for (GeneToGeneOrthologyDocument doc : docs) {
String curie = doc.getGeneToGeneOrthologyGenerated().getObjectGene().getIdentifier();
if (allNeoGeneIDs.contains(curie)) {
result.add(doc);
}
}
return result;
}
}
7 changes: 0 additions & 7 deletions ...in/java/org/alliancegenome/indexer/indexers/curation/SiteMapAccessionCurationIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import org.alliancegenome.exceptional.client.ExceptionCatcher;
import org.alliancegenome.indexer.config.IndexerConfig;
import org.alliancegenome.indexer.indexers.Indexer;
import org.alliancegenome.indexer.indexers.curation.service.BaseService;
import org.alliancegenome.indexer.indexers.document.SiteMapIdDocument;

import si.mazi.rescu.RestProxyFactory;
Expand All @@ -33,10 +32,6 @@ protected void index(ProcessDisplayHelper display) {

Map<String, List<String>> map = document.getIdsByType();

BaseService base = new BaseService();

map.get("allele").retainAll(base.getAllNeoAlleleIDs());

List<List<String>> alleleIdLists = partition(map.get("allele"), 15000);

for (int i = 0; i < alleleIdLists.size(); i++) {
Expand All @@ -49,8 +44,6 @@ protected void index(ProcessDisplayHelper display) {
indexDocument(doc);
}

map.get("gene").retainAll(base.getAllNeoGeneIDs());

List<List<String>> geneIdLists = partition(map.get("gene"), 15000);

for (int i = 0; i < geneIdLists.size(); i++) {
Expand Down
4 changes: 1 addition & 3 deletions ...a/org/alliancegenome/indexer/indexers/curation/service/AGMPhenotypeAnnotationService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,7 @@ public void run() {

SearchResponse<AGMPhenotypeAnnotation> response = agmApi.findForPublic(page, bufferSize, params);
for (AGMPhenotypeAnnotation pa : response.getResults()) {
if (isValidNeoEntity(getAllNeoModelIDs(), pa.getPhenotypeAnnotationSubject().getIdentifier())) {
fullList.offer(pa);
}
fullList.offer(pa);
display.progressProcess();
}

Expand Down
4 changes: 1 addition & 3 deletions ...rg/alliancegenome/indexer/indexers/curation/service/AllelePhenotypeAnnotationService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,7 @@ public void run() {

SearchResponse<AllelePhenotypeAnnotation> response = alleleApi.findForPublic(page, bufferSize, params);
for (AllelePhenotypeAnnotation pa : response.getResults()) {
if (isValidNeoEntity(getAllNeoAlleleIDs(), pa.getPhenotypeAnnotationSubject().getIdentifier())) {
fullList.offer(pa);
}
fullList.offer(pa);
display.progressProcess();
}

Expand Down
14 changes: 0 additions & 14 deletions ...ain/java/org/alliancegenome/indexer/indexers/curation/service/BaseInteractionService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,20 +38,6 @@ protected <E extends GeneInteraction> boolean hasNoObsoletedOrInternalEntities(E
return hasNoExcludedEntities(entitiesToBeValidated);
}

protected <E extends GeneInteraction> boolean hasInteractingGenesInNeo(E interaction) {
if (interaction.getGeneAssociationSubject() != null) {
if (!isValidNeoEntity(getAllNeoGeneIDs(), interaction.getGeneAssociationSubject().getIdentifier())) {
return false;
}
}
if (interaction.getGeneGeneAssociationObject() != null) {
if (!isValidNeoEntity(getAllNeoGeneIDs(), interaction.getGeneGeneAssociationObject().getIdentifier())) {
return false;
}
}
return true;
}

protected <E extends GeneInteraction> E reverseInteraction(E forwardInteraction, E reverseInteraction) {
if (forwardInteraction.getGeneAssociationSubject() == null || forwardInteraction.getGeneGeneAssociationObject() == null) {
return null;
Expand Down
85 changes: 0 additions & 85 deletions ...dexer/src/main/java/org/alliancegenome/indexer/indexers/curation/service/BaseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,98 +1,17 @@
package org.alliancegenome.indexer.indexers.curation.service;

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;

import org.alliancegenome.curation_api.model.entities.base.AuditedObject;
import org.alliancegenome.neo4j.repository.AlleleRepository;
import org.alliancegenome.neo4j.repository.GeneRepository;
import org.alliancegenome.neo4j.repository.VariantRepository;
import org.apache.commons.collections4.CollectionUtils;

import lombok.extern.log4j.Log4j2;
import net.nilosplace.process_display.util.ObjectFileStorage;

@Log4j2
public class BaseService {

private static HashSet<String> allNeoAlleleIDs;
private static HashSet<String> allNeoGeneIDs;
private static HashSet<String> allNeoModelIDs;
private static HashSet<String> allNeoVariantIDs;

public HashSet<String> getAllNeoAlleleIDs() {
if (allNeoAlleleIDs == null) {
String alleleIdsFileName = "allele_ids.gz";
List<String> alleleList = readFromCache(alleleIdsFileName, List.class);

if (CollectionUtils.isNotEmpty(alleleList)) {
allNeoAlleleIDs = new HashSet<>(alleleList);
} else {
AlleleRepository alleleRepository = new AlleleRepository();
allNeoAlleleIDs = new HashSet<>(alleleRepository.getAllAlleleIDs());
alleleRepository.close();
writeToCache(alleleIdsFileName, new ArrayList<>(allNeoAlleleIDs));
}
}
return allNeoAlleleIDs;
}

public HashSet<String> getAllNeoGeneIDs() {

if (allNeoGeneIDs == null) {
String geneIdsFileName = "gene_ids.gz";
List<String> geneList = readFromCache(geneIdsFileName, List.class);

if (CollectionUtils.isNotEmpty(geneList)) {
allNeoGeneIDs = new HashSet<>(geneList);
} else {
GeneRepository geneRepository = new GeneRepository();
allNeoGeneIDs = new HashSet<>(geneRepository.getAllGeneKeys());
geneRepository.close();
writeToCache(geneIdsFileName, new ArrayList<>(allNeoGeneIDs));
}
}

return allNeoGeneIDs;
}

public HashSet<String> getAllNeoModelIDs() {
if (allNeoModelIDs == null) {
String modelIdsFileName = "model_ids.gz";
List<String> modelList = readFromCache(modelIdsFileName, List.class);

if (CollectionUtils.isNotEmpty(modelList)) {
allNeoModelIDs = new HashSet<>(modelList);
} else {
AlleleRepository alleleRepository = new AlleleRepository();
allNeoModelIDs = new HashSet<>(alleleRepository.getAllModelKeys());
alleleRepository.close();
writeToCache(modelIdsFileName, new ArrayList<>(allNeoModelIDs));
}
}
return allNeoModelIDs;
}

public HashSet<String> getAllNeoVariantIDs() {
if (allNeoVariantIDs == null) {
String variantIdsFileName = "variant_ids.gz";
List<String> variantList = readFromCache(variantIdsFileName, List.class);

if (CollectionUtils.isNotEmpty(variantList)) {
allNeoVariantIDs = new HashSet<>(variantList);
} else {
VariantRepository variantRepository = new VariantRepository();
allNeoVariantIDs = new HashSet<>(variantRepository.getAllVariantKeys());
variantRepository.close();
writeToCache(variantIdsFileName, new ArrayList<>(allNeoVariantIDs));
}
}
return allNeoVariantIDs;
}

protected <E> E readFromCache(String fileName, Class<E> clazz) {
try {
ObjectFileStorage<E> storage = new ObjectFileStorage<>();
Expand Down Expand Up @@ -128,8 +47,4 @@ protected boolean hasNoExcludedEntities(List<AuditedObject> entitiesToBeValidate
return hasNoExcludedEntities.get();
}

protected static boolean isValidNeoEntity(HashSet<String> neoEntityIds, String curie) {
return neoEntityIds.contains(curie);
}

}
34 changes: 8 additions & 26 deletions ...a/org/alliancegenome/indexer/indexers/curation/service/GeneGeneticInteractionService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,40 +13,22 @@ public List<GeneGeneticInteraction> getFilteredAndReversedInteractions(List<Gene
List<GeneGeneticInteraction> validInteractions = new ArrayList<>();

for (GeneGeneticInteraction interaction: forwardInteractions) {
if (hasPerturbatingAllelesInNeo(interaction)) {
if (hasInteractingGenesInNeo(interaction)) {
if (hasNoObsoletedOrInternalEntities(interaction)) {
validInteractions.add(interaction);
try {
GeneGeneticInteraction reverseInteraction = generateReverseInteraction(interaction);
if (reverseInteraction != null) {
validInteractions.add(reverseInteraction);
}
} catch (IOException e) {
e.printStackTrace();
}
if (hasNoObsoletedOrInternalEntities(interaction)) {
validInteractions.add(interaction);
try {
GeneGeneticInteraction reverseInteraction = generateReverseInteraction(interaction);
if (reverseInteraction != null) {
validInteractions.add(reverseInteraction);
}
} catch (IOException e) {
e.printStackTrace();
}
}
}

return validInteractions;
}

private boolean hasPerturbatingAllelesInNeo(GeneGeneticInteraction interaction) {
if (interaction.getInteractorAGeneticPerturbation() != null) {
if (!isValidNeoEntity(getAllNeoAlleleIDs(), interaction.getInteractorAGeneticPerturbation().getIdentifier())) {
return false;
}
}
if (interaction.getInteractorBGeneticPerturbation() != null) {
if (!isValidNeoEntity(getAllNeoAlleleIDs(), interaction.getInteractorBGeneticPerturbation().getIdentifier())) {
return false;
}
}
return true;
}

private GeneGeneticInteraction generateReverseInteraction(GeneGeneticInteraction forwardInteraction) throws IOException {
GeneGeneticInteraction newInteraction = mapper.readValue(mapper.writeValueAsString(forwardInteraction), GeneGeneticInteraction.class);
newInteraction.setInteractorAGeneticPerturbation(forwardInteraction.getInteractorBGeneticPerturbation());
Expand Down
18 changes: 8 additions & 10 deletions ...org/alliancegenome/indexer/indexers/curation/service/GeneMolecularInteractionService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,15 @@ public List<GeneMolecularInteraction> getFilteredAndReversedInteractions(List<Ge
List<GeneMolecularInteraction> validInteractions = new ArrayList<>();

for (GeneMolecularInteraction interaction: forwardInteractions) {
if (hasInteractingGenesInNeo(interaction)) {
if (hasNoObsoletedOrInternalEntities(interaction)) {
validInteractions.add(interaction);
try {
GeneMolecularInteraction reverseInteraction = generateReverseInteraction(interaction);
if (reverseInteraction != null) {
validInteractions.add(reverseInteraction);
}
} catch (IOException e) {
e.printStackTrace();
if (hasNoObsoletedOrInternalEntities(interaction)) {
validInteractions.add(interaction);
try {
GeneMolecularInteraction reverseInteraction = generateReverseInteraction(interaction);
if (reverseInteraction != null) {
validInteractions.add(reverseInteraction);
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
Expand Down
4 changes: 1 addition & 3 deletions .../org/alliancegenome/indexer/indexers/curation/service/GenePhenotypeAnnotationService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,7 @@ public void run() {

SearchResponse<GenePhenotypeAnnotation> response = geneApi.findForPublic(page, bufferSize, "PhenotypeAnnotationView", params);
for (GenePhenotypeAnnotation pa : response.getResults()) {
if (isValidNeoEntity(getAllNeoGeneIDs(), pa.getPhenotypeAnnotationSubject().getIdentifier())) {
fullList.offer(pa);
}
fullList.offer(pa);
display.progressProcess();
}

Expand Down
Loading