Skip to content

Use pagination when updating CrisMetric records in index #516

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kshepherd wants to merge 1 commit into
base: main-cris
Choose a base branch
from
Open

Use pagination when updating CrisMetric records in index #516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 22 additions & 12 deletions dspace-api/src/main/java/org/dspace/metrics/UpdateCrisMetricsInSolrDocService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.dspace.discovery.IndexingService;
import org.dspace.discovery.SearchServiceException;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.dspace.services.ConfigurationService;
import org.dspace.utils.DSpace;

/**
Expand All @@ -30,6 +31,8 @@ public class UpdateCrisMetricsInSolrDocService {

private static final Logger log = LogManager.getLogger(UpdateCrisMetricsInSolrDocService.class);

private ConfigurationService configurationService = new DSpace().getConfigurationService();

private CrisMetricsService crisMetricsService = new DSpace().getServiceManager().getServiceByName(
CrisMetricsServiceImpl.class.getName(), CrisMetricsServiceImpl.class);

Expand All @@ -42,21 +45,28 @@ public void performUpdate(Context context, DSpaceRunnableHandler handler, boolea

public void performUpdate(Context context, DSpaceRunnableHandler handler, boolean optimize, UUID resourceUuid) {
try {
List<CrisMetrics> metrics = resourceUuid == null
? crisMetricsService.findAllLast(context,-1,-1)
: crisMetricsService.findLastMetricsByResourceId(context, resourceUuid, -1, -1);
int offset = 0;
int limit = configurationService.getIntProperty("metrics.indexer.page", 1000);
handler.logInfo("Metric update start");
for (CrisMetrics metric : metrics) {
try {
crisIndexingService.updateMetrics(context, metric);
} catch (RemoteSolrException rse) {
if (StringUtils.containsIgnoreCase(rse.getMessage(), "Did not find child ID Item-")) {
log.error(rse.getMessage());
} else {
throw rse;
List<CrisMetrics> metrics;
while (!(metrics = (resourceUuid == null
? crisMetricsService.findAllLast(context, limit, offset)
: crisMetricsService.findLastMetricsByResourceId(context, resourceUuid, limit, offset)
)).isEmpty()) {
for (CrisMetrics metric : metrics) {
try {
crisIndexingService.updateMetrics(context, metric);
} catch (RemoteSolrException rse) {
if (StringUtils.containsIgnoreCase(rse.getMessage(), "Did not find child ID Item-")) {
log.error(rse.getMessage());
} else {
throw rse;
}
}
}
offset += limit;
}

handler.logInfo("Metric update end");
if (optimize) {
handler.logInfo("Starting solr optimization");
Expand All @@ -68,4 +78,4 @@ public void performUpdate(Context context, DSpaceRunnableHandler handler, boolea
throw new RuntimeException(e.getMessage(), e);
}
}
}
}
6 changes: 5 additions & 1 deletion dspace/config/modules/metrics.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ metrics.scopus.person.instToken = ${scopus.instToken}
# The default limit for the items to be updated by the update-metrics script, used if no limit is provided as parameter
metrics.update-metrics-script.limit = 1750

# The page size of CrisMetrics rows to process when updating the solr index
# Default: 1000
metrics.indexer.page = 1000

#---------------------------------------------------------------#
# Configure altmetric.com badges. #
# See http://api.altmetric.com/embeds.html #
Expand Down Expand Up @@ -131,4 +135,4 @@ plumx.list-data-hide-mentions = false
plumx.list-data-hide-socialmedia = false
plumx.list-data-hide-citations = false
plumx.list-data-pass-hidden-categories = false
plumx.list-data-detail-same-page = false
plumx.list-data-detail-same-page = false