diff --git a/CHANGELOG.md b/CHANGELOG.md index 0302ae61..a14f781a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [7.4.1 - 2021-10-18] ### Fixed - Quality Report for large amount of data +- QB-Generator: Cache number of patients per mdrid (and per mdr id and value) ## [7.4.0 - 2021-09-13] ### Added diff --git a/src/main/java/de/samply/share/client/model/EnumConfiguration.java b/src/main/java/de/samply/share/client/model/EnumConfiguration.java index 4c747737..71eed159 100644 --- a/src/main/java/de/samply/share/client/model/EnumConfiguration.java +++ b/src/main/java/de/samply/share/client/model/EnumConfiguration.java @@ -78,6 +78,7 @@ public enum EnumConfiguration { QUALITY_REPORT_SHOW_STATISTICS_SHEET, QUALITY_REPORT_WORKBOOK_WINDOW, QUALITY_REPORT_MAX_NUMBER_OF_ROWS_PER_SHEET, + QUALITY_REPORT_COLUMNS_TO_BE_FILTERED, MONITORING_REPORT_COUNT_TOTAL, MONITORING_REPORT_COUNT_DKTKFLAG, MONITORING_REPORT_COUNT_REFERENCEQUERY, diff --git a/src/main/java/de/samply/share/client/quality/report/file/excel/row/factory/ExcelRowFactoryImpl.java b/src/main/java/de/samply/share/client/quality/report/file/excel/row/factory/ExcelRowFactoryImpl.java index a14d29d2..ec1223ff 100644 --- a/src/main/java/de/samply/share/client/quality/report/file/excel/row/factory/ExcelRowFactoryImpl.java +++ b/src/main/java/de/samply/share/client/quality/report/file/excel/row/factory/ExcelRowFactoryImpl.java @@ -1,9 +1,13 @@ package de.samply.share.client.quality.report.file.excel.row.factory; +import de.samply.share.client.model.EnumConfiguration; import de.samply.share.client.quality.report.file.excel.cell.element.ExcelCellElement; import de.samply.share.client.quality.report.file.excel.cell.element.StringExcelCellElement; import de.samply.share.client.quality.report.file.excel.row.context.ExcelRowContext; import de.samply.share.client.quality.report.file.excel.row.elements.ExcelRowElements; +import de.samply.share.client.quality.report.properties.PropertyUtils; +import java.util.HashSet; +import java.util.Set; import org.apache.poi.ss.SpreadsheetVersion; import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.Font; @@ -16,7 +20,29 @@ public class ExcelRowFactoryImpl implements ExcelRowFactory { private final StringExcelCellElement emptyElement = new StringExcelCellElement(""); + //TO DELETE: + private Set toBeFiltered = getColumnsToBeFiltered(); + // new HashSet<>( + //Arrays.asList(new Integer[]{0, 3, 4, 7, 9, 10})); + //Arrays.asList(new Integer[]{0, 3, 4, 5, 6, 7, 8, 9, 10})); + private Set getColumnsToBeFiltered() { + + Set columnstoBeFilteredI = new HashSet<>(); + + String[] columnsToBeFilteredS = PropertyUtils.getListOfProperties( + EnumConfiguration.QUALITY_REPORT_COLUMNS_TO_BE_FILTERED); + + for (String columnToBeFilteredS : columnsToBeFilteredS) { + Integer columnToBeFilteredI = new Integer(columnToBeFilteredS); + if (columnToBeFilteredI != null) { + columnstoBeFilteredI.add(columnToBeFilteredI); + } + } + + return columnstoBeFilteredI; + + } @Override public SXSSFSheet addRowTitles(SXSSFSheet sheet, ExcelRowContext excelRowContext) @@ -72,12 +98,14 @@ public SXSSFSheet addRow(SXSSFSheet sheet, ExcelRowElements excelRowElements) SXSSFRow row = sheet.createRow(rowNum); - addElementsToRow(row, excelRowElements); + boolean isToBeFiltered = sheet.getSheetName().equals("all elements"); + addElementsToRow(row, excelRowElements, isToBeFiltered); return sheet; } - private SXSSFRow addElementsToRow(SXSSFRow row, ExcelRowElements elements) { + private SXSSFRow addElementsToRow(SXSSFRow row, ExcelRowElements elements, + boolean isToBeFiltered) { for (int i = 0; i < elements.getMaxNumberOfElements() && i < SpreadsheetVersion.EXCEL2007.getMaxColumns(); @@ -85,7 +113,7 @@ private SXSSFRow addElementsToRow(SXSSFRow row, ExcelRowElements elements) { ExcelCellElement element = elements.getElement(i); - if (element != null) { + if ((element != null) && (!isToBeFiltered || !toBeFiltered.contains(i))) { element.addAsCell(row); } else { emptyElement.addAsCell(row); diff --git a/src/main/java/de/samply/share/client/quality/report/file/excel/sheet/ExplanatoryExcelSheetFactory.java b/src/main/java/de/samply/share/client/quality/report/file/excel/sheet/ExplanatoryExcelSheetFactory.java index 19ec8fb7..d00b2ed1 100644 --- a/src/main/java/de/samply/share/client/quality/report/file/excel/sheet/ExplanatoryExcelSheetFactory.java +++ b/src/main/java/de/samply/share/client/quality/report/file/excel/sheet/ExplanatoryExcelSheetFactory.java @@ -17,6 +17,7 @@ public class ExplanatoryExcelSheetFactory implements ExcelSheetFactory { protected static final Logger logger = LogManager.getLogger(ExplanatoryExcelSheetFactory.class); + private Integer workbookWindow; //private ExplanatoryExcelFileDownloader explanatoryExcelFileDownloader = // new ExplanatoryExcelFileDownloader(); @@ -74,7 +75,8 @@ private SXSSFWorkbook readWorkbook(File explanatoryExcelFile) throws ExcelSheetF logger.info("reading explanatory excel file"); XSSFWorkbook workbook = new XSSFWorkbook(fileInputStream); - return new SXSSFWorkbook(workbook); + return (workbookWindow != null) ? new SXSSFWorkbook(workbook, workbookWindow) + : new SXSSFWorkbook(workbook); } catch (IOException e) { throw new ExcelSheetFactoryException(e); @@ -84,7 +86,7 @@ private SXSSFWorkbook readWorkbook(File explanatoryExcelFile) throws ExcelSheetF @Override public void setMaxNumberOfRowsPerSheet(int maxNumberOfRowsPerSheet) { - // This method does not make sense here. + workbookWindow = maxNumberOfRowsPerSheet; } diff --git a/src/main/java/de/samply/share/client/quality/report/file/excel/workbook/ExcelWorkbookFactoryImpl002.java b/src/main/java/de/samply/share/client/quality/report/file/excel/workbook/ExcelWorkbookFactoryImpl002.java index cbaae107..31d26fb0 100644 --- a/src/main/java/de/samply/share/client/quality/report/file/excel/workbook/ExcelWorkbookFactoryImpl002.java +++ b/src/main/java/de/samply/share/client/quality/report/file/excel/workbook/ExcelWorkbookFactoryImpl002.java @@ -86,7 +86,7 @@ public SXSSFWorkbook createWorkbook(QualityResults qualityResults) if (isSheetSelectedToBeWritten(EnumConfiguration.QUALITY_REPORT_SHOW_INFO_SHEET)) { logger.info("Adding explanatory sheet to Excel quality report file"); if (explanatoryExcelSheetFactory != null) { - SXSSFWorkbook workbook2 = addExplanatorySheet(workbook); + SXSSFWorkbook workbook2 = addExplanatorySheet(workbook, workbookWindow); if (workbook2 != null) { workbook = workbook2; } @@ -220,10 +220,11 @@ private ExcelRowContext createExcelRowContext(QualityResults qualityResults, .createExcelRowContext(qualityResults, asmQualityResults, qualityResultsStatistics); } - private SXSSFWorkbook addExplanatorySheet(SXSSFWorkbook workbook) + private SXSSFWorkbook addExplanatorySheet(SXSSFWorkbook workbook, int workbookWindow) throws ExcelWorkbookFactoryException { try { + explanatoryExcelSheetFactory.setMaxNumberOfRowsPerSheet(workbookWindow); return explanatoryExcelSheetFactory.addSheet(workbook, null, null); } catch (ExcelSheetFactoryException e) { throw new ExcelWorkbookFactoryException(e); diff --git a/src/main/java/de/samply/share/client/quality/report/results/QualityResult.java b/src/main/java/de/samply/share/client/quality/report/results/QualityResult.java index 66975d75..0fc30172 100644 --- a/src/main/java/de/samply/share/client/quality/report/results/QualityResult.java +++ b/src/main/java/de/samply/share/client/quality/report/results/QualityResult.java @@ -6,15 +6,26 @@ public class QualityResult { - private int numberOfPatients = 0; + private Integer numberOfPatients = null; private boolean isValid = false; private final Set patientLocalIds = new HashSet<>(); private final Set patientDktkIds = new HashSet<>(); + /** + * Get number of patients. + * + * @return number of patients. + * + */ public int getNumberOfPatients() { - return (numberOfPatients > 0) ? numberOfPatients - : Math.max(patientLocalIds.size(), patientDktkIds.size()); + + if (numberOfPatients == null) { + numberOfPatients = Math.max(patientLocalIds.size(), patientDktkIds.size()); + } + + return numberOfPatients; + } public void setNumberOfPatients(int numberOfPatients) { @@ -52,6 +63,7 @@ public void addPatientLocalIds(Set patientLocalIds) { if (patientLocalIds != null && patientLocalIds.size() > 0) { this.patientLocalIds.addAll(patientLocalIds); + this.numberOfPatients = null; } } @@ -65,6 +77,7 @@ public void addPatientDktkId(String patientDktkId) { if (patientDktkId != null) { patientDktkIds.add(patientDktkId); + this.numberOfPatients = null; } } diff --git a/src/main/java/de/samply/share/client/quality/report/results/statistics/QualityResultsStatisticsImpl.java b/src/main/java/de/samply/share/client/quality/report/results/statistics/QualityResultsStatisticsImpl.java index 1fa0246f..09bbca4e 100644 --- a/src/main/java/de/samply/share/client/quality/report/results/statistics/QualityResultsStatisticsImpl.java +++ b/src/main/java/de/samply/share/client/quality/report/results/statistics/QualityResultsStatisticsImpl.java @@ -4,7 +4,9 @@ import de.samply.share.client.quality.report.results.QualityResult; import de.samply.share.client.quality.report.results.QualityResults; import de.samply.share.common.utils.MdrIdDatatype; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; public class QualityResultsStatisticsImpl implements QualityResultsStatistics, @@ -17,6 +19,8 @@ public class QualityResultsStatisticsImpl implements QualityResultsStatistics, private final OrConditionsEvaluator orConditionsEvaluator = new OrConditionsEvaluator(); private Integer totalNumberOfPatients; + private Map mdrIdNumberOfPatients = new HashMap<>(); + private MdrIdValueNumberOfPatients mdrIdValueNumberOfPatients = new MdrIdValueNumberOfPatients(); public QualityResultsStatisticsImpl(QualityResults qualityResults, @@ -30,6 +34,37 @@ private double getPercentage(int part, int total) { return (total > 0) ? 100.0d * ((double) part) / ((double) total) : 0; } + private class MdrIdValueNumberOfPatients { + + private Map mdrIdValueNumberOfPatients = new HashMap<>(); + + public void addMdrIdValueNumberOfPatients(MdrIdDatatype mdrId, String value, + Integer numberOfPatients) { + String id = convertToId(mdrId, value); + mdrIdValueNumberOfPatients.put(id, numberOfPatients); + } + + public Integer getNumberOfPatients(MdrIdDatatype mdrId, String value) { + + Integer numberOfPatients = null; + + if (mdrId != null && value != null) { + String id = convertToId(mdrId, value); + if (id != null) { + numberOfPatients = mdrIdValueNumberOfPatients.get(id); + } + } + + return numberOfPatients; + + } + + private String convertToId(MdrIdDatatype mdrId, String value) { + return mdrId.getMajor() + '-' + value; + } + + } + @Override public double getPercentageOfPatientsWithValueOutOfPatientsWithMdrId(MdrIdDatatype mdrId, String value) { @@ -53,8 +88,16 @@ public double getPercentageOfPatientsWithValueOutOfTotalPatients(MdrIdDatatype m private int getPatientsWithValue(MdrIdDatatype mdrID, String value) { - QualityResult result = qualityResults.getResult(mdrID, value); - return (result == null) ? 0 : result.getNumberOfPatients(); + Integer numberOfPatients = mdrIdValueNumberOfPatients.getNumberOfPatients(mdrID, value); + if (numberOfPatients == null) { + + QualityResult result = qualityResults.getResult(mdrID, value); + numberOfPatients = (result == null) ? 0 : result.getNumberOfPatients(); + mdrIdValueNumberOfPatients.addMdrIdValueNumberOfPatients(mdrID, value, numberOfPatients); + + } + + return numberOfPatients; } @@ -82,7 +125,13 @@ private int countPatients(MdrIdDatatype mdrId, QualityResultPatientIdsGetter pro private int getPatientsWithMdrId(MdrIdDatatype mdrId) { - return countPatients(mdrId, qualityResult -> qualityResult.getPatientLocalIds()); + Integer numberOfPatients = mdrIdNumberOfPatients.get(mdrId); + if (numberOfPatients == null) { + numberOfPatients = countPatients(mdrId, qualityResult -> qualityResult.getPatientLocalIds()); + mdrIdNumberOfPatients.put(mdrId, numberOfPatients); + } + + return numberOfPatients; }