diff --git a/pom.xml b/pom.xml index 5357f4fff7..51e4592d1b 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,15 @@ 2.25.3 3.5.11 2.4.3 + + 5.5.1 + 1.21.0 + 4.5.0 + 2.21.0 + 3.18.0 @@ -243,6 +252,10 @@ WEB-INF/lib/log4j-*-2.*.jar WEB-INF/lib/httpclient-4.5.5.jar + WEB-INF/lib/commons-codec-*.jar + WEB-INF/lib/commons-collections4-*.jar + WEB-INF/lib/commons-io-*.jar + WEB-INF/lib/commons-lang3-*.jar WEB-INF/lib/commons-logging-*.jar WEB-INF/lib/gson-*.jar WEB-INF/lib/pw-iso20022-*.jar @@ -1786,6 +1799,55 @@ End of JAX-RS dependencies. --> + + + org.apache.poi + poi + ${poi.version} + + + org.apache.logging.log4j + log4j-api + + + + + org.apache.poi + poi-ooxml + ${poi.version} + + + org.apache.logging.log4j + log4j-api + + + + + commons-codec + commons-codec + ${commons-codec.version} + + + org.apache.commons + commons-collections4 + ${commons-collections4.version} + + + commons-io + commons-io + ${commons-io.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + com.prowidesoftware pw-iso20022 diff --git a/src/main/java/edu/cornell/kfs/sys/CUKFSConstants.java b/src/main/java/edu/cornell/kfs/sys/CUKFSConstants.java index 9e4282fb20..4b627d79df 100644 --- a/src/main/java/edu/cornell/kfs/sys/CUKFSConstants.java +++ b/src/main/java/edu/cornell/kfs/sys/CUKFSConstants.java @@ -195,6 +195,7 @@ public static final class FileExtensions { public static final String CSV = ".csv"; public static final String DONE = ".done"; public static final String DATA = ".data"; + public static final String XLSX = ".xlsx"; } public static final class OptionalModuleNamespaces { diff --git a/src/main/java/edu/cornell/kfs/sys/CUKFSKeyConstants.java b/src/main/java/edu/cornell/kfs/sys/CUKFSKeyConstants.java index 2050200320..18326c19b4 100644 --- a/src/main/java/edu/cornell/kfs/sys/CUKFSKeyConstants.java +++ b/src/main/java/edu/cornell/kfs/sys/CUKFSKeyConstants.java @@ -229,6 +229,8 @@ public class CUKFSKeyConstants { public static final String ERROR_BANK_REQUIRED_PER_PAYMENT_METHOD = "error.document.withBanking.bank.required"; public static final String ERROR_BANK_INVALID = "error.document.withBanking.bank.invalid"; + public static final String MESSAGE_BATCH_UPLOAD_TITLE_CEMI_OUTPUT_DEFINITION = "message.batchUpload.title.cemiOutputDefinition"; + // NOTE: The following set of constants can be removed if KualiCo adds variants that are more publicly visible. public static final String ERROR_UPLOADFILE_EMPTY = "error.uploadFile.empty"; public static final String ERROR_DOCUMENT_INVALID_VALUE_ALLOWED_VALUES_PARAMETER = diff --git a/src/main/java/edu/cornell/kfs/sys/CemiBaseConstants.java b/src/main/java/edu/cornell/kfs/sys/CemiBaseConstants.java new file mode 100644 index 0000000000..af119d77f0 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/CemiBaseConstants.java @@ -0,0 +1,12 @@ +package edu.cornell.kfs.sys; + +public final class CemiBaseConstants { + + public static final String CEMI_OUTPUT_DEFINITION_FILE_TYPE_IDENTIFIER = "cemiOutputDefinitionFileType"; + + public enum CemiFieldDefinitionType { + STATIC, + STRING; + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/batch/CemiOutputDefinitionFileType.java b/src/main/java/edu/cornell/kfs/sys/batch/CemiOutputDefinitionFileType.java new file mode 100644 index 0000000000..2be8d28f10 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/batch/CemiOutputDefinitionFileType.java @@ -0,0 +1,19 @@ +package edu.cornell.kfs.sys.batch; + +import edu.cornell.kfs.sys.CUKFSKeyConstants; +import edu.cornell.kfs.sys.CemiBaseConstants; +import edu.cornell.kfs.sys.batch.xml.CemiOutputDefinition; + +public class CemiOutputDefinitionFileType extends CuXmlBatchInputFileTypeBase { + + @Override + public String getFileTypeIdentifier() { + return CemiBaseConstants.CEMI_OUTPUT_DEFINITION_FILE_TYPE_IDENTIFIER; + } + + @Override + public String getTitleKey() { + return CUKFSKeyConstants.MESSAGE_BATCH_UPLOAD_TITLE_CEMI_OUTPUT_DEFINITION; + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiCsvReader.java b/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiCsvReader.java new file mode 100644 index 0000000000..c0caa1034f --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiCsvReader.java @@ -0,0 +1,90 @@ +package edu.cornell.kfs.sys.batch.service.impl; + +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.Validate; +import org.kuali.kfs.sys.KFSConstants; + +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.CSVWriter; +import com.opencsv.ICSVParser; + +public class CemiCsvReader implements Closeable { + + private final FileInputStream inputStream; + private final InputStreamReader streamReader; + private final CSVReader csvReader; + private final File inputFile; + + public CemiCsvReader(final String inputFileName) throws IOException { + this(convertToFile(inputFileName)); + } + + private static File convertToFile(final String inputFileName) { + Validate.notBlank(inputFileName, "inputFileName cannot be blank"); + return new File(inputFileName); + } + + public CemiCsvReader(final File inputFile) throws IOException { + Validate.notNull(inputFile, "inputFile cannot be null"); + + FileInputStream inputStream = null; + InputStreamReader streamReader = null; + CSVReader csvReader = null; + boolean success = false; + + try { + inputStream = new FileInputStream(inputFile); + streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); + csvReader = buildCSVReader(streamReader); + this.inputStream = inputStream; + this.streamReader = streamReader; + this.csvReader = csvReader; + this.inputFile = inputFile; + success = true; + } finally { + if (!success) { + IOUtils.closeQuietly(csvReader, streamReader, inputStream); + } + } + } + + private static CSVReader buildCSVReader(final Reader fileReader) { + final ICSVParser csvParser = new CSVParserBuilder() + .withSeparator(KFSConstants.COMMA.charAt(0)) + .withQuoteChar(CSVWriter.DEFAULT_QUOTE_CHARACTER) + .build(); + + return new CSVReaderBuilder(fileReader) + .withCSVParser(csvParser) + .build(); + } + + @Override + public void close() throws IOException { + IOUtils.closeQuietly(csvReader, streamReader, inputStream); + } + + public Iterator iterator() { + return csvReader.iterator(); + } + + public CSVReader getCsvReader() { + return csvReader; + } + + public File getInputFile() { + return inputFile; + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiCsvWriter.java b/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiCsvWriter.java new file mode 100644 index 0000000000..e176efc02b --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiCsvWriter.java @@ -0,0 +1,74 @@ +package edu.cornell.kfs.sys.batch.service.impl; + +import java.io.Closeable; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.Validate; +import org.kuali.kfs.sys.KFSConstants; + +import com.opencsv.CSVWriter; +import com.opencsv.CSVWriterBuilder; +import com.opencsv.ICSVWriter; + +public class CemiCsvWriter implements Closeable { + + private final FileOutputStream outputStream; + private final OutputStreamWriter streamWriter; + private final ICSVWriter csvWriter; + private final String outputFileName; + + public CemiCsvWriter(final String outputFileName) throws IOException { + Validate.notBlank(outputFileName, "outputFileName cannot be blank"); + + FileOutputStream outputStream = null; + OutputStreamWriter streamWriter = null; + ICSVWriter csvWriter = null; + boolean success = false; + + try { + outputStream = new FileOutputStream(outputFileName); + streamWriter = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8); + csvWriter = buildCSVWriter(streamWriter); + this.outputStream = outputStream; + this.streamWriter = streamWriter; + this.csvWriter = csvWriter; + this.outputFileName = outputFileName; + success = true; + } finally { + if (!success) { + IOUtils.closeQuietly(csvWriter, streamWriter, outputStream); + } + } + } + + private static ICSVWriter buildCSVWriter(final Writer fileWriter) { + return new CSVWriterBuilder(fileWriter) + .withSeparator(KFSConstants.COMMA.charAt(0)) + .withQuoteChar(CSVWriter.DEFAULT_QUOTE_CHARACTER) + .withLineEnd(KFSConstants.NEWLINE) + .build(); + } + + @Override + public void close() throws IOException { + IOUtils.closeQuietly(csvWriter, streamWriter, outputStream); + } + + public void writeNext(final String[] nextLine) { + csvWriter.writeNext(nextLine); + } + + public ICSVWriter getCsvWriter() { + return csvWriter; + } + + public String getOutputFileName() { + return outputFileName; + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiExcelWriter.java b/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiExcelWriter.java new file mode 100644 index 0000000000..4f90c65854 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/batch/service/impl/CemiExcelWriter.java @@ -0,0 +1,147 @@ +package edu.cornell.kfs.sys.batch.service.impl; + +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.xssf.streaming.SXSSFCell; +import org.apache.poi.xssf.streaming.SXSSFRow; +import org.apache.poi.xssf.streaming.SXSSFSheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import edu.cornell.kfs.sys.batch.xml.CemiOutputDefinition; +import edu.cornell.kfs.sys.batch.xml.CemiSheetDefinition; + +public class CemiExcelWriter implements Closeable { + + private static final Logger LOG = LogManager.getLogger(); + + private final Map sheets; + + private FileInputStream fileInputStream; + private OPCPackage opcPackage; + private XSSFWorkbook templateWorkbook; + private SXSSFWorkbook streamedWorkbook; + private FileOutputStream fileOutputStream; + private boolean committed; + + public CemiExcelWriter(final CemiOutputDefinition outputDefinition, final File templateFile) + throws IOException, InvalidFormatException { + Validate.notNull(outputDefinition, "outputDefinition cannot be null"); + Validate.notNull(templateFile, "templateFile cannot be null"); + boolean setupSucceeded = false; + + try { + fileInputStream = new FileInputStream(templateFile); + opcPackage = OPCPackage.open(fileInputStream); + templateWorkbook = new XSSFWorkbook(opcPackage); + removeNonHeaderRowsFromTemplate(outputDefinition, templateWorkbook); + streamedWorkbook = new SXSSFWorkbook(templateWorkbook); + fileOutputStream = new FileOutputStream(templateFile); + + this.sheets = createSheetsMap(outputDefinition, streamedWorkbook); + committed = false; + setupSucceeded = true; + } finally { + if (!setupSucceeded) { + IOUtils.closeQuietly(fileOutputStream, streamedWorkbook, templateWorkbook, opcPackage, fileInputStream); + } + } + } + + private static void removeNonHeaderRowsFromTemplate(final CemiOutputDefinition outputDefinition, + final XSSFWorkbook templateWorkbook) { + for (final CemiSheetDefinition sheetDefinition : outputDefinition.getSheets()) { + final XSSFSheet sheet = templateWorkbook.getSheet(sheetDefinition.getName()); + Validate.validState(sheet != null, "Sheet not found in template: %s", sheetDefinition.getName()); + final int lastHeaderRowIndex = sheetDefinition.getNumHeaderRows() - 1; + + int lastRowIndex = sheet.getLastRowNum(); + while (lastRowIndex > lastHeaderRowIndex) { + final XSSFRow rowToDelete = sheet.getRow(lastRowIndex); + sheet.removeRow(rowToDelete); + lastRowIndex = sheet.getLastRowNum(); + } + } + } + + private static Map createSheetsMap( + final CemiOutputDefinition outputDefinition, final SXSSFWorkbook workbook) { + return outputDefinition.getSheets().stream() + .collect(Collectors.toUnmodifiableMap( + CemiSheetDefinition::getName, + sheetDefinition -> new CemiSheet( + sheetDefinition, workbook.getSheet(sheetDefinition.getName())) + )); + } + + public void writeRow(final String sheetName, final String[] rowData) { + Validate.notBlank(sheetName, "sheetName cannot be blank"); + Validate.notNull(rowData, "rowData cannot be null"); + + final CemiSheet sheet = sheets.get(sheetName); + Validate.validState(sheet != null, "Sheet not found: %s", sheetName); + final int rowDataLength = sheet.sheetDefinition.getFields().size(); + Validate.validState(rowDataLength == rowData.length, + "rowData for sheet %s should have had %s elements, but it actually had %s elements", + sheetName, rowDataLength, rowData.length); + + final int nextRowIndex = sheet.nextRowIndex.getAndAdd(1); + final SXSSFRow row = sheet.workbookSheet.createRow(nextRowIndex); + final int columnOffset = sheet.sheetDefinition.getStartColumnIndex(); + + for (int i = 0; i < rowDataLength; i++) { + final String fieldValue = rowData[i]; + final SXSSFCell cell = row.createCell(i + columnOffset); + if (StringUtils.isNotBlank(fieldValue)) { + cell.setCellValue(fieldValue); + } + } + } + + public void commit() throws IOException { + Validate.validState(!committed, "The source spreadsheet file has already been overwritten"); + streamedWorkbook.write(fileOutputStream); + committed = true; + } + + @Override + public void close() throws IOException { + if (!committed) { + LOG.warn("close, This instance has not yet committed its updates to the source spreadsheet file. " + + "Any changes that are only stored in auto-generated temporary files could be lost!"); + } + IOUtils.closeQuietly(fileOutputStream, streamedWorkbook, templateWorkbook, opcPackage, fileInputStream); + } + + private static final class CemiSheet { + private final CemiSheetDefinition sheetDefinition; + private final SXSSFSheet workbookSheet; + private final MutableInt nextRowIndex; + + private CemiSheet(final CemiSheetDefinition sheetDefinition, final SXSSFSheet workbookSheet) { + Validate.notNull(sheetDefinition, "sheetDefinition cannot be null"); + Validate.notNull(workbookSheet, "workbookSheet cannot be null; sheet might not exist in file: %s", + sheetDefinition.getName()); + this.sheetDefinition = sheetDefinition; + this.workbookSheet = workbookSheet; + this.nextRowIndex = new MutableInt(sheetDefinition.getNumHeaderRows()); + } + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiFieldDefinition.java b/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiFieldDefinition.java new file mode 100644 index 0000000000..89d622e958 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiFieldDefinition.java @@ -0,0 +1,122 @@ +package edu.cornell.kfs.sys.batch.xml; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; + +import edu.cornell.kfs.sys.CemiBaseConstants.CemiFieldDefinitionType; +import jakarta.xml.bind.Unmarshaller; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlType; + +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "cemiFieldType") +@XmlRootElement(name = "field") +public class CemiFieldDefinition { + + @XmlAttribute(name = "name", required = true) + private String name; + + @XmlAttribute(name = "type", required = true) + private CemiFieldDefinitionType type; + + // Currently not in use; verify if we need this. + @XmlAttribute(name = "max-length") + private int maxLength; + + @XmlAttribute(name = "key") + private String key; + + @XmlAttribute(name = "value") + private String value; + + public CemiFieldDefinition() { + this.maxLength = -1; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public CemiFieldDefinitionType getType() { + return type; + } + + public void setType(final CemiFieldDefinitionType type) { + this.type = type; + } + + public int getMaxLength() { + return maxLength; + } + + public void setMaxLength(final int maxLength) { + this.maxLength = maxLength; + } + + public String getKey() { + return key; + } + + public void setKey(final String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + + @Override + public boolean equals(final Object obj) { + return EqualsBuilder.reflectionEquals(obj, this); + } + + @Override + public int hashCode() { + return HashCodeBuilder.reflectionHashCode(this); + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this); + } + + void afterUnmarshal(final Unmarshaller unmarshaller, final Object parent) { + if (key != null) { + if (value != null) { + throw new IllegalStateException("'key' and 'value' attributes on 'field' tags are mutually exclusive"); + } else if (StringUtils.isBlank(key)) { + throw new IllegalStateException( + "'field' tags that specify the 'key' attribute must not set a blank key"); + } else if (type == CemiFieldDefinitionType.STATIC) { + throw new IllegalStateException( + "'field' tags that specify the 'key' attribute must not set 'type' to STATIC"); + } + } else if (StringUtils.isNotBlank(value)) { + if (type != CemiFieldDefinitionType.STATIC) { + throw new IllegalStateException( + "'field' tags that specify the 'value' attribute must set 'type' to STATIC"); + } + } else if (type == CemiFieldDefinitionType.STATIC) { + if (value == null) { + throw new IllegalStateException("'field' tags with 'type' of STATIC that are meant to have empty data " + + "must specify an empty 'value' attribute"); + } + } else { + throw new IllegalStateException("'field' tags must specify either the 'key' or 'value' attribute"); + } + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiOutputDefinition.java b/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiOutputDefinition.java new file mode 100644 index 0000000000..8fbcb96755 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiOutputDefinition.java @@ -0,0 +1,52 @@ +package edu.cornell.kfs.sys.batch.xml; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlType; + +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "cemiOutputDefinitionType", propOrder = { + "sheets" +}) +@XmlRootElement(name = "cemiOutputDefinition") +public class CemiOutputDefinition { + + @XmlElement(name = "sheet", required = true) + private List sheets; + + public List getSheets() { + if (sheets == null) { + sheets = new ArrayList<>(); + } + return sheets; + } + + public void setSheets(final List sheets) { + this.sheets = sheets; + } + + @Override + public boolean equals(final Object obj) { + return EqualsBuilder.reflectionEquals(obj, this); + } + + @Override + public int hashCode() { + return HashCodeBuilder.reflectionHashCode(this); + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this); + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiSheetDefinition.java b/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiSheetDefinition.java new file mode 100644 index 0000000000..350c319ef7 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/batch/xml/CemiSheetDefinition.java @@ -0,0 +1,86 @@ +package edu.cornell.kfs.sys.batch.xml; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlType; + +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "cemiSheetType", propOrder = { + "fields" +}) +@XmlRootElement(name = "sheet") +public class CemiSheetDefinition { + + @XmlElement(name = "field", required = true) + private List fields; + + @XmlAttribute(name = "name", required = true) + private String name; + + @XmlAttribute(name = "num-header-rows", required = true) + private int numHeaderRows; + + @XmlAttribute(name = "start-column-index", required = true) + private int startColumnIndex; + + public List getFields() { + if (fields == null) { + fields = new ArrayList<>(); + } + return fields; + } + + public void setFields(final List fields) { + this.fields = fields; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public int getNumHeaderRows() { + return numHeaderRows; + } + + public void setNumHeaderRows(final int numHeaderRows) { + this.numHeaderRows = numHeaderRows; + } + + public int getStartColumnIndex() { + return startColumnIndex; + } + + public void setStartColumnIndex(final int startColumnIndex) { + this.startColumnIndex = startColumnIndex; + } + + @Override + public boolean equals(final Object obj) { + return EqualsBuilder.reflectionEquals(obj, this); + } + + @Override + public int hashCode() { + return HashCodeBuilder.reflectionHashCode(this); + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this); + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/util/CemiUtils.java b/src/main/java/edu/cornell/kfs/sys/util/CemiUtils.java new file mode 100644 index 0000000000..f78642da9b --- /dev/null +++ b/src/main/java/edu/cornell/kfs/sys/util/CemiUtils.java @@ -0,0 +1,29 @@ +package edu.cornell.kfs.sys.util; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Locale; + +import org.apache.commons.lang3.StringUtils; + +import edu.cornell.kfs.sys.CUKFSConstants; + +public final class CemiUtils { + + private static final DateTimeFormatter FILE_DATE_TIME_FORMATTER = DateTimeFormatter + .ofPattern(CUKFSConstants.DATE_FORMAT_yyyyMMdd_HHmmss, Locale.US) + .withZone(ZoneId.of(CUKFSConstants.TIME_ZONE_US_EASTERN)); + + public static final String generateFileNameContainingDateTime( + final LocalDateTime dateTime, final String fileNamePrefix, final String fileExtension) { + final String dateTimeString = FILE_DATE_TIME_FORMATTER.format(dateTime); + return StringUtils.join(fileNamePrefix, dateTimeString, fileExtension); + } + + public static final String convertToBooleanValueForFileExtract(final boolean value) { + return Boolean.toString(value) + .toUpperCase(Locale.US); + } + +} diff --git a/src/main/java/edu/cornell/kfs/sys/util/CuSqlChunk.java b/src/main/java/edu/cornell/kfs/sys/util/CuSqlChunk.java index 42c6f119a3..ed0cf5b4a4 100644 --- a/src/main/java/edu/cornell/kfs/sys/util/CuSqlChunk.java +++ b/src/main/java/edu/cornell/kfs/sys/util/CuSqlChunk.java @@ -226,4 +226,12 @@ private static CuSqlChunk asSqlInCondition( } } + public static CuSqlChunk asSqlBetweenCondition(final String columnName, final int sqlType, + final Object rangeStartValue, final Object rangeEndValue) { + return new CuSqlChunk() + .append(columnName) + .append(" BETWEEN ").appendAsParameter(sqlType, rangeStartValue) + .append(" AND ").appendAsParameter(sqlType, rangeEndValue); + } + } diff --git a/src/main/java/edu/cornell/kfs/vnd/CemiVendorConstants.java b/src/main/java/edu/cornell/kfs/vnd/CemiVendorConstants.java new file mode 100644 index 0000000000..6b20f40aa5 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/CemiVendorConstants.java @@ -0,0 +1,60 @@ +package edu.cornell.kfs.vnd; + +import java.util.List; +import java.util.Map; + +import org.kuali.kfs.vnd.VendorConstants; +import org.kuali.kfs.vnd.VendorConstants.AddressTypes; + +import edu.cornell.kfs.vnd.CUVendorConstants.CUAddressTypes; + +public final class CemiVendorConstants { + + public static final String SUPPLIER_ID_FORMAT = "'SUP'000000"; + public static final String SUPPLIER_REFERENCE_ID_FORMAT = "ITH_{0}-{1}"; + public static final String ADDRESS_ID_FORMAT = "{0}_{1}_{2}"; + public static final int SUPPLIER_HEADER_ROWS_PER_SHEET = 6; + + public static final String SUPPLIER_OUTPUT_DEFINITION_FILE_PATH = "classpath:edu/cornell/kfs/vnd/batch/CemiSupplierExtractFileOutputDefinition.xml"; + public static final String SUPPLIER_TEMPLATE_FILE_PATH = "classpath:edu/cornell/kfs/vnd/batch/Supplier.xlsx"; + public static final String SUPPLIER_EXTRACT_FILENAME_PREFIX = "Supplier_ITH_"; + + public static final String DEFAULT_SUPPLIER_CATEGORY = "Foundation_Default"; + public static final String DEFAULT_PAYMENT_TYPE = "Check"; + public static final String DEFAULT_CURRENCY = "USD"; + public static final String DEFAULT_NAME_USAGE = "Reference"; + public static final String DEFAULT_ADDRESS_TYPE = "BUSINESS"; + + public static final String USA_EIN_TAX_TYPE = "USA-EIN"; + public static final String USA_SSN_TAX_TYPE = "USA-SSN"; + + public static final Map> ADDRESS_USES = Map.ofEntries( + Map.entry(AddressTypes.PURCHASE_ORDER, List.of("PROCUREMENT", "SHIPPING")), + Map.entry(AddressTypes.REMIT, List.of("REMIT")), + Map.entry(CUAddressTypes.TAX, List.of("TAX")) + ); + + public static final Map> ADDRESS_TENANTED_USES = Map.ofEntries( + Map.entry(AddressTypes.PURCHASE_ORDER, List.of("Procurement", "Shipping")), + Map.entry(AddressTypes.REMIT, List.of("Remit_To")), + Map.entry(CUAddressTypes.TAX, List.of("Tax")) + ); + + public static final Map TAX_ID_TYPES = Map.ofEntries( + Map.entry(VendorConstants.TAX_TYPE_FEIN, USA_EIN_TAX_TYPE), + Map.entry(VendorConstants.TAX_TYPE_SSN, USA_SSN_TAX_TYPE)); + + public static final class CemiQuerySettingsIds { + public static final String SUPPLIERS = "SUPPLIERS"; + } + + public static final class SupplierExtractSheets { + public static final String SUPPLIER = "Supplier"; + } + + public static final class TaxAuthorityFormTypes { + public static final String FORM_1099_MISC = "1099_MISC"; + public static final String FORM_1042S = "1042-S"; + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/CuVendorParameterConstants.java b/src/main/java/edu/cornell/kfs/vnd/CuVendorParameterConstants.java index c1527cde33..96fc94895f 100644 --- a/src/main/java/edu/cornell/kfs/vnd/CuVendorParameterConstants.java +++ b/src/main/java/edu/cornell/kfs/vnd/CuVendorParameterConstants.java @@ -12,4 +12,6 @@ public class CuVendorParameterConstants { public static final String WORKDAY_WEBSERVICE_CREDENTIAL_GROUP_CODE = "WORKDAYAPI"; public static final String WORKDAY_WEBSERVICE_CREDENTIAL_KEY = "workdayemployeessn-usernamepassword"; + public static final String CEMI_SUPPLIER_EXTRACT_DATE_RANGE = "CEMI_SUPPLIER_EXTRACT_DATE_RANGE"; + } diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/CreateCemiSupplierExtractStep.java b/src/main/java/edu/cornell/kfs/vnd/batch/CreateCemiSupplierExtractStep.java new file mode 100644 index 0000000000..c9fc89bee1 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/CreateCemiSupplierExtractStep.java @@ -0,0 +1,28 @@ +package edu.cornell.kfs.vnd.batch; + +import java.time.LocalDateTime; + +import org.kuali.kfs.sys.batch.AbstractStep; + +import edu.cornell.kfs.vnd.batch.service.CemiSupplierExtractService; + +public class CreateCemiSupplierExtractStep extends AbstractStep { + + private CemiSupplierExtractService cemiSupplierExtractService; + + @Override + public boolean execute(final String jobName, final LocalDateTime jobRunDate) throws InterruptedException { + cemiSupplierExtractService.resetState(); + cemiSupplierExtractService.initializeVendorActivityDateRangeSettings(); + cemiSupplierExtractService.populateListOfBaseVendorData(); + cemiSupplierExtractService.populateListOfInScopeVendors(); + cemiSupplierExtractService.generateIntermediateSupplierExtractData(jobRunDate); + cemiSupplierExtractService.generateSupplierExtractFile(jobRunDate); + return true; + } + + public void setCemiSupplierExtractService(final CemiSupplierExtractService cemiSupplierExtractService) { + this.cemiSupplierExtractService = cemiSupplierExtractService; + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/dto/CemiSupplier.java b/src/main/java/edu/cornell/kfs/vnd/batch/dto/CemiSupplier.java new file mode 100644 index 0000000000..e91d719864 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/dto/CemiSupplier.java @@ -0,0 +1,105 @@ +package edu.cornell.kfs.vnd.batch.dto; + +import java.text.MessageFormat; + +import org.apache.commons.lang3.StringUtils; +import org.kuali.kfs.sys.KFSConstants; +import org.kuali.kfs.vnd.businessobject.VendorDetail; +import org.kuali.kfs.vnd.businessobject.VendorHeader; + +import edu.cornell.kfs.sys.util.CemiUtils; +import edu.cornell.kfs.vnd.CUVendorConstants.VendorOwnershipCodes; +import edu.cornell.kfs.vnd.CemiVendorConstants; +import edu.cornell.kfs.vnd.CemiVendorConstants.TaxAuthorityFormTypes; + +@SuppressWarnings("deprecation") +public class CemiSupplier { + + private final VendorDetail vendorDetail; + private final String supplierId; + private final String supplierReferenceId; + private final String taxAuthorityFormType; + private final String taxIdType; + private final String taxIdValue; + + /* + * For POJO properties that need to go into the spreadsheet (and the future temp table), + * either create a related getter, or retrieve it from a Vendor Header/Detail getter. + * When using the latter, specify a nested "vendorHeader.propName" or "vendorDetail.propName" + * property in the XML definition. + * + * This particular POJO populates derived values via static methods and keeps such values immutable. + * If necessary, this object can be modified into a mutable one and/or a different mechanism could + * be implemented to compute the derived values. + */ + public CemiSupplier(final VendorDetail vendorDetail, final String supplierId) { + this.vendorDetail = vendorDetail; + this.supplierId = supplierId; + this.supplierReferenceId = buildSupplierReferenceId(vendorDetail); + this.taxAuthorityFormType = determineTaxAuthorityFormType(vendorDetail); + this.taxIdType = determineTaxIdType(vendorDetail); + this.taxIdValue = "TODO"; + } + + private static String buildSupplierReferenceId(final VendorDetail vendor) { + return MessageFormat.format(CemiVendorConstants.SUPPLIER_REFERENCE_ID_FORMAT, + Integer.toString(vendor.getVendorHeaderGeneratedIdentifier()), + Integer.toString(vendor.getVendorDetailAssignedIdentifier())); + } + + private static String determineTaxAuthorityFormType(final VendorDetail vendor) { + if (StringUtils.isNotBlank(vendor.getVendorHeader().getVendorW8TypeCode())) { + return TaxAuthorityFormTypes.FORM_1042S; + } else if (StringUtils.equals(vendor.getVendorHeader().getVendorOwnershipCode(), + VendorOwnershipCodes.INDIVIDUAL_OR_SOLE_PROPRIETOR_OR_SMLLC)) { + return TaxAuthorityFormTypes.FORM_1099_MISC; + } else { + return KFSConstants.EMPTY_STRING; + } + } + + private static String determineTaxIdType(final VendorDetail vendor) { + final String kfsTaxType = StringUtils.defaultString(vendor.getVendorHeader().getVendorTaxTypeCode()); + return CemiVendorConstants.TAX_ID_TYPES.get(kfsTaxType); + } + + public VendorDetail getVendorDetail() { + return vendorDetail; + } + + public VendorHeader getVendorHeader() { + return vendorDetail.getVendorHeader(); + } + + public String getSupplierId() { + return supplierId; + } + + public String getSupplierReferenceId() { + return supplierReferenceId; + } + + public String getTaxAuthorityFormType() { + return taxAuthorityFormType; + } + + public String getIrs1099SupplierFlag() { + return CemiUtils.convertToBooleanValueForFileExtract( + StringUtils.equals(taxAuthorityFormType, TaxAuthorityFormTypes.FORM_1099_MISC)); + } + + public String getTaxIdType() { + return taxIdType; + } + + public String getTaxIdValue() { + return taxIdValue; + } + + public String getTransactionTaxId() { + return CemiUtils.convertToBooleanValueForFileExtract( + StringUtils.isNoneBlank(taxIdType, taxIdValue) + && !StringUtils.equals(taxIdType, CemiVendorConstants.USA_SSN_TAX_TYPE)); + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/dto/CemiSupplierAddress.java b/src/main/java/edu/cornell/kfs/vnd/batch/dto/CemiSupplierAddress.java new file mode 100644 index 0000000000..8f361aa6b6 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/dto/CemiSupplierAddress.java @@ -0,0 +1,5 @@ +package edu.cornell.kfs.vnd.batch.dto; + +public class CemiSupplierAddress { + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierDataBuilder.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierDataBuilder.java new file mode 100644 index 0000000000..aac9eef16b --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierDataBuilder.java @@ -0,0 +1,13 @@ +package edu.cornell.kfs.vnd.batch.service; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + +import org.kuali.kfs.vnd.businessobject.VendorDetail; + +public interface CemiSupplierDataBuilder extends Closeable { + + void writeSupplierDataToIntermediateStorage(final Iterator vendors) throws IOException; + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierExtractService.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierExtractService.java new file mode 100644 index 0000000000..819576170f --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierExtractService.java @@ -0,0 +1,19 @@ +package edu.cornell.kfs.vnd.batch.service; + +import java.time.LocalDateTime; + +public interface CemiSupplierExtractService { + + void resetState(); + + void initializeVendorActivityDateRangeSettings(); + + void populateListOfBaseVendorData(); + + void populateListOfInScopeVendors(); + + void generateIntermediateSupplierExtractData(final LocalDateTime jobRunDate); + + void generateSupplierExtractFile(final LocalDateTime jobRunDate); + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierFileAppender.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierFileAppender.java new file mode 100644 index 0000000000..897b71c2a9 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/CemiSupplierFileAppender.java @@ -0,0 +1,13 @@ +package edu.cornell.kfs.vnd.batch.service; + +import java.io.IOException; + +import edu.cornell.kfs.sys.batch.service.impl.CemiExcelWriter; + +public interface CemiSupplierFileAppender { + + void populateSupplierFileFromIntermediateDataStorage(final CemiExcelWriter fileWriter) throws IOException; + + void cleanUpIntermediateStorage() throws IOException; + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierDataBuilderBase.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierDataBuilderBase.java new file mode 100644 index 0000000000..5906b6a5b5 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierDataBuilderBase.java @@ -0,0 +1,86 @@ +package edu.cornell.kfs.vnd.batch.service.impl; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.time.LocalDateTime; +import java.util.Iterator; + +import org.apache.commons.collections4.IteratorUtils; +import org.apache.commons.lang3.Validate; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.kuali.kfs.krad.util.ObjectUtils; +import org.kuali.kfs.vnd.businessobject.VendorDetail; + +import edu.cornell.kfs.sys.batch.xml.CemiFieldDefinition; +import edu.cornell.kfs.sys.batch.xml.CemiOutputDefinition; +import edu.cornell.kfs.vnd.CemiVendorConstants; +import edu.cornell.kfs.vnd.CemiVendorConstants.SupplierExtractSheets; +import edu.cornell.kfs.vnd.batch.dto.CemiSupplier; +import edu.cornell.kfs.vnd.batch.service.CemiSupplierDataBuilder; + +public abstract class CemiSupplierDataBuilderBase implements CemiSupplierDataBuilder { + + private static final Logger LOG = LogManager.getLogger(); + + protected final CemiOutputDefinition outputDefinition; + protected final LocalDateTime jobRunDate; + protected final boolean maskSensitiveData; + protected final DecimalFormat supplierIdFormatter; + protected int vendorCount; + + protected CemiSupplierDataBuilderBase(final CemiOutputDefinition outputDefinition, + final LocalDateTime jobRunDate, final boolean maskSensitiveData) { + Validate.notNull(outputDefinition, "outputDefinition cannot be null"); + Validate.notNull(jobRunDate, "jobRunDate cannot be null"); + this.outputDefinition = outputDefinition; + this.jobRunDate = jobRunDate; + this.maskSensitiveData = maskSensitiveData; + this.supplierIdFormatter = new DecimalFormat(CemiVendorConstants.SUPPLIER_ID_FORMAT); + } + + /* + * NOTE: It is assumed that when the iterator returns a parent vendor, the subsequent iterations + * will return ALL of its child vendors (if any) BEFORE returning the next unrelated parent vendor. + */ + @Override + public void writeSupplierDataToIntermediateStorage(final Iterator vendors) throws IOException { + for (final VendorDetail vendor : IteratorUtils.asIterable(vendors)) { + vendorCount++; + if (vendorCount % 1000 == 0) { + LOG.info("writeSupplierDataToIntermediateStorage, Writing {} Vendors and counting...", vendorCount); + } + final String supplierId = supplierIdFormatter.format(vendorCount); + final CemiSupplier supplier = new CemiSupplier(vendor, supplierId); + writeSupplierRow(supplier); + } + LOG.info("writeSupplierDataToIntermediateStorage, Finished writing {} Vendors", vendorCount); + } + + protected void writeSupplierRow(final CemiSupplier supplier) throws IOException { + writeDataToIntermediateStorage(SupplierExtractSheets.SUPPLIER, supplier); + } + + /* + * The subclass that writes the vendor data to the temp tables needs to implement this method. + * If desired, the implementation can keep connections/files/etc. open until close() is called. + * See the CSV implementation for an example. + */ + protected abstract void writeDataToIntermediateStorage( + final String sheetName, final Object rowObject) throws IOException; + + // The temp table implementation can use (or override) this method to retrieve the column value to be inserted. + protected String getFieldValue(final CemiFieldDefinition field, final Object rowObject) { + switch (field.getType()) { + case STATIC: + return field.getValue(); + + case STRING: + return (String) ObjectUtils.getPropertyValue(rowObject, field.getKey()); + + default: + throw new IllegalStateException("Unknown field type: " + field.getType()); + } + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierDataBuilderCsvImpl.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierDataBuilderCsvImpl.java new file mode 100644 index 0000000000..c030a80ee0 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierDataBuilderCsvImpl.java @@ -0,0 +1,86 @@ +package edu.cornell.kfs.vnd.batch.service.impl; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; + +import edu.cornell.kfs.sys.CUKFSConstants; +import edu.cornell.kfs.sys.CUKFSConstants.FileExtensions; +import edu.cornell.kfs.sys.batch.service.impl.CemiCsvWriter; +import edu.cornell.kfs.sys.batch.xml.CemiFieldDefinition; +import edu.cornell.kfs.sys.batch.xml.CemiOutputDefinition; +import edu.cornell.kfs.sys.batch.xml.CemiSheetDefinition; +import edu.cornell.kfs.sys.util.CemiUtils; + +public class CemiSupplierDataBuilderCsvImpl extends CemiSupplierDataBuilderBase { + + private final Map csvWriters; + private final Map sheetDefinitions; + + public CemiSupplierDataBuilderCsvImpl(final CemiOutputDefinition outputDefinition, final LocalDateTime jobRunDate, + final String baseFileDirectory, final boolean maskSensitiveData) throws IOException { + super(outputDefinition, jobRunDate, maskSensitiveData); + Validate.notBlank(baseFileDirectory, "baseFileDirectory cannot be blank"); + + this.sheetDefinitions = outputDefinition.getSheets().stream() + .collect(Collectors.toUnmodifiableMap(CemiSheetDefinition::getName, Function.identity())); + + final Map writers = new HashMap<>(); + CemiCsvWriter nextWriter = null; + boolean setupSucceeded = false; + + try { + for (final CemiSheetDefinition sheetDefinition : outputDefinition.getSheets()) { + final String sheetName = sheetDefinition.getName(); + final String fileName = CemiUtils.generateFileNameContainingDateTime( + jobRunDate, sheetName + CUKFSConstants.UNDERSCORE, FileExtensions.CSV); + final String fileNameWithPath = StringUtils.join(baseFileDirectory, CUKFSConstants.SLASH, fileName); + nextWriter = new CemiCsvWriter(fileNameWithPath); + writers.put(sheetName, nextWriter); + nextWriter = null; + } + this.csvWriters = Map.copyOf(writers); + setupSucceeded = true; + } finally { + if (!setupSucceeded) { + IOUtils.closeQuietly(nextWriter); + for (final CemiCsvWriter writer : writers.values()) { + IOUtils.closeQuietly(writer); + } + } + } + } + + @Override + public void close() throws IOException { + for (final CemiCsvWriter csvWriter : csvWriters.values()) { + IOUtils.closeQuietly(csvWriter); + } + } + + @Override + protected void writeDataToIntermediateStorage(String sheetName, Object rowObject) throws IOException { + final CemiSheetDefinition sheetDefinition = sheetDefinitions.get(sheetName); + final CemiCsvWriter csvWriter = csvWriters.get(sheetName); + Validate.validState(sheetDefinition != null, "Unexpected CEMI Supplier datasheet: %s", sheetName); + Validate.validState(csvWriter != null, "Unexpected non-writeable Supplier datasheet: %s", sheetName); + + final String[] csvRow = new String[sheetDefinition.getFields().size()]; + int fieldIndex = 0; + for (final CemiFieldDefinition field : sheetDefinition.getFields()) { + final String fieldValue = getFieldValue(field, rowObject); + csvRow[fieldIndex] = fieldValue; + fieldIndex++; + } + + csvWriter.writeNext(csvRow); + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierExtractServiceImpl.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierExtractServiceImpl.java new file mode 100644 index 0000000000..d956983384 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierExtractServiceImpl.java @@ -0,0 +1,241 @@ +package edu.cornell.kfs.vnd.batch.service.impl; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.text.ParseException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.Iterator; +import java.util.stream.Stream; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.kuali.kfs.core.api.datetime.DateTimeService; +import org.kuali.kfs.coreservice.framework.parameter.ParameterService; +import org.kuali.kfs.vnd.businessobject.VendorDetail; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import edu.cornell.kfs.core.api.util.CuCoreUtilities; +import edu.cornell.kfs.sys.CUKFSConstants; +import edu.cornell.kfs.sys.CUKFSConstants.FileExtensions; +import edu.cornell.kfs.sys.batch.CemiOutputDefinitionFileType; +import edu.cornell.kfs.sys.batch.service.impl.CemiExcelWriter; +import edu.cornell.kfs.sys.batch.xml.CemiOutputDefinition; +import edu.cornell.kfs.sys.util.CemiUtils; +import edu.cornell.kfs.vnd.CemiVendorConstants; +import edu.cornell.kfs.vnd.CuVendorParameterConstants; +import edu.cornell.kfs.vnd.batch.CreateCemiSupplierExtractStep; +import edu.cornell.kfs.vnd.batch.service.CemiSupplierExtractService; +import edu.cornell.kfs.vnd.dataaccess.CemiVendorDao; +import edu.cornell.kfs.vnd.dataaccess.CuVendorDao; + +public class CemiSupplierExtractServiceImpl implements CemiSupplierExtractService { + + private static final Logger LOG = LogManager.getLogger(); + + private String supplierFileCreationDirectory; + private String supplierFileExportDirectory; + private CuVendorDao cuVendorDao; + private CemiVendorDao cemiVendorDao; + private CemiOutputDefinitionFileType cemiOutputDefinitionFileType; + private ParameterService parameterService; + private DateTimeService dateTimeService; + + @Transactional(propagation = Propagation.REQUIRES_NEW) + @Override + public void resetState() { + LOG.info("resetState, Deleting the list of extractable Vendors from the previous run (if present)..."); + cemiVendorDao.clearExistingListOfBaseVendorData(); + cemiVendorDao.clearExistingListOfExtractableVendorIds(); + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + @Override + public void initializeVendorActivityDateRangeSettings() { + LOG.info("initializeVendorActivityDateRangeSettings, Setting from/to date range from parameter value..."); + final Collection parameterValues = parameterService.getParameterValuesAsString( + CreateCemiSupplierExtractStep.class, CuVendorParameterConstants.CEMI_SUPPLIER_EXTRACT_DATE_RANGE); + final String[] dateStrings = parameterValues.toArray(String[]::new); + Validate.validState(dateStrings.length == 2, "Parameter %s should have had 2 values, but had %s instead", + CuVendorParameterConstants.CEMI_SUPPLIER_EXTRACT_DATE_RANGE, dateStrings.length); + + final LocalDate fromDate = parseDate(dateStrings[0]); + final LocalDate toDate = parseDate(dateStrings[1]); + Validate.validState(fromDate.compareTo(toDate) <= 0, + "Parameter %s contained a 'from' date that is later than the 'to' date", + CuVendorParameterConstants.CEMI_SUPPLIER_EXTRACT_DATE_RANGE); + + cemiVendorDao.updateSupplierExtractQuerySettings(fromDate, toDate); + } + + private LocalDate parseDate(final String value) { + try { + return dateTimeService.convertToLocalDate(value); + } catch (final ParseException e) { + LOG.error("parseDate, failed to parse date string: {}", value, e); + throw new RuntimeException(e); + } + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + @Override + public void populateListOfBaseVendorData() { + LOG.info("populateListOfBaseVendorData, Preparing base Vendor data needed for subsequent Vendor query..."); + cemiVendorDao.prepareBaseVendorDataNeededForMainVendorIdQuery(); + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + @Override + public void populateListOfInScopeVendors() { + LOG.info("populateListOfInScopeVendors, Querying and storing the list of extractable Vendors..."); + cemiVendorDao.queryAndStoreVendorIdsForSupplierExtract(); + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + @Override + public void generateIntermediateSupplierExtractData(final LocalDateTime jobRunDate) { + LOG.info("generateIntermediateSupplierExtractData, Generating data rows for Supplier spreadsheet " + + "and placing in intermediate storage..."); + try { + final CemiOutputDefinition outputDefinition = getOutputDefinitionForSupplierExtract(); + generateSupplierExtractData(outputDefinition, jobRunDate); + } catch (final Exception e) { + LOG.error("generateIntermediateSupplierExtractData, Creation of Supplier Extract data failed", e); + throw new RuntimeException(e); + } + } + + private void generateSupplierExtractData(final CemiOutputDefinition outputDefinition, + final LocalDateTime jobRunDate) throws IOException { + try ( + // Replace this builder with a temp table implementation when ready. + final CemiSupplierDataBuilderCsvImpl dataBuilder = new CemiSupplierDataBuilderCsvImpl( + getOutputDefinitionForSupplierExtract(), jobRunDate, supplierFileCreationDirectory, false); + final Stream vendors = cuVendorDao.getVendorsForCemiSupplierExtractAsCloseableStream(); + ) { + final Iterator vendorsIterator = vendors.iterator(); + dataBuilder.writeSupplierDataToIntermediateStorage(vendorsIterator); + } + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + @Override + public void generateSupplierExtractFile(final LocalDateTime jobRunDate) { + try { + LOG.info("generateSupplierExtractFile, Starting creation of CEMI Supplier Extract file..."); + final String newFileName = CemiUtils.generateFileNameContainingDateTime( + jobRunDate, CemiVendorConstants.SUPPLIER_EXTRACT_FILENAME_PREFIX, FileExtensions.XLSX); + final File tempFile = qualifyAndGetFilePath(supplierFileCreationDirectory, newFileName); + final File finalFile = qualifyAndGetFilePath(supplierFileExportDirectory, newFileName); + Validate.validState(!tempFile.exists(), "Temporary file already exists: %s", newFileName); + Validate.validState(!finalFile.exists(), "Final file already exists: %s", newFileName); + + LOG.info("generateSupplierExtractFile, Copying template file..."); + copyTemplateFileTo(tempFile); + + LOG.info("generateSupplierExtractFile, Updating copied template with supplier data..."); + updateSupplierExtractFile(tempFile, jobRunDate); + + LOG.info("generateSupplierExtractFile, Moving file to export folder..."); + moveSupplierExtractFileToExportDirectory(tempFile, finalFile); + + LOG.info("generateSupplierExtractFile, Success! Created the following extract file: {}", newFileName); + } catch (final Exception e) { + LOG.error("generateSupplierExtractFile, Creation of Supplier Extract file failed", e); + throw new RuntimeException(e); + } + } + + private File qualifyAndGetFilePath(final String prefix, final String fileName) { + return new File(StringUtils.join(prefix, CUKFSConstants.SLASH, fileName)); + } + + private void copyTemplateFileTo(final File newFile) throws IOException { + try ( + final InputStream inputStream = CuCoreUtilities.getResourceAsStream( + CemiVendorConstants.SUPPLIER_TEMPLATE_FILE_PATH); + final OutputStream outputStream = new FileOutputStream(newFile); + ) { + IOUtils.copy(inputStream, outputStream); + } + } + + private void updateSupplierExtractFile(final File file, final LocalDateTime jobRunDate) + throws IOException, InvalidFormatException { + final CemiOutputDefinition outputDefinition = getOutputDefinitionForSupplierExtract(); + + try ( + final CemiExcelWriter writer = new CemiExcelWriter(outputDefinition, file); + ) { + // Replace this appender with a temp table implementation when ready. + final CemiSupplierFileAppenderCsvImpl supplierFileAppender = new CemiSupplierFileAppenderCsvImpl( + outputDefinition, jobRunDate, supplierFileCreationDirectory); + supplierFileAppender.populateSupplierFileFromIntermediateDataStorage(writer); + writer.commit(); + supplierFileAppender.cleanUpIntermediateStorage(); + } + } + + private CemiOutputDefinition getOutputDefinitionForSupplierExtract() throws IOException { + try ( + final InputStream inputStream = CuCoreUtilities.getResourceAsStream( + CemiVendorConstants.SUPPLIER_OUTPUT_DEFINITION_FILE_PATH); + ) { + final byte[] fileContents = IOUtils.toByteArray(inputStream); + return cemiOutputDefinitionFileType.parse(fileContents); + } + } + + private void moveSupplierExtractFileToExportDirectory(final File sourceFile, final File targetFile) { + try { + final Path creationFilePath = sourceFile.toPath(); + final Path exportFilePath = targetFile.toPath(); + Files.move(creationFilePath, exportFilePath, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + LOG.error("moveSupplierExtractFileToExportDirectory, Failed to move file to export directory", e); + throw new UncheckedIOException(e); + } + } + + public void setSupplierFileCreationDirectory(final String supplierFileCreationDirectory) { + this.supplierFileCreationDirectory = supplierFileCreationDirectory; + } + + public void setSupplierFileExportDirectory(final String supplierFileExportDirectory) { + this.supplierFileExportDirectory = supplierFileExportDirectory; + } + + public void setCuVendorDao(final CuVendorDao cuVendorDao) { + this.cuVendorDao = cuVendorDao; + } + + public void setCemiVendorDao(final CemiVendorDao cemiVendorDao) { + this.cemiVendorDao = cemiVendorDao; + } + + public void setCemiOutputDefinitionFileType(final CemiOutputDefinitionFileType cemiOutputDefinitionFileType) { + this.cemiOutputDefinitionFileType = cemiOutputDefinitionFileType; + } + + public void setParameterService(final ParameterService parameterService) { + this.parameterService = parameterService; + } + + public void setDateTimeService(final DateTimeService dateTimeService) { + this.dateTimeService = dateTimeService; + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierFileAppenderBase.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierFileAppenderBase.java new file mode 100644 index 0000000000..ca9d2a0271 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierFileAppenderBase.java @@ -0,0 +1,58 @@ +package edu.cornell.kfs.vnd.batch.service.impl; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.Iterator; +import java.util.stream.Stream; + +import org.apache.commons.collections4.IteratorUtils; +import org.apache.commons.lang3.Validate; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import edu.cornell.kfs.sys.batch.service.impl.CemiExcelWriter; +import edu.cornell.kfs.sys.batch.xml.CemiOutputDefinition; +import edu.cornell.kfs.sys.batch.xml.CemiSheetDefinition; +import edu.cornell.kfs.vnd.batch.service.CemiSupplierFileAppender; + +public abstract class CemiSupplierFileAppenderBase implements CemiSupplierFileAppender { + + private static final Logger LOG = LogManager.getLogger(); + + protected final CemiOutputDefinition outputDefinition; + protected final LocalDateTime jobRunDate; + + protected CemiSupplierFileAppenderBase(final CemiOutputDefinition outputDefinition, final LocalDateTime jobRunDate) { + Validate.notNull(outputDefinition, "outputDefinition cannot be null"); + Validate.notNull(jobRunDate, "jobRunDate cannot be null"); + this.outputDefinition = outputDefinition; + this.jobRunDate = jobRunDate; + } + + @Override + public void populateSupplierFileFromIntermediateDataStorage(final CemiExcelWriter fileWriter) throws IOException { + for (final CemiSheetDefinition sheetDefinition : outputDefinition.getSheets()) { + LOG.info("populateSupplierFileFromIntermediateDataStorage, Writing sheet: {}", sheetDefinition.getName()); + populateSheetFromIntermediateDataStorage(sheetDefinition, fileWriter); + } + } + + protected void populateSheetFromIntermediateDataStorage( + final CemiSheetDefinition sheetDefinition, final CemiExcelWriter fileWriter) throws IOException { + try ( + final Stream sheetData = getCloseableSheetDataStreamFromIntermediateStorage(sheetDefinition); + ) { + final String sheetName = sheetDefinition.getName(); + final Iterator sheetDataIterator = sheetData.iterator(); + for (final String[] sheetRow : IteratorUtils.asIterable(sheetDataIterator)) { + fileWriter.writeRow(sheetName, sheetRow); + } + } + } + + // This is what needs to be implemented in a subclass when switching storage from CSV to temp table. + // JDBCTemplate and/or CuOjbUtils should have methods for returning DB data as a stream. + protected abstract Stream getCloseableSheetDataStreamFromIntermediateStorage( + final CemiSheetDefinition sheetDefinition) throws IOException; + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierFileAppenderCsvImpl.java b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierFileAppenderCsvImpl.java new file mode 100644 index 0000000000..a6909ef994 --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/batch/service/impl/CemiSupplierFileAppenderCsvImpl.java @@ -0,0 +1,69 @@ +package edu.cornell.kfs.vnd.batch.service.impl; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; + +import edu.cornell.kfs.sys.CUKFSConstants; +import edu.cornell.kfs.sys.CUKFSConstants.FileExtensions; +import edu.cornell.kfs.sys.batch.service.impl.CemiCsvReader; +import edu.cornell.kfs.sys.batch.xml.CemiOutputDefinition; +import edu.cornell.kfs.sys.batch.xml.CemiSheetDefinition; +import edu.cornell.kfs.sys.util.CemiUtils; + +public class CemiSupplierFileAppenderCsvImpl extends CemiSupplierFileAppenderBase { + + private final String baseFilePath; + + public CemiSupplierFileAppenderCsvImpl(final CemiOutputDefinition outputDefinition, + final LocalDateTime jobRunDate, final String baseFilePath) { + super(outputDefinition, jobRunDate); + Validate.notBlank(baseFilePath, "baseFilePath cannot be blank"); + this.baseFilePath = baseFilePath; + } + + @Override + protected Stream getCloseableSheetDataStreamFromIntermediateStorage( + final CemiSheetDefinition sheetDefinition) throws IOException { + final File csvFile = getFileForIntermediateDataStorage(sheetDefinition.getName()); + Validate.validState(csvFile.exists(), "%s sheet CSV file not found", sheetDefinition.getName()); + + CemiCsvReader csvReader = null; + boolean setupSuccessful = false; + try { + csvReader = new CemiCsvReader(csvFile); + final CemiCsvReader csvReaderForOnCloseHandler = csvReader; + final Spliterator spliterator = Spliterators.spliteratorUnknownSize(csvReader.iterator(), 0); + final Stream sheetDataStream = StreamSupport.stream(() -> spliterator, 0, false) + .onClose(() -> IOUtils.closeQuietly(csvReaderForOnCloseHandler)); + setupSuccessful = true; + return sheetDataStream; + } finally { + if (!setupSuccessful) { + IOUtils.closeQuietly(csvReader); + } + } + } + + // Not necessarily needed; can be left empty in the temp table implementation. + @Override + public void cleanUpIntermediateStorage() throws IOException { + + } + + private final File getFileForIntermediateDataStorage(final String sheetName) { + final String fileName = CemiUtils.generateFileNameContainingDateTime( + jobRunDate, sheetName + CUKFSConstants.UNDERSCORE, FileExtensions.CSV); + final String fileNameWithPath = StringUtils.join(baseFilePath, CUKFSConstants.SLASH, fileName); + return new File(fileNameWithPath); + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/dataaccess/CemiVendorDao.java b/src/main/java/edu/cornell/kfs/vnd/dataaccess/CemiVendorDao.java new file mode 100644 index 0000000000..6da2a22a4e --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/dataaccess/CemiVendorDao.java @@ -0,0 +1,17 @@ +package edu.cornell.kfs.vnd.dataaccess; + +import java.time.LocalDate; + +public interface CemiVendorDao { + + void clearExistingListOfBaseVendorData(); + + void clearExistingListOfExtractableVendorIds(); + + void updateSupplierExtractQuerySettings(final LocalDate fromDate, final LocalDate toDate); + + void prepareBaseVendorDataNeededForMainVendorIdQuery(); + + void queryAndStoreVendorIdsForSupplierExtract(); + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/dataaccess/CuVendorDao.java b/src/main/java/edu/cornell/kfs/vnd/dataaccess/CuVendorDao.java index 5b952fe9e5..5432977de1 100644 --- a/src/main/java/edu/cornell/kfs/vnd/dataaccess/CuVendorDao.java +++ b/src/main/java/edu/cornell/kfs/vnd/dataaccess/CuVendorDao.java @@ -5,6 +5,7 @@ import java.util.stream.Stream; import org.kuali.kfs.krad.bo.BusinessObject; +import org.kuali.kfs.vnd.businessobject.VendorDetail; import org.kuali.kfs.vnd.dataaccess.VendorDao; import edu.cornell.kfs.vnd.businessobject.VendorWithTaxId; @@ -15,4 +16,6 @@ public interface CuVendorDao extends VendorDao { Stream getPotentialEmployeeVendorsAsCloseableStream(); + Stream getVendorsForCemiSupplierExtractAsCloseableStream(); + } diff --git a/src/main/java/edu/cornell/kfs/vnd/dataaccess/impl/CemiVendorDaoJdbcImpl.java b/src/main/java/edu/cornell/kfs/vnd/dataaccess/impl/CemiVendorDaoJdbcImpl.java new file mode 100644 index 0000000000..4855116baa --- /dev/null +++ b/src/main/java/edu/cornell/kfs/vnd/dataaccess/impl/CemiVendorDaoJdbcImpl.java @@ -0,0 +1,99 @@ +package edu.cornell.kfs.vnd.dataaccess.impl; + +import java.sql.Types; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.kuali.kfs.core.api.datetime.DateTimeService; + +import edu.cornell.kfs.sys.CUKFSConstants; +import edu.cornell.kfs.sys.util.CuSqlChunk; +import edu.cornell.kfs.sys.util.CuSqlQuery; +import edu.cornell.kfs.sys.util.CuSqlQueryPlatformAwareDaoBaseJdbc; +import edu.cornell.kfs.vnd.CemiVendorConstants.CemiQuerySettingsIds; +import edu.cornell.kfs.vnd.dataaccess.CemiVendorDao; + +public class CemiVendorDaoJdbcImpl extends CuSqlQueryPlatformAwareDaoBaseJdbc implements CemiVendorDao { + + private static final Logger LOG = LogManager.getLogger(); + + private DateTimeService dateTimeService; + + @Override + public void clearExistingListOfBaseVendorData() { + final CuSqlQuery query = CuSqlQuery.of("TRUNCATE TABLE KFS.CU_CEMI_VNDR_BASE_DATA_T"); + executeUpdate(query); + } + + @Override + public void clearExistingListOfExtractableVendorIds() { + final CuSqlQuery query = CuSqlQuery.of("TRUNCATE TABLE KFS.CU_CEMI_SPLR_EXTR_VNDR_T"); + executeUpdate(query); + } + + @Override + public void updateSupplierExtractQuerySettings(final LocalDate fromDate, final LocalDate toDate) { + final ZoneId easternTimeZone = ZoneId.of(CUKFSConstants.TIME_ZONE_US_EASTERN); + final LocalDate currentDate = dateTimeService.getLocalDateNow(); + + final ZonedDateTime fromDateTime = ZonedDateTime.of( + fromDate, LocalTime.of(0, 0, 0, 0), easternTimeZone); + final ZonedDateTime toDateTime = ZonedDateTime.of( + toDate, LocalTime.of(23, 59, 59, 0), easternTimeZone); + final ZonedDateTime startOfYear = ZonedDateTime.of( + LocalDate.of(currentDate.getYear(), 1, 1), LocalTime.of(0, 0, 0, 0), easternTimeZone); + final ZonedDateTime september1stOfPriorYear = ZonedDateTime.of( + LocalDate.of(currentDate.getYear() - 1, 9, 1), LocalTime.of(0, 0, 0, 0), easternTimeZone); + + final CuSqlQuery query = new CuSqlChunk() + .append("UPDATE KFS.CU_CEMI_QUERY_SETTINGS_T ") + .append("SET DATETIME_RANGE_FROM = ").appendAsParameter(Types.TIMESTAMP, fromDateTime) + .append(", DATETIME_RANGE_TO = ").appendAsParameter(Types.TIMESTAMP, toDateTime) + .append(", START_OF_YEAR = ").appendAsParameter(Types.TIMESTAMP, startOfYear) + .append(", SEPT_1_OF_PRIOR_YEAR = ").appendAsParameter(Types.TIMESTAMP, september1stOfPriorYear) + .append(" WHERE SETTINGS_ID = ").appendAsParameter(CemiQuerySettingsIds.SUPPLIERS) + .toQuery(); + + final int numRowsUpdated = executeUpdate(query); + if (numRowsUpdated != 1) { + LOG.error("updateSupplierExtractQuerySettings, Query should have updated 1 row, but it updated {} instead", + numRowsUpdated); + throw new RuntimeException("Failed to update settings for ID: " + CemiQuerySettingsIds.SUPPLIERS); + } + } + + @Override + public void prepareBaseVendorDataNeededForMainVendorIdQuery() { + final CuSqlQuery query = new CuSqlChunk() + .append("INSERT INTO KFS.CU_CEMI_VNDR_BASE_DATA_T ") + .append("(VNDR_HDR_GNRTD_ID, VNDR_DTL_ASND_ID, PAYEE_ID, VNDR_PARENT_IND, LAST_UPDT_TS) ") + .append("SELECT ") + .append("VNDR_HDR_GNRTD_ID, VNDR_DTL_ASND_ID, PAYEE_ID, VNDR_PARENT_IND, LAST_UPDT_TS ") + .append("FROM KFS.CU_CEMI_VNDR_BASE_DATA_SRC_V") + .toQuery(); + + final int numRowsInserted = executeUpdate(query); + LOG.info("prepareBaseVendorDataNeededForMainVendorIdQuery, Found {} vendors for main query", numRowsInserted); + } + + @Override + public void queryAndStoreVendorIdsForSupplierExtract() { + final CuSqlQuery query = new CuSqlChunk() + .append("INSERT INTO KFS.CU_CEMI_SPLR_EXTR_VNDR_T (VNDR_HDR_GNRTD_ID, VNDR_DTL_ASND_ID) ") + .append("SELECT VNDR_HDR_GNRTD_ID, VNDR_DTL_ASND_ID ") + .append("FROM KFS.PUR_CEMI_VNDR_EXTRACT_V") + .toQuery(); + + final int numRowsInserted = executeUpdate(query); + LOG.info("queryAndStoreVendorIdsForSupplierExtract, Found {} vendors to extract", numRowsInserted); + } + + public void setDateTimeService(final DateTimeService dateTimeService) { + this.dateTimeService = dateTimeService; + } + +} diff --git a/src/main/java/edu/cornell/kfs/vnd/dataaccess/impl/CuVendorDaoOjb.java b/src/main/java/edu/cornell/kfs/vnd/dataaccess/impl/CuVendorDaoOjb.java index 8e3b09f54b..2ef2bf728d 100644 --- a/src/main/java/edu/cornell/kfs/vnd/dataaccess/impl/CuVendorDaoOjb.java +++ b/src/main/java/edu/cornell/kfs/vnd/dataaccess/impl/CuVendorDaoOjb.java @@ -273,6 +273,30 @@ private VendorWithTaxId mapToVendorWithTaxId(final Object[] queryResultRow) { return vendor; } + @Override + public Stream getVendorsForCemiSupplierExtractAsCloseableStream() { + final String vendorIdCondition = "(A0.VNDR_HDR_GNRTD_ID, A0.VNDR_DTL_ASND_ID) IN (" + + "SELECT VNDR_HDR_GNRTD_ID, VNDR_DTL_ASND_ID FROM KFS.CU_CEMI_SPLR_EXTR_VNDR_T)"; + final Criteria criteria = new Criteria(); + criteria.addSql(vendorIdCondition); + + /* + * NOTE: The sort order below is crucial to simplify processing the Vendors in a streaming manner. + * When iterating over the Vendors below, a parent Vendor will be immediately followed + * by its children BEFORE the next parent Vendor is encountered. That way, when the processing code, + * iterates over the data but needs to populate child Vendor data based on what's in its parent, + * only a single parent Vendor needs its reference kept short-term. + */ + final QueryByCriteria query = new QueryByCriteria(VendorDetail.class, criteria); + query.addOrderByAscending(KFSPropertyConstants.VENDOR_HEADER_GENERATED_ID); + query.addOrderByDescending(VendorPropertyConstants.VENDOR_PARENT_INDICATOR); + query.addOrderByAscending(KFSPropertyConstants.VENDOR_DETAIL_ASSIGNED_ID); + + return CuOjbUtils.buildCloseableStreamForQueryResults( + VendorDetail.class, + () -> getPersistenceBrokerTemplate().getIteratorByQuery(query)); + } + @Override public DatabasePlatform getDbPlatform() { return dbPlatform; diff --git a/src/main/java/org/kuali/kfs/vnd/dataaccess/impl/VendorDaoImpl.java b/src/main/java/org/kuali/kfs/vnd/dataaccess/impl/VendorDaoImpl.java index 03dd08a694..30c4215708 100644 --- a/src/main/java/org/kuali/kfs/vnd/dataaccess/impl/VendorDaoImpl.java +++ b/src/main/java/org/kuali/kfs/vnd/dataaccess/impl/VendorDaoImpl.java @@ -719,4 +719,9 @@ public Stream getPotentialEmployeeVendorsAsCloseableStream() { return ((CuVendorDao) vendorDaoOjb).getPotentialEmployeeVendorsAsCloseableStream(); } + @Override + public Stream getVendorsForCemiSupplierExtractAsCloseableStream() { + return ((CuVendorDao) vendorDaoOjb).getVendorsForCemiSupplierExtractAsCloseableStream(); + } + } diff --git a/src/main/resources/CU-ApplicationResources.properties b/src/main/resources/CU-ApplicationResources.properties index 6fe195cede..6630695c4e 100755 --- a/src/main/resources/CU-ApplicationResources.properties +++ b/src/main/resources/CU-ApplicationResources.properties @@ -722,6 +722,8 @@ errors.reject.invoice.po.vendor.mismatch=Vendor from Kuali PO does not match wit error.cam.building.room.combination.invalid=Invalid Room #%s for Building Code %s error.cam.building.code.invalid=Invalid Building Code %s +message.batchUpload.title.cemiOutputDefinition=CEMI Output Definition Batch Upload + error.uploadFile.extension=File '{0}' is not an allowed attachment type. error.paymentrequest.at.least.one.must.be.entered=At least one {0} must be entered. diff --git a/src/main/resources/edu/cornell/kfs/sys/batch/cemiOutputDefinition.xsd b/src/main/resources/edu/cornell/kfs/sys/batch/cemiOutputDefinition.xsd new file mode 100644 index 0000000000..59dd08cce1 --- /dev/null +++ b/src/main/resources/edu/cornell/kfs/sys/batch/cemiOutputDefinition.xsd @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/edu/cornell/kfs/sys/cu-spring-sys.xml b/src/main/resources/edu/cornell/kfs/sys/cu-spring-sys.xml index c650692376..bf99107c32 100644 --- a/src/main/resources/edu/cornell/kfs/sys/cu-spring-sys.xml +++ b/src/main/resources/edu/cornell/kfs/sys/cu-spring-sys.xml @@ -488,4 +488,15 @@ p:dataDictionaryService-ref="dataDictionaryService" p:configurationService-ref="configurationService"/> + + + diff --git a/src/main/resources/edu/cornell/kfs/vnd/batch/CemiSupplierExtractFileOutputDefinition.xml b/src/main/resources/edu/cornell/kfs/vnd/batch/CemiSupplierExtractFileOutputDefinition.xml new file mode 100644 index 0000000000..f2c5e2ad26 --- /dev/null +++ b/src/main/resources/edu/cornell/kfs/vnd/batch/CemiSupplierExtractFileOutputDefinition.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/edu/cornell/kfs/vnd/batch/Supplier.xlsx b/src/main/resources/edu/cornell/kfs/vnd/batch/Supplier.xlsx new file mode 100644 index 0000000000..520cf18b36 Binary files /dev/null and b/src/main/resources/edu/cornell/kfs/vnd/batch/Supplier.xlsx differ diff --git a/src/main/resources/edu/cornell/kfs/vnd/cu-spring-vnd.xml b/src/main/resources/edu/cornell/kfs/vnd/cu-spring-vnd.xml index 1e50751a8f..7d8daf83dd 100644 --- a/src/main/resources/edu/cornell/kfs/vnd/cu-spring-vnd.xml +++ b/src/main/resources/edu/cornell/kfs/vnd/cu-spring-vnd.xml @@ -32,6 +32,7 @@ vendorInactivateConvertBatchJob createVendorEmployeeComparisonSearchFileJob processVendorEmployeeComparisonResultFileJob + createCemiSupplierExtractJob @@ -42,6 +43,8 @@ ${staging.directory}/vnd/emplCompareWorkday/outbound ${staging.directory}/vnd/emplCompareWorkday/outbound/being-written ${staging.directory}/vnd/emplCompareWorkday/result + ${staging.directory}/vnd/cemiSupplierExtract/outbound + ${staging.directory}/vnd/cemiSupplierExtract/outbound/being-written @@ -271,4 +274,36 @@ p:parameterService-ref="parameterService" p:webServiceCredentialService-ref="webServiceCredentialService"/> + + + + + + + + + + + + + + + + \ No newline at end of file