From d4c751395c366a8b32425ec3a8c1ae3b80e1720b Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Thu, 28 Aug 2025 10:47:19 -0700 Subject: [PATCH 1/2] added data source column and header --- .../controller/ElrReportsController.java | 7 ++ .../rawmessage/dto/RawElrDto.java | 1 + .../rawmessage/service/RawElrService.java | 1 + .../report/repository/model/RawElrModel.java | 70 ++----------------- .../db/changelog/dataingest-changelog.yaml | 8 +++ .../db/dataingest/tables/di-service-004.sql | 13 ++++ .../ElrReportsControllerMockTest.java | 8 +-- .../controller/ElrReportsControllerTest.java | 4 ++ 8 files changed, 45 insertions(+), 67 deletions(-) create mode 100644 data-ingestion-service/src/main/resources/db/dataingest/tables/di-service-004.sql diff --git a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsController.java b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsController.java index d85e73296..9d3ad44ac 100644 --- a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsController.java +++ b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsController.java @@ -73,6 +73,11 @@ public ElrReportsController(RawElrService rawELRService, description = "To use batch job(1) or RTI(2)", required = false, schema = @Schema(type = "string")), + @Parameter(in = ParameterIn.HEADER, + name = "dataSource", + description = "Data origin, for example if data originated from Rhapsody - Rhapsody can be specified for the value. The default value is API", + required = false, + schema = @Schema(type = "string")), @Parameter(in = ParameterIn.HEADER, name = "customMapper", description = "The optional custom mapper field that find and replaces the text in the ELR message." + @@ -83,6 +88,7 @@ public ElrReportsController(RawElrService rawELRService, @PostMapping(consumes = MediaType.TEXT_PLAIN_VALUE, path = "/api/elrs") public ResponseEntity save(@RequestBody final String payload, @RequestHeader("msgType") String type, @RequestHeader(name = "version", defaultValue = "1") String version, + @RequestHeader(name = "dataSource", defaultValue = "API") String dataSource, @RequestHeader(name="customMapper", defaultValue = "") String customMapper) throws KafkaProducerException { if (type.isEmpty()) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Required headers should not be null"); @@ -91,6 +97,7 @@ public ResponseEntity save(@RequestBody final String payload, @RequestHe } RawElrDto rawElrDto = new RawElrDto(); + rawElrDto.setDataSource(dataSource); customMetricsBuilder.incrementMessagesProcessed(); if (type.equalsIgnoreCase(HL7_ELR)) { diff --git a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/dto/RawElrDto.java b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/dto/RawElrDto.java index 43331675c..044805c1c 100644 --- a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/dto/RawElrDto.java +++ b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/dto/RawElrDto.java @@ -16,6 +16,7 @@ public class RawElrDto { private String type; private String payload; private String version; + private String dataSource; private Boolean validationActive = false; private String customMapper; } diff --git a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/service/RawElrService.java b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/service/RawElrService.java index b3eba565a..47ce843a9 100644 --- a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/service/RawElrService.java +++ b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/rawmessage/service/RawElrService.java @@ -162,6 +162,7 @@ private List createRawElrModelsForBatch(List hl7Messages,Ra for(String hl7Message : hl7Messages) { RawElrModel rawElrModel=convert(rawElrDto); rawElrModel.setPayload(hl7Message); + rawElrModel.setDataSource(rawElrDto.getDataSource()); rawElrModels.add(rawElrModel); } return rawElrModels; diff --git a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/report/repository/model/RawElrModel.java b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/report/repository/model/RawElrModel.java index e06c83cc3..7abaab4ac 100644 --- a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/report/repository/model/RawElrModel.java +++ b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/report/repository/model/RawElrModel.java @@ -1,12 +1,16 @@ package gov.cdc.dataingestion.report.repository.model; import jakarta.persistence.*; +import lombok.Getter; +import lombok.Setter; import org.hibernate.annotations.GenericGenerator; import java.sql.Timestamp; @Entity @Table(name = "elr_raw") +@Getter +@Setter /** 1118 - require constructor complaint 125 - comment complaint @@ -26,6 +30,9 @@ public class RawElrModel { private String type; private String payload; + @Column(name = "data_source") + private String dataSource; + @Column(name = "version") private String version; @@ -41,67 +48,4 @@ public class RawElrModel { @Column(name = "updated_by") private String updatedBy; - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getPayload() { - return payload; - } - - public void setPayload(String payload) { - this.payload = payload; - } - - public Timestamp getCreatedOn() { - return createdOn; - } - - public void setCreatedOn(Timestamp createdOn) { - this.createdOn = createdOn; - } - - public Timestamp getUpdatedOn() { - return updatedOn; - } - - public void setUpdatedOn(Timestamp updatedOn) { - this.updatedOn = updatedOn; - } - - public String getCreatedBy() { - return createdBy; - } - - public void setCreatedBy(String createdBy) { - this.createdBy = createdBy; - } - - public String getUpdatedBy() { - return updatedBy; - } - - public void setUpdatedBy(String updatedBy) { - this.updatedBy = updatedBy; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } } diff --git a/data-ingestion-service/src/main/resources/db/changelog/dataingest-changelog.yaml b/data-ingestion-service/src/main/resources/db/changelog/dataingest-changelog.yaml index 6399fbf45..fb8ffc464 100644 --- a/data-ingestion-service/src/main/resources/db/changelog/dataingest-changelog.yaml +++ b/data-ingestion-service/src/main/resources/db/changelog/dataingest-changelog.yaml @@ -33,4 +33,12 @@ databaseChangeLog: changes: - sqlFile: path: db/dataingest/tables/di-service-003.sql + splitStatements: false + - changeSet: + id: 5 + author: liquibase + runOnChange: true + changes: + - sqlFile: + path: db/dataingest/tables/di-service-004.sql splitStatements: false \ No newline at end of file diff --git a/data-ingestion-service/src/main/resources/db/dataingest/tables/di-service-004.sql b/data-ingestion-service/src/main/resources/db/dataingest/tables/di-service-004.sql new file mode 100644 index 000000000..16883e74c --- /dev/null +++ b/data-ingestion-service/src/main/resources/db/dataingest/tables/di-service-004.sql @@ -0,0 +1,13 @@ +USE NBS_DataIngest; +GO + +IF NOT EXISTS ( + SELECT 1 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = 'elr_raw' + AND COLUMN_NAME = 'data_source' +) + BEGIN + ALTER TABLE elr_raw + ADD data_source NVARCHAR(255) NULL; + END diff --git a/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerMockTest.java b/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerMockTest.java index 6d5311fe9..a333a3a0f 100644 --- a/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerMockTest.java +++ b/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerMockTest.java @@ -58,7 +58,7 @@ void testSave_HL7_ELR_Type() throws KafkaProducerException { when(rawELRService.submissionElr(any(RawElrDto.class))).thenReturn(expectedResponse); - ResponseEntity response = elrReportsController.save(payload, type, version,""); + ResponseEntity response = elrReportsController.save(payload, type, version,"", ""); assertEquals(HttpStatus.OK, response.getStatusCode()); assertEquals(expectedResponse, response.getBody()); @@ -74,7 +74,7 @@ void testSave_XML_ELR_Type() throws KafkaProducerException { when(rawELRService.submissionElrXml(any(RawElrDto.class))).thenReturn(expectedResponse); - ResponseEntity response = elrReportsController.save(payload, type, version,""); + ResponseEntity response = elrReportsController.save(payload, type, version,"", ""); assertEquals(HttpStatus.OK, response.getStatusCode()); assertEquals(expectedResponse, response.getBody()); @@ -88,7 +88,7 @@ void testSave_Invalid_Type() { String version = "1"; assertThrows(ResponseStatusException.class, () -> - elrReportsController.save(payload, type, version,"")); + elrReportsController.save(payload, type, version,"", "")); } @Test @@ -98,7 +98,7 @@ void testSave_Missing_Type() { String version = "1"; assertThrows(ResponseStatusException.class, () -> - elrReportsController.save(payload, type, version,"")); + elrReportsController.save(payload, type, version,"", "")); } @Test diff --git a/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerTest.java b/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerTest.java index c84792e04..1e417f621 100644 --- a/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerTest.java +++ b/data-ingestion-service/src/test/java/gov/cdc/dataingestion/rawmessage/controller/ElrReportsControllerTest.java @@ -42,6 +42,7 @@ void testSaveHL7Message() throws Exception { String messageType = "HL7"; mockMvc.perform(MockMvcRequestBuilders.post("/api/elrs") .header("msgType", messageType) + .header("dataSource", "API") .contentType("text/plain") .content(hl7Payload) .with(SecurityMockMvcRequestPostProcessors.jwt())) @@ -53,6 +54,7 @@ void testSaveHL7Message() throws Exception { rawElrDto.setValidationActive(true); rawElrDto.setVersion("1"); rawElrDto.setCustomMapper(""); + rawElrDto.setDataSource("API"); verify(rawELRService).submissionElr(rawElrDto); } @@ -63,6 +65,7 @@ void testSaveElrXmlMessage() throws Exception { String messageType = "HL7-XML"; mockMvc.perform(MockMvcRequestBuilders.post("/api/elrs") .header("msgType", messageType) + .header("dataSource", "API") .contentType("text/plain") .content(xmlPayload) .with(SecurityMockMvcRequestPostProcessors.jwt())) @@ -74,6 +77,7 @@ void testSaveElrXmlMessage() throws Exception { rawElrDto.setValidationActive(true); rawElrDto.setVersion("1"); rawElrDto.setCustomMapper(null); + rawElrDto.setDataSource("API"); verify(rawELRService).submissionElrXml(rawElrDto); } From e233575d8d07c9b142dd430055e9ce45531d839f Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Thu, 28 Aug 2025 11:15:49 -0700 Subject: [PATCH 2/2] added data source to status endpoint --- .../cdc/dataingestion/reportstatus/model/RawMessageStatus.java | 1 + .../dataingestion/reportstatus/service/ReportStatusService.java | 1 + 2 files changed, 2 insertions(+) diff --git a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/model/RawMessageStatus.java b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/model/RawMessageStatus.java index 5c68260be..27f64283d 100644 --- a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/model/RawMessageStatus.java +++ b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/model/RawMessageStatus.java @@ -15,6 +15,7 @@ public class RawMessageStatus { private String rawMessageId; // private String rawPayload; + private String dataSource; private String rawCreatedBy; private String rawCreatedOn; private String rawPipeLineStatus; diff --git a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/service/ReportStatusService.java b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/service/ReportStatusService.java index 3d0e061be..b43addb1b 100644 --- a/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/service/ReportStatusService.java +++ b/data-ingestion-service/src/main/java/gov/cdc/dataingestion/reportstatus/service/ReportStatusService.java @@ -72,6 +72,7 @@ public List getMessageStatus(String rawMessageID) { msgStatus.getRawInfo().setRawCreatedBy(rawMessageData.get().getCreatedBy()); msgStatus.getRawInfo().setRawCreatedOn(TimeStampHelper.convertTimestampToString(rawMessageData.get().getCreatedOn())); msgStatus.getRawInfo().setRawPipeLineStatus(MSG_STATUS_SUCCESS); + msgStatus.getRawInfo().setDataSource(rawMessageData.get().getDataSource()); if (rawMessageData.get().getType().equalsIgnoreCase(HL7_ELR)) { Optional validatedMessageData = iValidatedELRRepository.findByRawId(msgStatus.getRawInfo().getRawMessageId());