Skip to content

#272 MetaData Read를 Paging을 통해 최적화 #273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LDEV4966 wants to merge 7 commits into
base: dev
Choose a base branch
from
Open

#272 MetaData Read를 Paging을 통해 최적화 #273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2079f88
Feat : CustomizedMetaDataRepository 내 findByProjectIdWithPagingAndFil…
LDEV4966 May 13, 2022
1e8a7b3
Feat : Request Parm 통한 page&size 동적 처리
LDEV4966 May 15, 2022
9c40bb9
Fix : 머신러닝 테스트 코드 오류 수정
BEOKS May 17, 2022
b5031f9
Merge branch 'dev' into DSM-272
BEOKS May 18, 2022
5cfe29f
Fix : findByProjectIdWithPaging 기반으로 findByProjectId TestCode 수정
LDEV4966 May 18, 2022
2483b93
Fix :기존에 제거한 findByProjectId 기능을 원상복구
LDEV4966 May 19, 2022
4c72ccf
Refactor : MetaDataService.java
BEOKS May 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions server/DSMP/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.projectlombok:lombok'
implementation 'org.springframework.boot:spring-boot-starter-security'
implementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
implementation 'org.hamcrest:hamcrest-core:2.2'
annotationProcessor 'org.projectlombok:lombok'// lombok
implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' // mongoDB
implementation 'com.fasterxml.jackson.core:jackson-databind' //jackson
Expand Down
2 changes: 1 addition & 1 deletion .../DSMP/src/main/java/com/knuipalab/dsmp/http/httpResponse/success/SuccessDataResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class SuccessDataResponse<T> extends BasicResponse {

private LocalDateTime timestamp ;

private int count;
private int count; // 바디 값이 리스트인 경우 리스트 개수

private int status;

Expand Down
3 changes: 3 additions & 0 deletions server/DSMP/src/main/java/com/knuipalab/dsmp/metadata/CustomizedMetaDataRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package com.knuipalab.dsmp.metadata;

import org.springframework.data.domain.Page;

import java.util.HashMap;

public interface CustomizedMetaDataRepository {
void updateType(String metadataId, String type);
void setMalignancyClassification(String metadataId, HashMap classificationSet);
Page<MetaData> findByProjectIdWithPaging(String projectId, int page, int size, HashMap parmMap);
}
41 changes: 41 additions & 0 deletions server/DSMP/src/main/java/com/knuipalab/dsmp/metadata/CustomizedMetaDataRepositoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
package com.knuipalab.dsmp.metadata;

import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.data.support.PageableExecutionUtils;
import org.springframework.util.Assert;


import java.util.HashMap;
import java.util.Iterator;
import java.util.List;

public class CustomizedMetaDataRepositoryImpl implements CustomizedMetaDataRepository{

Expand Down Expand Up @@ -35,4 +43,37 @@ public void setMalignancyClassification(String metadataId, HashMap classificatio
});
mongoTemplate.updateFirst(query,update,"metadata");
}

// Iterator<String> keys = parmMap.keySet().iterator();
// while (keys.hasNext()){
// String key = keys.next();
// System.out.println(key);
// }


@Override
public Page<MetaData> findByProjectIdWithPaging(String projectId,int page, int size, HashMap parmMap) {

Pageable pageable = PageRequest.of(page,size, Sort.unsorted());

Query query = new Query()
.with(pageable)
.skip(pageable.getPageSize() * pageable.getPageNumber()) // offset
.limit(pageable.getPageSize());

//Add Filtered
query.addCriteria(Criteria.where("projectId").is(projectId));

List<MetaData> filteredMetaData = mongoTemplate.find(query, MetaData.class, "metadata");
Page<MetaData> metaDataPage = PageableExecutionUtils.getPage(
filteredMetaData,
pageable,
() -> mongoTemplate.count(query.skip(-1).limit(-1),MetaData.class)
// query.skip(-1).limit(-1)의 이유는 현재 쿼리가 페이징 하려고 하는 offset 까지만 보기에 이를 맨 처음부터 끝까지러 set 해줘 정확한 도큐먼트 개수를 구한다.
);

return metaDataPage;
}


}
15 changes: 12 additions & 3 deletions server/DSMP/src/main/java/com/knuipalab/dsmp/metadata/MetaDataApiController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@

import org.bson.Document;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import java.util.HashMap;
import java.util.List;

@RestController
Expand All @@ -20,16 +21,23 @@ public class MetaDataApiController {

// projectId가 같은 metadata를 모두 반환
@GetMapping("api/MetaData/{projectId}")
public ResponseEntity< ? extends BasicResponse> findByProjectId(@PathVariable String projectId, MultipartFile multipartFile){
public ResponseEntity< ? extends BasicResponse> findByProjectId(@PathVariable String projectId){
List<MetaDataResponseDto> metaDataResponseDtos = metaDataService.findByProjectId(projectId);
return ResponseEntity.ok().body(new SuccessDataResponse<List<MetaDataResponseDto>>(metaDataResponseDtos));
}

// projectId가 같은 metadata를 반환 with paging
@GetMapping("api/MetaData/{projectId}/pagination")
public ResponseEntity< ? extends BasicResponse> findByProjectIdWithPaging(@PathVariable String projectId, @RequestParam HashMap<String,Object> parmMap){
Page<MetaData> metaDataPage = metaDataService.findByProjectIdWithPaging(projectId,parmMap);
return ResponseEntity.ok().body(new SuccessDataResponse<Page<MetaData>>(metaDataPage));
}

// metatdata를 projectId와 함께 저장
@PostMapping("api/MetaData/{projectId}")
public ResponseEntity<? extends BasicResponse> insert(@PathVariable String projectId,@RequestBody Document body){
MetaDataCreateRequestDto metaDataCreateRequestDto = new MetaDataCreateRequestDto(projectId,body);
metaDataService.insert(metaDataCreateRequestDto);
metaDataService.save(metaDataCreateRequestDto);
return ResponseEntity.ok().body(new SuccessResponse());
}

Expand Down Expand Up @@ -63,4 +71,5 @@ public ResponseEntity<? extends BasicResponse> deleteById(@PathVariable String
return ResponseEntity.ok().body(new SuccessResponse());
}


}
89 changes: 59 additions & 30 deletions server/DSMP/src/main/java/com/knuipalab/dsmp/metadata/MetaDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
import com.knuipalab.dsmp.http.httpResponse.error.handler.exception.MetaDataNotFoundException;
import com.knuipalab.dsmp.patient.PatientService;
import com.knuipalab.dsmp.project.ProjectService;
import org.bson.Document;
import lombok.Getter;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;

Expand All @@ -26,63 +28,90 @@ public class MetaDataService {
@Autowired
private PatientService patientService;

private void throwGlobalExceptionWhenProjectNotExist(String projectId){
projectService.findById(projectId);
}
@Transactional (readOnly = true)
public List<MetaDataResponseDto> findByProjectId(String projectId) {

projectService.findById(projectId);

throwGlobalExceptionWhenProjectNotExist(projectId);
return metaDataRepository.findByProjectId(projectId).stream()
.map( metaData -> new MetaDataResponseDto(metaData) )
.collect(Collectors.toList());
}
@Getter
public class PageRequestConfiguration{
int pageNumber;
int size;
PageRequestConfiguration(HashMap parmMap,int defaultPageNumber,int defaultSize){
if(parmMap.containsKey("page")){
this.pageNumber = Integer.parseInt(parmMap.get("page").toString());
}
else{
this.pageNumber=defaultPageNumber;
}
if(parmMap.containsKey("size")){
this.size = Integer.parseInt(parmMap.get("size").toString());
}
else{
this.size=defaultSize;
}
}
}
@Transactional (readOnly = true)
public Page<MetaData> findByProjectIdWithPaging(String projectId, HashMap httpRequestParameterMap) {

PageRequestConfiguration pageRequestConfiguration=new PageRequestConfiguration(httpRequestParameterMap,0,20);
if(httpRequestParameterMap.containsKey("page")){
httpRequestParameterMap.remove("page");
}
if(httpRequestParameterMap.containsKey("size")){
httpRequestParameterMap.remove("size");
}
throwGlobalExceptionWhenProjectNotExist(projectId);
return metaDataRepository.findByProjectIdWithPaging(projectId,
pageRequestConfiguration.getPageNumber(), pageRequestConfiguration.getSize(), httpRequestParameterMap);

}

@Transactional
public void insert(MetaDataCreateRequestDto metaDataCreateRequestDto){
public void save(MetaDataCreateRequestDto metaDataCreateRequestDto){

projectService.findById(metaDataCreateRequestDto.getProjectId()); // 존재하는 프로젝트 id인지 확인.
throwGlobalExceptionWhenProjectNotExist(metaDataCreateRequestDto.getProjectId());

MetaData metaData = new MetaData().builder()
.projectId(metaDataCreateRequestDto.getProjectId())
.body(metaDataCreateRequestDto.getBody()).build();

patientService.addProjectCount(metaData.getPatientIdFromBody()); // patient 처리.
patientService.addProjectCount(metaData.getPatientIdFromBody());

metaDataRepository.save(metaData); // 저장
metaDataRepository.save(metaData);
}

@Transactional
public void insertAllByMetaDataList(MetaDataCreateAllRequestDto metaDataCreateAllRequestDto){

List<MetaData> metaDataList=new ArrayList<MetaData>();

projectService.findById(metaDataCreateAllRequestDto.getProjectId()); // 존재하는 프로젝트 id인지 확인.
String projectId = metaDataCreateAllRequestDto.getProjectId();
throwGlobalExceptionWhenProjectNotExist(projectId);

if(metaDataCreateAllRequestDto.getBodyList() != null){

List<Document>bodyList = metaDataCreateAllRequestDto.getBodyList();

String projectId = metaDataCreateAllRequestDto.getProjectId();

for(Document body : bodyList){
MetaData metaData = new MetaData().builder()
.projectId(projectId)
.body(body).build();
metaDataList.add(metaData);
patientService.addProjectCount(metaData.getPatientIdFromBody());
}

List<MetaData> metaDataList = getMetaDataFromMetaDataCreateAllRequestDto(metaDataCreateAllRequestDto, projectId);
metaDataList.forEach(metaData -> patientService.addProjectCount(metaData.getPatientIdFromBody()));
int chunk_size = 1000;
for (List<MetaData> batch : Lists.partition(metaDataList,chunk_size)) {
metaDataRepository.saveAll(batch);
}

Lists.partition(metaDataList,chunk_size).forEach(batch -> metaDataRepository.saveAll(batch) );
}
}

@NotNull
private List<MetaData> getMetaDataFromMetaDataCreateAllRequestDto(MetaDataCreateAllRequestDto metaDataCreateAllRequestDto, String projectId) {
return metaDataCreateAllRequestDto.getBodyList().stream()
.map(body -> new MetaData().builder().projectId(projectId).body(body).build())
.collect(Collectors.toList());
}

@Transactional
public void deleteAllByMetaDataIdList(MetaDataDeleteAllRequestDto metaDataDeleteAllRequestDto){

projectService.findById(metaDataDeleteAllRequestDto.getProjectId()); // 존재하는 프로젝트 id인지 확인.
throwGlobalExceptionWhenProjectNotExist((metaDataDeleteAllRequestDto.getProjectId()));

List<MetaData> metaDataList = metaDataDeleteAllRequestDto.getMetadataIdList().stream()
.map( metadataId -> metaDataRepository.findById(metadataId).orElseThrow(()-> new MetaDataNotFoundException(ErrorCode.METADATA_NOT_FOUND)))
Expand Down
28 changes: 23 additions & 5 deletions server/DSMP/src/test/java/com/knuipalab/dsmp/QA/metadata/MetaDataQA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.javafaker.Faker;
import com.google.common.collect.Lists;
import com.knuipalab.dsmp.metadata.CustomizedMetaDataRepository;
import com.knuipalab.dsmp.metadata.MetaData;
import com.knuipalab.dsmp.metadata.MetaDataRepository;
import com.knuipalab.dsmp.project.Project;
Expand All @@ -20,9 +21,11 @@
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.context.annotation.Profile;
import org.springframework.data.domain.Page;
import org.springframework.test.context.junit.jupiter.SpringExtension;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

@SpringBootTest
Expand All @@ -36,6 +39,9 @@ public class MetaDataQA {
@SpyBean
MetaDataRepository metaDataRepository;

@SpyBean
CustomizedMetaDataRepository customizedMetaDataRepository;

Logger log = (Logger) LoggerFactory.getLogger(MetaDataQA.class);

public User createMockUser(){
Expand Down Expand Up @@ -75,7 +81,7 @@ public String createMockStrBodyList(){

long beforeTime = System.currentTimeMillis();

long METADATA_SIZE = 1000;
long METADATA_SIZE = 200;

Faker faker = new Faker();
StringBuilder strBodyList = new StringBuilder();
Expand All @@ -102,9 +108,9 @@ public String createMockStrBodyList(){
strBodyList.append("]");

long afterTime = System.currentTimeMillis(); // 코드 실행 후에 시간 받아오기
long secDiffTime = (afterTime - beforeTime)/1000; //두 시간에 차 계산
long secDiffTime = (afterTime - beforeTime); //두 시간에 차 계산

log.info("createMockStrBodyList 실행 시간(m) : "+secDiffTime);
log.info("createMockStrBodyList 실행 시간(ms) : "+secDiffTime);

return strBodyList.toString();
}
Expand Down Expand Up @@ -175,7 +181,7 @@ public void saveAllQA(){
long afterTime = System.currentTimeMillis(); // 코드 실행 후에 시간 받아오기
long secDiffTime = (afterTime - beforeTime); //두 시간에 차 계산

log.info("saveAll QA 실행 시간(m) : "+secDiffTime);
log.info("saveAll QA 실행 시간(ms) : "+secDiffTime);

}

Expand All @@ -188,8 +194,20 @@ public void findAllQA(){

long afterTime = System.currentTimeMillis(); // 코드 실행 후에 시간 받아오기
long secDiffTime = (afterTime - beforeTime); //두 시간에 차 계산
log.info("findAll 개수 : "+metaDataList.size());
log.info("findAll QA 실행 시간(ms) : "+secDiffTime);
}

log.info("findAll QA 실행 시간(m) : "+secDiffTime);
@Profile("QA")
@Test
public void findByProjectIdWithPaging() {
long beforeTime = System.currentTimeMillis();
HashMap<String,Object> parmMap = new HashMap<>();
Page<MetaData> metaDataPage = customizedMetaDataRepository.findByProjectIdWithPaging("54321",11123,20,parmMap);
long afterTime = System.currentTimeMillis(); // 코드 실행 후에 시간 받아오기
long secDiffTime = (afterTime - beforeTime); //두 시간에 차 계산
log.info("findByProjectIdWithPaging 개수 : "+metaDataPage.getContent().size());
log.info("findByProjectIdWithPaging QA 실행 시간(ms) : "+secDiffTime);
}


Expand Down
7 changes: 3 additions & 4 deletions .../java/com/knuipalab/dsmp/controller/machineLearning/MachineLearningApiControllerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;

import static org.mockito.BDDMockito.given;

@ExtendWith(SpringExtension.class)
@WebMvcTest(controllers = MachineLearningApiController.class)
Expand Down Expand Up @@ -51,7 +51,7 @@ public void setup() {
}

@WithMockUser
@DisplayName("Type Sampling Test by ProjectId - Success")
@DisplayName("Type Sampling Test by ProjectId")
@Test
void typeSamplingTest() throws Exception {

Expand All @@ -63,10 +63,9 @@ void typeSamplingTest() throws Exception {
}

@WithMockUser
@DisplayName("Set Malignancy Classification Test by ProjectId - Success")
@DisplayName("Set Malignancy Classification Test by ProjectId and Model Name")
@Test
void setMalignancyClassificationTest() throws Exception {

mvc.perform(put("/api/MetaData/MalignancyClassification/54321/124214"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.status",is(200)))
Expand Down
Loading