From 30e30bfc69461e59f9c9909324a334a736cbd8fa Mon Sep 17 00:00:00 2001 From: haribonyam Date: Mon, 12 Jan 2026 01:53:41 +0900 Subject: [PATCH 01/10] =?UTF-8?q?[#9]=20feat=20:=20=EA=B4=80=EA=B4=91?= =?UTF-8?q?=EC=A7=80=20=EC=A0=95=EB=B3=B4=20=EC=88=98=EC=A7=91=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD=20-=20=EA=B8=B0=EC=A1=B4=20=EC=9B=90=EB=B3=B8=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=20json=20=ED=8C=8C=EC=8B=B1=20?= =?UTF-8?q?=EB=B0=8F=20=EC=99=B8=EB=B6=80=20api=20=ED=98=B8=EC=B6=9C?= =?UTF-8?q?=EC=97=90=EC=84=9C=20=EC=99=B8=EB=B6=80=20api=EB=A1=9C=EB=A7=8C?= =?UTF-8?q?=20=EC=88=98=EC=A7=91=20-=20=EB=8B=A4=EA=B5=AD=EC=96=B4=20?= =?UTF-8?q?=EC=A7=80=EC=9B=90(=EC=98=81=EC=96=B4)=20-=20=EB=B3=80=EA=B2=BD?= =?UTF-8?q?=EC=82=AC=ED=95=AD=20=EB=8B=A4=EC=88=98=20->=20=EB=8F=84?= =?UTF-8?q?=EC=8A=A8=ED=8A=B8=EC=AA=BD=EB=8F=84=20=EA=BC=AD=20=EB=B3=B4?= =?UTF-8?q?=EA=B3=A0=20=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81=20=EC=A4=80?= =?UTF-8?q?=EB=B9=84=ED=95=B4=EC=95=BC=ED=95=A0=EB=93=AF=ED=95=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/config/ObjectMapperConfig.java | 30 +- .../core/common/config/RestClientConfig.java | 8 +- .../core/controller/MasterDataController.java | 23 +- .../com/earseo/core/dto/etl/AreaItemDto.java | 16 + .../earseo/core/dto/etl/CommonItemDto.java | 7 - .../earseo/core/dto/etl/FilteredDataDto.java | 5 - .../com/earseo/core/dto/etl/ImageItemDto.java | 4 - .../com/earseo/core/dto/etl/JoinItemDto.java | 10 - .../earseo/core/dto/etl/MasterItemDto.java | 62 +- .../earseo/core/dto/etl/MiddleDataDto.java | 8 - .../earseo/core/dto/etl/SpotCategoryItem.java | 4 + .../earseo/core/dto/tourApi/AreaResponse.java | 45 + .../core/dto/tourApi/CommonResponse.java | 27 + .../dto/tourApi/CultureDetailResponse.java | 51 ++ .../core/dto/tourApi/DetailResponse.java | 7 + .../dto/tourApi/FestivalDetailResponse.java | 48 + .../dto/tourApi/LeportsDetailResponse.java | 51 ++ .../dto/tourApi/ShoppingDetailResponse.java | 49 + .../core/dto/tourApi/SightDetailResponse.java | 44 + .../java/com/earseo/core/entity/EnMaster.java | 92 ++ .../entity/{Master.java => KoMaster.java} | 34 +- .../com/earseo/core/entity/MiddleData.java | 104 --- .../com/earseo/core/entity/SpotCategory.java | 36 + ...epository.java => EnMasterRepository.java} | 4 +- .../core/repository/KoMasterRepository.java | 9 + .../core/repository/MiddleRepository.java | 18 - .../repository/SpotCategoryRepository.java | 9 + .../core/service/MasterDataService.java | 864 +++++++++++------- .../core/service/master/CategoryGroup.java | 20 + .../core/service/master/DetailType.java | 28 + .../core/service/master/ExcludeRule.java | 31 + .../core/service/master/SubCategoryGroup.java | 62 ++ .../core/service/master/TourApiPath.java | 19 + 33 files changed, 1318 insertions(+), 511 deletions(-) create mode 100644 src/main/java/com/earseo/core/dto/etl/AreaItemDto.java delete mode 100644 src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java delete mode 100644 src/main/java/com/earseo/core/dto/etl/ImageItemDto.java delete mode 100644 src/main/java/com/earseo/core/dto/etl/JoinItemDto.java delete mode 100644 src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java create mode 100644 src/main/java/com/earseo/core/dto/etl/SpotCategoryItem.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/AreaResponse.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/CommonResponse.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/CultureDetailResponse.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/DetailResponse.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/FestivalDetailResponse.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/LeportsDetailResponse.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/ShoppingDetailResponse.java create mode 100644 src/main/java/com/earseo/core/dto/tourApi/SightDetailResponse.java create mode 100644 src/main/java/com/earseo/core/entity/EnMaster.java rename src/main/java/com/earseo/core/entity/{Master.java => KoMaster.java} (75%) delete mode 100644 src/main/java/com/earseo/core/entity/MiddleData.java create mode 100644 src/main/java/com/earseo/core/entity/SpotCategory.java rename src/main/java/com/earseo/core/repository/{MasterRepository.java => EnMasterRepository.java} (58%) create mode 100644 src/main/java/com/earseo/core/repository/KoMasterRepository.java delete mode 100644 src/main/java/com/earseo/core/repository/MiddleRepository.java create mode 100644 src/main/java/com/earseo/core/repository/SpotCategoryRepository.java create mode 100644 src/main/java/com/earseo/core/service/master/CategoryGroup.java create mode 100644 src/main/java/com/earseo/core/service/master/DetailType.java create mode 100644 src/main/java/com/earseo/core/service/master/ExcludeRule.java create mode 100644 src/main/java/com/earseo/core/service/master/SubCategoryGroup.java create mode 100644 src/main/java/com/earseo/core/service/master/TourApiPath.java diff --git a/src/main/java/com/earseo/core/common/config/ObjectMapperConfig.java b/src/main/java/com/earseo/core/common/config/ObjectMapperConfig.java index c972d9e..d5a06cd 100644 --- a/src/main/java/com/earseo/core/common/config/ObjectMapperConfig.java +++ b/src/main/java/com/earseo/core/common/config/ObjectMapperConfig.java @@ -1,18 +1,42 @@ package com.earseo.core.common.config; +import com.fasterxml.jackson.core.StreamWriteConstraints; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.cfg.CoercionAction; +import com.fasterxml.jackson.databind.cfg.CoercionInputShape; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.type.LogicalType; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; @Configuration public class ObjectMapperConfig { @Bean + @Primary public ObjectMapper objectMapper() { - return new ObjectMapper() - .registerModule(new JavaTimeModule()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ObjectMapper mapper = JsonMapper.builder() + .addModule(new JavaTimeModule()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true) + .build(); + + mapper.coercionConfigDefaults() + .setCoercion(CoercionInputShape.EmptyString, CoercionAction.AsNull); + + mapper.getFactory() + .setStreamWriteConstraints( + StreamWriteConstraints.builder() + .maxNestingDepth(3000) + .build() + ); + + mapper.enable(SerializationFeature.INDENT_OUTPUT); + + return mapper; } } diff --git a/src/main/java/com/earseo/core/common/config/RestClientConfig.java b/src/main/java/com/earseo/core/common/config/RestClientConfig.java index fb0f6d4..29b3388 100644 --- a/src/main/java/com/earseo/core/common/config/RestClientConfig.java +++ b/src/main/java/com/earseo/core/common/config/RestClientConfig.java @@ -2,13 +2,17 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; import org.springframework.web.client.RestClient; @Configuration public class RestClientConfig { @Bean - public RestClient.Builder restClientBuilder() { - return RestClient.builder(); + @Primary + public RestClient restClient(RestClient.Builder builder) { + return builder + .baseUrl("https://apis.data.go.kr") + .build(); } } \ No newline at end of file diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java index 2e599fe..995f138 100644 --- a/src/main/java/com/earseo/core/controller/MasterDataController.java +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -1,9 +1,9 @@ package com.earseo.core.controller; import com.earseo.core.common.BaseResponse; -import com.earseo.core.dto.etl.FilteredDataDto; -import com.earseo.core.dto.etl.MiddleDataDto; +import com.earseo.core.dto.etl.AreaItemDto; import com.earseo.core.service.MasterDataService; +import com.earseo.core.service.master.TourApiPath; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -18,18 +18,23 @@ public class MasterDataController { private final MasterDataService masterDataService; - @GetMapping("/admin/core/master") - public ResponseEntity> rawDataProcess() throws IOException { - List filteredData = masterDataService.getRawInfo(); - List middleData = masterDataService.getMiddleData(filteredData); - masterDataService.createMasterTable(); + @GetMapping("/admin/core/master/ko") + public ResponseEntity> createTableKo() throws IOException { + List list = masterDataService.getTourApiArea(TourApiPath.KoArea.getPath()); + masterDataService.createMasterTable(list, TourApiPath.KoCommon.getPath(), TourApiPath.KoDetail.getPath(), "ko"); + return ResponseEntity.ok(BaseResponse.ok(null)); + } + + @GetMapping("/admin/core/master/en") + public ResponseEntity> createTableEn() throws IOException { + List list = masterDataService.getTourApiArea(TourApiPath.EnArea.getPath()); + masterDataService.createMasterTable(list, TourApiPath.EnCommon.getPath(), TourApiPath.EnDetail.getPath(), "en"); return ResponseEntity.ok(BaseResponse.ok(null)); } @GetMapping("/admin/core/init") public ResponseEntity> init(){ - masterDataService.initData(); + masterDataService.createCategory(); return ResponseEntity.ok(BaseResponse.ok(null)); } - } diff --git a/src/main/java/com/earseo/core/dto/etl/AreaItemDto.java b/src/main/java/com/earseo/core/dto/etl/AreaItemDto.java new file mode 100644 index 0000000..4952be7 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/AreaItemDto.java @@ -0,0 +1,16 @@ +package com.earseo.core.dto.etl; + +import com.earseo.core.dto.tourApi.AreaResponse; + +public record AreaItemDto(String contentId, String contentTypeId, String title, String address, + String detailAddress, String catCode, Double mapX, Double mapY, + String imageUrl, String thumbnailUrl, String tel, Integer mlevel, String modifiedtime) { + + public static AreaItemDto from(AreaResponse.Item item){ + return new AreaItemDto( + item.contentid(), item.contenttypeid(), item.title(), item.addr1(), item.addr2(), item.cat2(), + Double.parseDouble(item.mapx()), Double.parseDouble(item.mapy()), item.firstimage(), item.firstimage2(), + item.tel(),item.mlevel().equals("")?null:Integer.parseInt(item.mlevel()),item.modifiedtime() + ); + } +} diff --git a/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java b/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java index f95c4a8..cb16a68 100644 --- a/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java +++ b/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java @@ -1,13 +1,6 @@ package com.earseo.core.dto.etl; public record CommonItemDto( - String title, - String addr1, - String addr2, - String mapX, - String mapY, String modifiedTime, - String tel, - String mLevel, String overview ) {} diff --git a/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java b/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java deleted file mode 100644 index 613aee5..0000000 --- a/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.earseo.core.dto.etl; - -public record FilteredDataDto(String contentId, String contentTypeId, String cat1, - String cat2, String cat3, String outl) { -} diff --git a/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java b/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java deleted file mode 100644 index 01a3385..0000000 --- a/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.earseo.core.dto.etl; - -public record ImageItemDto(String imgrul, String smallimgurl) { -} diff --git a/src/main/java/com/earseo/core/dto/etl/JoinItemDto.java b/src/main/java/com/earseo/core/dto/etl/JoinItemDto.java deleted file mode 100644 index e253135..0000000 --- a/src/main/java/com/earseo/core/dto/etl/JoinItemDto.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.earseo.core.dto.etl; - -public record JoinItemDto( - Long id, - String contentId, - String title, - String outl, - String script -) { -} diff --git a/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java b/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java index 50a7369..3398aea 100644 --- a/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java +++ b/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java @@ -1,10 +1,68 @@ package com.earseo.core.dto.etl; +import com.earseo.core.entity.EnMaster; +import com.earseo.core.entity.KoMaster; +import lombok.Builder; import org.locationtech.jts.geom.Point; +@Builder public record MasterItemDto(String contentId, String contentTypeId, String cat1, - String cat2, String cat3, String ocat1, String ocat2, String ocat3, - String outl, String title, String addr1, String addr2, String addr3, + String cat2, String cat1Code, String cat2Code, String title, + String addr1, String addr2, String addr3, Double mapX, Double mapY, String modifiedtime, String tel, Integer mLevel, String overview, String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee) { + + public KoMaster toKo() { + return KoMaster.builder() + .contentId(contentId) + .contentTypeId(contentTypeId) + .cat1(cat1) + .cat2(cat2) + .cat1Code(cat1Code) + .cat2Code(cat2Code) + .title(title) + .addr1(addr1) + .addr2(addr2) + .addr3(addr3) + .mapX(mapX) + .mapY(mapY) + .modifiedtime(modifiedtime) + .tel(tel) + .mLevel(mLevel) + .overview(overview) + .originImgUrl(originImgUrl) + .smallImgUrl(smallImgUrl) + .usetime(usetime) + .restdate(restdate) + .parking(parking) + .usefee(usefee) + .build(); + } + + public EnMaster toEn() { + return EnMaster.builder() + .contentId(contentId) + .contentTypeId(contentTypeId) + .cat1(cat1) + .cat2(cat2) + .cat1Code(cat1Code) + .cat2Code(cat2Code) + .title(title) + .addr1(addr1) + .addr2(addr2) + .addr3(addr3) + .mapX(mapX) + .mapY(mapY) + .modifiedtime(modifiedtime) + .tel(tel) + .mLevel(mLevel) + .overview(overview) + .originImgUrl(originImgUrl) + .smallImgUrl(smallImgUrl) + .usetime(usetime) + .restdate(restdate) + .parking(parking) + .usefee(usefee) + .build(); + } } diff --git a/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java b/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java deleted file mode 100644 index 968d661..0000000 --- a/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.earseo.core.dto.etl; - -public record MiddleDataDto(String contentId, String contentTypeId, String cat1, - String cat2, String cat3, String outl, String title, String addr1, String addr2, - String mapX, String mapY, String modifiedtime,String tel, String mLevel,String overview, - String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee - ) { -} diff --git a/src/main/java/com/earseo/core/dto/etl/SpotCategoryItem.java b/src/main/java/com/earseo/core/dto/etl/SpotCategoryItem.java new file mode 100644 index 0000000..1e6bfb3 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/SpotCategoryItem.java @@ -0,0 +1,4 @@ +package com.earseo.core.dto.etl; + +public record SpotCategoryItem(String code, String name, String originName, String originCode) { +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/AreaResponse.java b/src/main/java/com/earseo/core/dto/tourApi/AreaResponse.java new file mode 100644 index 0000000..f3154ce --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/AreaResponse.java @@ -0,0 +1,45 @@ +package com.earseo.core.dto.tourApi; + +import java.util.List; + +public record AreaResponse(Response response) { + + public record Response( + Body body, + Header header + ){} + + public record Header( + String resultCode, + String resultMsg + ) {} + + public record Body(Items items, int numOfRows){} + + public record Items(List item){} + + public record Item( + String contentid, + String contenttypeid, + String title, + + String addr1, + String addr2, + + String cat1, + String cat2, + String cat3, + + String mapx, + String mapy, + String mlevel, + + String firstimage, + String firstimage2, + + String tel, + + String modifiedtime + + ) {} +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/CommonResponse.java b/src/main/java/com/earseo/core/dto/tourApi/CommonResponse.java new file mode 100644 index 0000000..f269dc7 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/CommonResponse.java @@ -0,0 +1,27 @@ +package com.earseo.core.dto.tourApi; + +import java.util.List; + +public record CommonResponse(Response response){ + + public record Response( + Body body, + Header header + ){} + + public record Header( + String resultCode, + String resultMsg + ) {} + + public record Body(Items items, int numOfRows){} + + public record Items(List item){} + + public record Item( + String contentid, + String contenttypeid, + String overview + + ) {} +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/CultureDetailResponse.java b/src/main/java/com/earseo/core/dto/tourApi/CultureDetailResponse.java new file mode 100644 index 0000000..f96dc51 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/CultureDetailResponse.java @@ -0,0 +1,51 @@ +package com.earseo.core.dto.tourApi; + +import com.earseo.core.dto.etl.DetailItemDto; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +public record CultureDetailResponse(Response response) implements DetailResponse { + + public record Response( + Body body, + Header header + ){} + + public record Header( + String resultCode, + String resultMsg + ) {} + + public record Body(Items items){} + + public record Items(List item){} + + public record Item( + String contentid, + String contenttypeid, + String usefee, + @JsonProperty("restdateculture") + String restdate, + @JsonProperty("usetimeculture") + String usetime, + @JsonProperty("parkingculture") + String parking + ) {} + + @Override + public DetailItemDto getDetailItemDto() { + if (response == null || + response.body() == null || + response.body().items() == null || + response.body().items().item() == null || + response.body().items().item().isEmpty()) { + + return null; // 혹은 null 반환 + } + Item item = response.body.items.item.get(0); + return new DetailItemDto(item.usefee, item.parking, item.restdate, item.usetime); + } +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/DetailResponse.java b/src/main/java/com/earseo/core/dto/tourApi/DetailResponse.java new file mode 100644 index 0000000..3ffec8d --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/DetailResponse.java @@ -0,0 +1,7 @@ +package com.earseo.core.dto.tourApi; + +import com.earseo.core.dto.etl.DetailItemDto; + +public interface DetailResponse { + DetailItemDto getDetailItemDto(); +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/FestivalDetailResponse.java b/src/main/java/com/earseo/core/dto/tourApi/FestivalDetailResponse.java new file mode 100644 index 0000000..7425abe --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/FestivalDetailResponse.java @@ -0,0 +1,48 @@ +package com.earseo.core.dto.tourApi; + +import com.earseo.core.dto.etl.DetailItemDto; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record FestivalDetailResponse(Response response) implements DetailResponse{ + public record Response( + Body body, + Header header + ){} + + public record Header( + String resultCode, + String resultMsg + ) {} + + public record Body(Items items){} + + public record Items(List item){} + + public record Item( + String contentid, + String contenttypeid, + String usefee, + @JsonProperty("restdatefestival") + String restdate, + @JsonProperty("usetimefestival") + String usetime, + @JsonProperty("parkingfestival") + String parking + ) {} + + @Override + public DetailItemDto getDetailItemDto() { + if (response == null || + response.body() == null || + response.body().items() == null || + response.body().items().item() == null || + response.body().items().item().isEmpty()) { + + return null; + } + Item item = response.body.items.item.get(0); + return new DetailItemDto(item.usefee, item.parking, item.restdate, item.usetime); + } +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/LeportsDetailResponse.java b/src/main/java/com/earseo/core/dto/tourApi/LeportsDetailResponse.java new file mode 100644 index 0000000..60fc39e --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/LeportsDetailResponse.java @@ -0,0 +1,51 @@ +package com.earseo.core.dto.tourApi; + +import com.earseo.core.dto.etl.DetailItemDto; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; + +import java.util.List; + +public record LeportsDetailResponse(Response response) implements DetailResponse { + public record Response( + Body body, + Header header + ){} + + public record Header( + String resultCode, + String resultMsg + ) {} + + public record Body(Items items){} + + public record Items(List item){} + + public record Item( + String contentid, + String contenttypeid, + @JsonProperty("usefeeleports") + String usefee, + @JsonProperty("restdateleports") + String restdate, + @JsonProperty("usetimeleports") + String usetime, + @JsonProperty("parkingleports") + String parking + ) {} + + @Override + public DetailItemDto getDetailItemDto() { + if (response == null || + response.body() == null || + response.body().items() == null || + response.body().items().item() == null || + response.body().items().item().isEmpty()) { + + return null; + } + Item item = response.body.items.item.get(0); + return new DetailItemDto(item.usefee, item.parking, item.restdate, item.usetime); + } +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/ShoppingDetailResponse.java b/src/main/java/com/earseo/core/dto/tourApi/ShoppingDetailResponse.java new file mode 100644 index 0000000..8678d70 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/ShoppingDetailResponse.java @@ -0,0 +1,49 @@ +package com.earseo.core.dto.tourApi; + +import com.earseo.core.dto.etl.DetailItemDto; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record ShoppingDetailResponse(Response response) implements DetailResponse{ + public record Response( + Body body, + Header header + ){} + + public record Header( + String resultCode, + String resultMsg + ) {} + + public record Body(Items items){} + + public record Items(List item){} + + public record Item( + String contentid, + String contenttypeid, + @JsonProperty("usefeeshopping") + String usefee, + @JsonProperty("restdateshopping") + String restdate, + @JsonProperty("usetimeshopping") + String usetime, + @JsonProperty("parkingshopping") + String parking + ) {} + + @Override + public DetailItemDto getDetailItemDto() { + if (response == null || + response.body() == null || + response.body().items() == null || + response.body().items().item() == null || + response.body().items().item().isEmpty()) { + + return null; + } + Item item = response.body.items.item.get(0); + return new DetailItemDto(item.usefee, item.parking, item.restdate, item.usetime); + } +} diff --git a/src/main/java/com/earseo/core/dto/tourApi/SightDetailResponse.java b/src/main/java/com/earseo/core/dto/tourApi/SightDetailResponse.java new file mode 100644 index 0000000..1c59fbc --- /dev/null +++ b/src/main/java/com/earseo/core/dto/tourApi/SightDetailResponse.java @@ -0,0 +1,44 @@ +package com.earseo.core.dto.tourApi; + +import com.earseo.core.dto.etl.DetailItemDto; + +import java.util.List; + +public record SightDetailResponse(Response response) implements DetailResponse{ + public record Response( + Body body, + Header header + ){} + + public record Header( + String resultCode, + String resultMsg + ) {} + + public record Body(Items items){} + + public record Items(List item){} + + public record Item( + String contentid, + String contenttypeid, + String usefee, + String restdate, + String usetime, + String parking + ) {} + + @Override + public DetailItemDto getDetailItemDto() { + if (response == null || + response.body() == null || + response.body().items() == null || + response.body().items().item() == null || + response.body().items().item().isEmpty()) { + + return null; + } + Item item = response.body.items.item.get(0); + return new DetailItemDto(item.usefee, item.parking, item.restdate, item.usetime); + } +} diff --git a/src/main/java/com/earseo/core/entity/EnMaster.java b/src/main/java/com/earseo/core/entity/EnMaster.java new file mode 100644 index 0000000..61784eb --- /dev/null +++ b/src/main/java/com/earseo/core/entity/EnMaster.java @@ -0,0 +1,92 @@ +package com.earseo.core.entity; + +import com.earseo.core.dto.etl.MasterItemDto; +import com.earseo.core.service.master.CategoryGroup; +import com.earseo.core.service.master.SubCategoryGroup; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@NoArgsConstructor +@AllArgsConstructor +@Getter @Builder +public class EnMaster { @Id + +@GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "content_id") + private String contentId; + + @Column(name = "content_type_id") + private String contentTypeId; + + @Column(name = "cat1") + private String cat1; + + @Column(name = "cat2") + private String cat2; + + @Column(name = "cat1_code") + private String cat1Code; + + @Column(name = "cat2_code") + private String cat2Code; + + @Column(name = "title") + private String title; + + @Column(name = "addr1") + private String addr1; + + @Column(name = "addr2") + private String addr2; + + @Column(name = "addr3") + private String addr3; + + @Column(name = "map_x") + private Double mapX; + + @Column(name = "map_y") + private Double mapY; + + @Column(name = "modifiedtime") + private String modifiedtime; + + @Column(name = "tel", columnDefinition = "TEXT") + private String tel; + + @Column(name = "m_level") + private Integer mLevel; + + @Column(name = "overview", columnDefinition = "TEXT") + private String overview; + + @Column(name = "origin_img_url", columnDefinition = "TEXT") + private String originImgUrl; + + @Column(name = "small_img_url", columnDefinition = "TEXT") + private String smallImgUrl; + + @Column(name = "use_time", columnDefinition = "TEXT") + private String usetime; + + @Column(name = "rest_date", columnDefinition = "TEXT") + private String restdate; + + @Column(name = "parking", columnDefinition = "TEXT") + private String parking; + + @Column(name = "use_fee", columnDefinition = "TEXT") + private String usefee; + + public EnMaster from(MasterItemDto masterItemDto){ + return EnMaster.builder() + .build(); + } + +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/entity/Master.java b/src/main/java/com/earseo/core/entity/KoMaster.java similarity index 75% rename from src/main/java/com/earseo/core/entity/Master.java rename to src/main/java/com/earseo/core/entity/KoMaster.java index 8602683..6f2506f 100644 --- a/src/main/java/com/earseo/core/entity/Master.java +++ b/src/main/java/com/earseo/core/entity/KoMaster.java @@ -1,25 +1,24 @@ package com.earseo.core.entity; +import com.earseo.core.dto.etl.MasterItemDto; +import com.earseo.core.service.master.CategoryGroup; +import com.earseo.core.service.master.SubCategoryGroup; import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; -import org.locationtech.jts.geom.Point; - -import java.awt.*; @Entity -@AllArgsConstructor +@Getter @Builder @NoArgsConstructor -@Builder @Getter -public class Master { - +@AllArgsConstructor +public class KoMaster { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(name = "content_id", nullable = false, unique = true) + @Column(name = "content_id") private String contentId; @Column(name = "content_type_id") @@ -31,20 +30,11 @@ public class Master { @Column(name = "cat2") private String cat2; - @Column(name = "cat3") - private String cat3; - - @Column(name = "ocat1") - private String ocat1; - - @Column(name = "ocat2") - private String ocat2; - - @Column(name = "ocat3") - private String ocat3; + @Column(name = "cat1_code") + private String cat1Code; - @Column(name = "outl", columnDefinition = "TEXT") - private String outl; + @Column(name = "cat2_code") + private String cat2Code; @Column(name = "title") private String title; @@ -94,6 +84,4 @@ public class Master { @Column(name = "use_fee", columnDefinition = "TEXT") private String usefee; - @Column(name = "geom", columnDefinition = "geometry(Point,4326)") - private Point geom; } diff --git a/src/main/java/com/earseo/core/entity/MiddleData.java b/src/main/java/com/earseo/core/entity/MiddleData.java deleted file mode 100644 index 880ec7f..0000000 --- a/src/main/java/com/earseo/core/entity/MiddleData.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.earseo.core.entity; - -import com.earseo.core.dto.etl.MiddleDataDto; -import jakarta.persistence.*; -import lombok.*; - -@Entity -@Table(name = "middle_data") -@Getter -@Setter -@NoArgsConstructor -@AllArgsConstructor -@Builder -public class MiddleData { - - @Id - @Column(name = "content_id", nullable = false, unique = true) - private String contentId; - - @Column(name = "content_type_id") - private String contentTypeId; - - @Column(name = "cat1") - private String cat1; - - @Column(name = "cat2") - private String cat2; - - @Column(name = "cat3") - private String cat3; - - @Column(name = "outl", columnDefinition = "TEXT") - private String outl; - - @Column(name = "title", columnDefinition = "TEXT") - private String title; - - @Column(name = "addr1", columnDefinition = "TEXT") - private String addr1; - - @Column(name = "addr2", columnDefinition = "TEXT") - private String addr2; - - @Column(name = "map_x", columnDefinition = "TEXT") - private String mapX; - - @Column(name = "map_y", columnDefinition = "TEXT") - private String mapY; - - @Column(name = "modified_time", columnDefinition = "TEXT") - private String modifiedtime; - - @Column(name = "tel", columnDefinition = "TEXT") - private String tel; - - @Column(name = "m_level", columnDefinition = "TEXT") - private String mLevel; - - @Column(name = "overview", columnDefinition = "TEXT") - private String overview; - - @Column(name = "origin_img_url", columnDefinition = "TEXT") - private String originImgUrl; - - @Column(name = "small_img_url", columnDefinition = "TEXT") - private String smallImgUrl; - - @Column(name = "use_time", columnDefinition = "TEXT") - private String usetime; - - @Column(name = "rest_date", columnDefinition = "TEXT") - private String restdate; - - @Column(name = "parking", columnDefinition = "TEXT") - private String parking; - - @Column(name = "use_fee", columnDefinition = "TEXT") - private String usefee; - - - public MiddleData(MiddleDataDto dto) { - this.contentId = dto.contentId(); - this.contentTypeId = dto.contentTypeId(); - this.cat1 = dto.cat1(); - this.cat2 = dto.cat2(); - this.cat3 = dto.cat3(); - this.outl = dto.outl(); - this.title = dto.title(); - this.addr1 = dto.addr1(); - this.addr2 = dto.addr2(); - this.mapX = dto.mapX(); - this.mapY = dto.mapY(); - this.modifiedtime = dto.modifiedtime(); - this.tel = dto.tel(); - this.mLevel = dto.mLevel(); - this.overview = dto.overview(); - this.originImgUrl = dto.originImgUrl(); - this.smallImgUrl = dto.smallImgUrl(); - this.usetime = dto.usetime(); - this.restdate = dto.restdate(); - this.parking = dto.parking(); - this.usefee = dto.usefee(); - } -} diff --git a/src/main/java/com/earseo/core/entity/SpotCategory.java b/src/main/java/com/earseo/core/entity/SpotCategory.java new file mode 100644 index 0000000..587fc8c --- /dev/null +++ b/src/main/java/com/earseo/core/entity/SpotCategory.java @@ -0,0 +1,36 @@ +package com.earseo.core.entity; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Builder +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class SpotCategory { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer id; + + private String parentCode; + + private String parentKoName; + + private String parentEnName; + + private String KoName; + + private String EngName; + + private String code; + + private String originCode; +} diff --git a/src/main/java/com/earseo/core/repository/MasterRepository.java b/src/main/java/com/earseo/core/repository/EnMasterRepository.java similarity index 58% rename from src/main/java/com/earseo/core/repository/MasterRepository.java rename to src/main/java/com/earseo/core/repository/EnMasterRepository.java index 2b36ec1..2bf7ea8 100644 --- a/src/main/java/com/earseo/core/repository/MasterRepository.java +++ b/src/main/java/com/earseo/core/repository/EnMasterRepository.java @@ -1,9 +1,9 @@ package com.earseo.core.repository; -import com.earseo.core.entity.Master; +import com.earseo.core.entity.EnMaster; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @Repository -public interface MasterRepository extends JpaRepository { +public interface EnMasterRepository extends JpaRepository { } diff --git a/src/main/java/com/earseo/core/repository/KoMasterRepository.java b/src/main/java/com/earseo/core/repository/KoMasterRepository.java new file mode 100644 index 0000000..0233593 --- /dev/null +++ b/src/main/java/com/earseo/core/repository/KoMasterRepository.java @@ -0,0 +1,9 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.KoMaster; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface KoMasterRepository extends JpaRepository { +} diff --git a/src/main/java/com/earseo/core/repository/MiddleRepository.java b/src/main/java/com/earseo/core/repository/MiddleRepository.java deleted file mode 100644 index 19cf5af..0000000 --- a/src/main/java/com/earseo/core/repository/MiddleRepository.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.earseo.core.repository; - -import com.earseo.core.entity.MiddleData; -import jakarta.transaction.Transactional; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; -import org.springframework.stereotype.Repository; - -@Repository -public interface MiddleRepository extends JpaRepository { - - @Modifying - @Transactional - @Query("DELETE FROM MiddleData m WHERE m.contentTypeId = :contentTypeId") - int deleteByContentType(@Param("contentTypeId") String contentTypeId); -} diff --git a/src/main/java/com/earseo/core/repository/SpotCategoryRepository.java b/src/main/java/com/earseo/core/repository/SpotCategoryRepository.java new file mode 100644 index 0000000..caae06c --- /dev/null +++ b/src/main/java/com/earseo/core/repository/SpotCategoryRepository.java @@ -0,0 +1,9 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.SpotCategory; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface SpotCategoryRepository extends CrudRepository { +} diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index adb1bc6..01647ce 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -2,233 +2,166 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.PutObjectRequest; -import com.earseo.core.common.BaseResponse; import com.earseo.core.dto.etl.*; -import com.earseo.core.entity.Category; -import com.earseo.core.entity.Master; -import com.earseo.core.entity.MiddleData; -import com.earseo.core.repository.CategoryRepository; -import com.earseo.core.repository.MasterRepository; -import com.earseo.core.repository.MiddleRepository; +import com.earseo.core.dto.tourApi.AreaResponse; +import com.earseo.core.dto.tourApi.CommonResponse; +import com.earseo.core.dto.tourApi.DetailResponse; +import com.earseo.core.entity.*; +import com.earseo.core.repository.*; +import com.earseo.core.service.master.*; import com.fasterxml.jackson.core.StreamWriteConstraints; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; -import io.swagger.v3.core.util.Json; +import jakarta.annotation.PostConstruct; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.Point; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ClassPathResource; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.client.RestClient; -import org.springframework.web.util.UriComponentsBuilder; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.net.URI; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @Service @RequiredArgsConstructor +@Slf4j public class MasterDataService { private final ObjectMapper objectMapper; + private final RestClient restClient; private final CategoryRepository categoryRepository; - private final MiddleRepository middleRepository; - private final MasterRepository masterRepository; - private final GeometryFactory geometryFactory = new GeometryFactory(); + private final GeometryFactory geometryFactory = new GeometryFactory(); private final AmazonS3 amazonS3; + private final SpotCategoryRepository spotCategoryRepository; + private final KoMasterRepository koMasterRepository; + private final EnMasterRepository enMasterRepository; @Value("${api.key}") private String ApiKeys; @Value(("${cloud.aws.s3.bucket}")) private String bucketName; - private String key; - private int index; + private final Integer NUM_OF_ROWS = 100; + private final Integer BATCH_SIZE = 15; - public List getRawInfo() { - try { - InputStream rawJson = new ClassPathResource("TourAPI_seoul.json").getInputStream(); - List rawJsonDtos = objectMapper.readValue( - rawJson, - new TypeReference>() { - } - ); + private List keyList; + private int keyIndex = 0; - List filtered = rawJsonDtos.stream() - .filter(dto -> !dto.contentTypeId().equals("25")) - .filter(dto -> !dto.contentTypeId().equals("32")) - .filter(dto -> !dto.contentTypeId().equals("39")) - .filter(dto -> !dto.cat3().equals("A04011000")) - .toList(); + @PostConstruct + public void init() { + this.keyList = Arrays.asList(ApiKeys.split(",")); + } - return filtered; + private String getValidKey() { + return keyList.get(keyIndex); + } - } catch (Exception e) { - return List.of(); - } + private void rotateKey() { + keyIndex = (keyIndex + 1) % keyList.size(); } - @Transactional - public void initData() { - List cat1s = fetchCategoryApi(null, null); - List allCategories = new ArrayList<>(); - allCategories.addAll(cat1s); - - for (CategoryItemDto cat1 : cat1s) { - List cat2s = fetchCategoryApi(cat1.code(), null); - allCategories.addAll(cat2s); - for (CategoryItemDto cat2 : cat2s) { - List cat3s = fetchCategoryApi(cat1.code(), cat2.code()); - allCategories.addAll(cat3s); + public List getTourApiArea(String tourApiPrefix) { + return fetchTourApiArea(tourApiPrefix); + } + + private T executeWithRetry(Supplier apiCall) { + int retryCount = 0; + while (retryCount < keyList.size() * 2) { + try { + return apiCall.get(); + } catch (Exception e) { + if (e instanceof org.springframework.web.client.HttpClientErrorException.TooManyRequests || + e instanceof org.springframework.web.client.UnknownContentTypeException || + e instanceof org.springframework.web.client.HttpServerErrorException || + e.getMessage().contains("429") || + e.getMessage().contains("502") || + e.getMessage().contains("HttpMessageNotReadableException")) { + + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + rotateKey(); + retryCount++; + } else { + throw e; + } } } + throw new RuntimeException("TOUR API 호출 에러"); + } - List categories = allCategories.stream() - .map(c -> Category.builder() - .code(c.code()) - .name(c.name()) - .build()) - .toList(); - categoryRepository.saveAll(categories); - } + public void createMasterTable(List areaItemList, String commonPath, String detailPath, String lang) throws IOException { + int batchSize = BATCH_SIZE; + List masterItemList = new ArrayList<>(); + for (int i = 0; i < areaItemList.size(); i += batchSize) { + List batch = + areaItemList.subList(i, Math.min(i + batchSize, areaItemList.size())); - public List fetchCategoryApi(String cat1, String cat2) { - String key = ApiKeys.split(",")[0]; - - RestClient client = RestClient.create(); - - URI uri = UriComponentsBuilder - .fromHttpUrl("https://apis.data.go.kr/B551011/KorService2/categoryCode2") - .queryParam("serviceKey", key) - .queryParam("MobileApp", "AppTest") - .queryParam("MobileOS", "ETC") - .queryParam("pageNo", 1) - .queryParam("numOfRows", 10) - .queryParam("_type", "json") - .queryParamIfPresent("cat1", Optional.ofNullable(cat1)) - .queryParamIfPresent("cat2", Optional.ofNullable(cat2)) - .build(true) - .toUri(); - try { - JsonNode jsonNode = client.get() - .uri(uri) - .retrieve() - .body(JsonNode.class); - JsonNode items = jsonNode - .path("response") - .path("body") - .path("items") - .path("item"); - - List list = new ArrayList<>(); - - for (JsonNode item : items) { - list.add(new CategoryItemDto(item.get("code").asText(), item.get("name").asText())); - } - return list; - } catch (Exception e) { - e.printStackTrace(); + masterItemList.addAll(processBatch(batch, commonPath, detailPath, lang)); } - return null; + + loadData(masterItemList, lang); + } @Transactional - public void createMasterTable() throws IOException { - List middleDataList = middleRepository.findAll(); - List masterItemDtos = new ArrayList<>(); - List masters = new ArrayList<>(); - - for (MiddleData middle : middleDataList) { - String addr3 = null; - if (middle.getAddr1() != null && middle.getAddr1().startsWith("서울특별시")) { - addr3 = "서울시 " + middle.getAddr1().split(" ")[1]; - } + public void loadData(List list, String lang) throws IOException { - Point geom = null; - try { - if (middle.getMapX() != null && middle.getMapY() != null) { - double x = Double.parseDouble(middle.getMapX()); - double y = Double.parseDouble(middle.getMapY()); - geom = geometryFactory.createPoint(new org.locationtech.jts.geom.Coordinate(x, y)); - } - } catch (NumberFormatException e) { - geom = null; - } + String date = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); - Master master = Master.builder() - .contentId(middle.getContentId()) - .contentTypeId(middle.getContentTypeId()) - .cat1(middle.getCat1()) - .cat2(middle.getCat2()) - .cat3(middle.getCat3()) - .ocat1(middle.getCat1()) - .ocat2(middle.getCat2()) - .ocat3(middle.getCat3()) - .outl(middle.getOutl()) - .title(middle.getTitle()) - .addr1(middle.getAddr1()) - .addr2(middle.getAddr2()) - .addr3(addr3) - .mapX(middle.getMapX() != null ? Double.valueOf(middle.getMapX()) : null) - .mapY(middle.getMapY() != null ? Double.valueOf(middle.getMapY()) : null) - .modifiedtime(middle.getModifiedtime()) - .tel(middle.getTel()) - .mLevel( - (middle.getMLevel() != null && !middle.getMLevel().isBlank()) ? Integer.parseInt(middle.getMLevel().trim()) : null - ) - .overview(middle.getOverview()) - .originImgUrl(middle.getOriginImgUrl()) - .smallImgUrl(middle.getSmallImgUrl()) - .usetime(middle.getUsetime()) - .restdate(middle.getRestdate()) - .parking(middle.getParking()) - .usefee(middle.getUsefee()) - .geom(geom) - .build(); + if ("ko".equals(lang)) { + List entities = + list.stream().map(MasterItemDto::toKo).toList(); - masters.add(master); + koMasterRepository.saveAll(entities); - MasterItemDto dto = new MasterItemDto( - master.getContentId(), master.getContentTypeId(), master.getCat1(), master.getCat2(), master.getCat3(), - master.getOcat1(), master.getOcat2(), master.getOcat3(), master.getOutl(), master.getTitle(), - master.getAddr1(), master.getAddr2(), master.getAddr3(), master.getMapX(), master.getMapY(), - master.getModifiedtime(), master.getTel(), master.getMLevel(), master.getOverview(), master.getOriginImgUrl(), - master.getSmallImgUrl(), master.getUsetime(), master.getRestdate(), master.getParking(), master.getUsefee() + writeAndUploadJson( + entities, + "master_data_ko.json", + "master/master_data_ko_" + date + ".json" ); - masterItemDtos.add(dto); - } + } else if ("en".equals(lang)) { + List entities = + list.stream().map(MasterItemDto::toEn).toList(); - File jsonFile = new File("master_data.json"); + enMasterRepository.saveAll(entities); - masterRepository.saveAll(masters); - objectMapper - .getFactory() - .setStreamWriteConstraints( - StreamWriteConstraints.builder() - .maxNestingDepth(3000) - .build() - ); + writeAndUploadJson( + entities, + "master_data_en.json", + "master/master_data_en_" + date + ".json" + ); + } + } - objectMapper.enable(SerializationFeature.INDENT_OUTPUT); - objectMapper.writeValue(jsonFile, masterItemDtos); // S3 저장으로 리팩토링 예정 + private void writeAndUploadJson( + Object data, + String fileName, + String s3Key + ) throws IOException { - String date = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); - String s3Key = "master/master_data_" + date + ".json"; + File jsonFile = new File(fileName); + + objectMapper.writeValue(jsonFile, data); amazonS3.putObject( new PutObjectRequest( @@ -239,179 +172,482 @@ public void createMasterTable() throws IOException { ); } - @Transactional - public List getMiddleData(List filteredData) { + public List processBatch(List batch, String commonPath, String detailPath, String lang) { + List masterItemDtoList = new ArrayList<>(); - this.key = ApiKeys.split(",")[0]; - this.index = 0; + for (AreaItemDto areaItemDto : batch) { - List middleDataList = new ArrayList<>(); + DetailItemDto detailItemDto = fetchTourApiDetail(areaItemDto, detailPath); + if (detailItemDto == null) continue; - for (FilteredDataDto filtered : filteredData) { + String overview = fetchTourApiCommon(areaItemDto, commonPath); + if (overview == null) continue; - String contentId = filtered.contentId(); - String contentTypeId = filtered.contentTypeId(); + SubCategoryGroup subCat = SubCategoryGroup.fromCode(areaItemDto.catCode()); + CategoryGroup cat = subCat.getCategoryGroup(); - JsonNode commonNode = fetchTourApi( - "https://apis.data.go.kr/B551011/KorService2/detailCommon2", - contentId, - null - ); + String cat1, cat2, cat1Code, cat2Code; + if (lang.equals("ko")) { + cat1 = cat.getKoName(); + cat2 = subCat.getKoName(); + cat1Code = cat.getCode(); + cat2Code = subCat.getCode(); + } else { + cat1 = cat.getEnName(); + cat2 = cat.getEnName(); + cat1Code = cat.getCode(); + cat2Code = cat.getCode(); + } - JsonNode detailNode = fetchTourApi( - "https://apis.data.go.kr/B551011/KorService2/detailIntro2", - contentId, - contentTypeId - ); + MasterItemDto masterItemDto = MasterItemDto.builder() + .contentId(areaItemDto.contentId()) + .cat1(cat1) + .cat2(cat2) + .cat1Code(cat1Code) + .cat2Code(cat2Code) + .contentTypeId(areaItemDto.contentTypeId()) + .addr1(areaItemDto.address()) + .addr2(areaItemDto.detailAddress()) + .addr3(parseSeoulAddr(areaItemDto.address())) + .mapX(areaItemDto.mapX()) + .mapY(areaItemDto.mapY()) + .mLevel(areaItemDto.mlevel()) + .modifiedtime(areaItemDto.modifiedtime()) + .originImgUrl(areaItemDto.imageUrl()) + .title(areaItemDto.title()) + .smallImgUrl(areaItemDto.thumbnailUrl()) + .overview(overview) + .parking(detailItemDto.parking()) + .usefee(detailItemDto.usefee()) + .restdate(detailItemDto.restdate()) + .tel(areaItemDto.tel()) + .usetime(detailItemDto.usetime()) + .build(); + masterItemDtoList.add(masterItemDto); + } - JsonNode imageNode = fetchTourApi( - "https://apis.data.go.kr/B551011/KorService2/detailImage2", - contentId, - null - ); + return masterItemDtoList; + } - if (commonNode == null || detailNode == null || imageNode == null) continue; - - JsonNode commonItem = getItem(commonNode); - JsonNode detailItem = getItem(detailNode); - JsonNode imageItem = getItem(imageNode); - - CommonItemDto commonDto = parseCommon(commonItem); - DetailItemDto detailDto = parseDetail(detailItem, contentTypeId); - ImageItemDto imageDto = parseImage(imageItem); - - MiddleDataDto dto = new MiddleDataDto( - contentId, - contentTypeId, - filtered.cat1(), - filtered.cat2(), - filtered.cat3(), - filtered.outl(), - commonDto.title(), - commonDto.addr1(), - commonDto.addr2(), - commonDto.mapX(), - commonDto.mapY(), - commonDto.modifiedTime(), - commonDto.tel(), - commonDto.mLevel(), - commonDto.overview(), - imageDto.imgrul(), - imageDto.smallimgurl(), - detailDto.usetime(), - detailDto.restdate(), - detailDto.parking(), - detailDto.usefee() + public List fetchTourApiArea(String path) { + List result = new ArrayList<>(); + + int numOfRow = 100; + int pageNo = 1; + int now = 0; + + do { + final int currentPage = pageNo; + + AreaResponse response = executeWithRetry(() -> + restClient.get() + .uri(uriBuilder -> uriBuilder + .path(path) + .queryParam("serviceKey", getValidKey()) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("_type", "json") + .queryParam("numOfRows", numOfRow) + .queryParam("pageNo", currentPage) + .queryParam("areaCode", "1") + .build() + ) + .retrieve() + .body(AreaResponse.class) ); - middleDataList.add(dto); - } + AreaResponse.Body body = response.response().body(); + + List items = + body.items() != null ? body.items().item() : List.of(); + + result.addAll(items); + now = body.numOfRows(); + pageNo++; - middleRepository.saveAll( - middleDataList.stream() - .map(MiddleData::new) - .toList() + } while (now == numOfRow); + log.info("Tour API AREA : DONE"); + return result.stream() + .filter(ExcludeRule::isInclude) + .map(AreaItemDto::from).toList(); + } + + public DetailItemDto fetchTourApiDetail(AreaItemDto areaItemDto, String prefix) { + DetailType detailType = DetailType.from(areaItemDto.contentTypeId()); + + DetailResponse response = executeWithRetry(() -> + restClient.get() + .uri(uriBuilder -> uriBuilder + .path(prefix) + .queryParam("contentId", areaItemDto.contentId()) + .queryParam("contentTypeId", detailType.getId()) + .queryParam("serviceKey", getValidKey()) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("_type", "json") + .build() + ) + .retrieve() + .body(detailType.getResponseType()) ); - return middleDataList; + return (response == null) ? null : response.getDetailItemDto(); } - public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) { - - RestClient client = RestClient.create(); - - URI uri = UriComponentsBuilder - .fromHttpUrl(url) - .queryParam("serviceKey", this.key) - .queryParam("MobileApp", "AppTest") - .queryParam("MobileOS", "ETC") - .queryParam("pageNo", 1) - .queryParam("numOfRows", 10) - .queryParam("_type", "json") - .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) - .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) - .build(true) - .toUri(); - - try { - return client.get().uri(uri).retrieve().body(JsonNode.class); - - } catch (Exception e) { - // key 변경 후 retry - if (this.index + 1 < ApiKeys.split(",").length) { - this.index++; - this.key = ApiKeys.split(",")[this.index]; - return fetchTourApi(url, contentId, contentTypeId); - } + public String fetchTourApiCommon(AreaItemDto areaItemDto, String prefix) { + CommonResponse response = executeWithRetry(() -> + restClient.get() + .uri(uriBuilder -> uriBuilder + .path(prefix) + .queryParam("contentId", areaItemDto.contentId()) + .queryParam("serviceKey", getValidKey()) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("_type", "json") + .build() + ) + .retrieve() + .body(CommonResponse.class) + ); + + if (response == null || response.response() == null || response.response().body().items() == null) { + log.info("[비었음] 컨텐츠 아이디 : {} ", areaItemDto.contentId()); return null; } + return response.response().body().items().item().get(0).overview(); } - private JsonNode getItem(JsonNode root) { - return root.path("response").path("body") - .path("items").path("item").get(0); - } + public void createMaster() { - private CommonItemDto parseCommon(JsonNode item) { - if (item == null) { - return new CommonItemDto(null, null, null, null, null, null, null, null, null); - } + //ist middleItemDtos = fetchTourApiCommon(areaItemDtos); + } - return new CommonItemDto( - item.path("title").asText(), - item.path("addr1").asText(), - item.path("addr2").asText(), - item.path("mapx").asText(), - item.path("mapy").asText(), - item.path("modifiedtime").asText(), - item.path("tel").asText(), - item.path("mlevel").asText(), - item.path("overview").asText() - ); + @Transactional + public void createCategory() { + List categories = categoryRepository.findAll(); + + List spotCategories = categories.stream() + .filter(category -> category.getCode().length() == 5) + .map(category -> SubCategoryGroup.from(category)) // Optional + .filter(Optional::isPresent) // 없는 건 걸러내기 + .map(Optional::get) + .map(subCategoryGroup -> SpotCategory.builder() + .originCode(subCategoryGroup.getCode()) + .code(subCategoryGroup.getCode()) + .EngName(subCategoryGroup.getEnName()) + .KoName(subCategoryGroup.getKoName()) + .parentCode(subCategoryGroup.getCategoryGroup().getCode()) + .parentEnName(subCategoryGroup.getCategoryGroup().getEnName()) + .parentKoName(subCategoryGroup.getCategoryGroup().getKoName()) + .build()) + .toList(); + spotCategoryRepository.saveAll(spotCategories); } - private ImageItemDto parseImage(JsonNode item) { - if (item == null) return new ImageItemDto(null, null); + private String parseSeoulAddr(String addr) { + if (addr == null) return null; - return new ImageItemDto( - item.path("originimgurl").asText(null), - item.path("smallimageurl").asText(null) - ); - } + Pattern pattern = Pattern.compile("^서울특별시\\s+(\\S+)"); + Matcher matcher = pattern.matcher(addr); - private DetailItemDto parseDetail(JsonNode item, String typeId) { - if (item == null) return new DetailItemDto(null, null, null, null); + if (matcher.find()) { + return "서울시 " + matcher.group(1); + } - return switch (typeId) { - case "12" -> new DetailItemDto( - null, - item.path("parking").asText(), - item.path("restdate").asText(), - item.path("usetime").asText() - ); - case "14" -> new DetailItemDto( - item.path("usefee").asText(), - item.path("parkingculture").asText(), - item.path("restdateculture").asText(), - item.path("usetimeculture").asText() - ); - case "15" -> new DetailItemDto( - item.path("usetimefestival").asText(), - null, null, null - ); - case "28" -> new DetailItemDto( - item.path("usefeeleports").asText(), - item.path("parkingleports").asText(), - item.path("restdateleports").asText(), - item.path("usetimeleports").asText() - ); - case "38" -> new DetailItemDto( - null, - item.path("parkingshopping").asText(), - null, - null - ); - default -> new DetailItemDto(null, null, null, null); - }; + return null; } } + +// +// public List getRawInfo() { +// try { +// InputStream rawJson = new ClassPathResource("TourAPI_seoul.json").getInputStream(); +// List rawJsonDtos = objectMapper.readValue( +// rawJson, +// new TypeReference>() { +// } +// ); +// +// List filtered = rawJsonDtos.stream() +// .filter(dto -> !dto.contentTypeId().equals("25")) +// .filter(dto -> !dto.contentTypeId().equals("32")) +// .filter(dto -> !dto.contentTypeId().equals("39")) +// .filter(dto -> !dto.cat3().equals("A04011000")) +// .toList(); +// +// return filtered; +// +// } catch (Exception e) { +// return List.of(); +// } +// } +//} +// @Transactional +// public void createMasterTable() throws IOException { +// List middleDataList = middleRepository.findAll(); +// List masterItemDtos = new ArrayList<>(); +// List masters = new ArrayList<>(); +// +// for (MiddleData middle : middleDataList) { +// String addr3 = null; +// if (middle.getAddr1() != null && middle.getAddr1().startsWith("서울특별시")) { +// addr3 = "서울시 " + middle.getAddr1().split(" ")[1]; +// } +// +// Point geom = null; +// try { +// if (middle.getMapX() != null && middle.getMapY() != null) { +// double x = Double.parseDouble(middle.getMapX()); +// double y = Double.parseDouble(middle.getMapY()); +// geom = geometryFactory.createPoint(new org.locationtech.jts.geom.Coordinate(x, y)); +// } +// } catch (NumberFormatException e) { +// geom = null; +// } +// +// Master master = Master.builder() +// .contentId(middle.getContentId()) +// .contentTypeId(middle.getContentTypeId()) +// .cat1(middle.getCat1()) +// .cat2(middle.getCat2()) +// .cat3(middle.getCat3()) +// .ocat1(middle.getCat1()) +// .ocat2(middle.getCat2()) +// .ocat3(middle.getCat3()) +// .outl(middle.getOutl()) +// .title(middle.getTitle()) +// .addr1(middle.getAddr1()) +// .addr2(middle.getAddr2()) +// .addr3(addr3) +// .mapX(middle.getMapX() != null ? Double.valueOf(middle.getMapX()) : null) +// .mapY(middle.getMapY() != null ? Double.valueOf(middle.getMapY()) : null) +// .modifiedtime(middle.getModifiedtime()) +// .tel(middle.getTel()) +// .mLevel( +// (middle.getMLevel() != null && !middle.getMLevel().isBlank()) ? Integer.parseInt(middle.getMLevel().trim()) : null +// ) +// .overview(middle.getOverview()) +// .originImgUrl(middle.getOriginImgUrl()) +// .smallImgUrl(middle.getSmallImgUrl()) +// .usetime(middle.getUsetime()) +// .restdate(middle.getRestdate()) +// .parking(middle.getParking()) +// .usefee(middle.getUsefee()) +// .geom(geom) +// .build(); +// +// masters.add(master); +// +// MasterItemDto dto = new MasterItemDto( +// master.getContentId(), master.getContentTypeId(), master.getCat1(), master.getCat2(), master.getCat3(), +// master.getOcat1(), master.getOcat2(), master.getOcat3(), master.getOutl(), master.getTitle(), +// master.getAddr1(), master.getAddr2(), master.getAddr3(), master.getMapX(), master.getMapY(), +// master.getModifiedtime(), master.getTel(), master.getMLevel(), master.getOverview(), master.getOriginImgUrl(), +// master.getSmallImgUrl(), master.getUsetime(), master.getRestdate(), master.getParking(), master.getUsefee() +// ); +// +// masterItemDtos.add(dto); +// } +// +// File jsonFile = new File("master_data.json"); +// +// masterRepository.saveAll(masters); +// objectMapper +// .getFactory() +// .setStreamWriteConstraints( +// StreamWriteConstraints.builder() +// .maxNestingDepth(3000) +// .build() +// ); +// +// objectMapper.enable(SerializationFeature.INDENT_OUTPUT); +// objectMapper.writeValue(jsonFile, masterItemDtos); // S3 저장으로 리팩토링 예정 +// +// String date = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); +// String s3Key = "master/master_data_" + date + ".json"; +// +// amazonS3.putObject( +// new PutObjectRequest( +// bucketName, +// s3Key, +// jsonFile +// ) +// ); +// } +// +// @Transactional +// public List getMiddleData(List filteredData) { +// +//// this.key = ApiKeys.split(",")[0]; +//// this.index = 0; +// +// List middleDataList = new ArrayList<>(); +// +// for (FilteredDataDto filtered : filteredData) { +// +// String contentId = filtered.contentId(); +// String contentTypeId = filtered.contentTypeId(); +// +// JsonNode commonNode = fetchTourApi( +// "https://apis.data.go.kr/B551011/KorService2/detailCommon2", +// contentId, +// null +// ); +// +// JsonNode detailNode = fetchTourApi( +// "https://apis.data.go.kr/B551011/KorService2/detailIntro2", +// contentId, +// contentTypeId +// ); +// +// JsonNode imageNode = fetchTourApi( +// "https://apis.data.go.kr/B551011/KorService2/detailImage2", +// contentId, +// null +// ); +// +// if (commonNode == null || detailNode == null || imageNode == null) continue; +// +// JsonNode commonItem = getItem(commonNode); +// JsonNode detailItem = getItem(detailNode); +// JsonNode imageItem = getItem(imageNode); +// +// CommonItemDto commonDto = parseCommon(commonItem); +// DetailItemDto detailDto = parseDetail(detailItem, contentTypeId); +// ImageItemDto imageDto = parseImage(imageItem); +// +// MiddleDataDto dto = new MiddleDataDto( +// contentId, +// contentTypeId, +// filtered.cat1(), +// filtered.cat2(), +// filtered.cat3(), +// filtered.outl(), +// commonDto.title(), +// commonDto.addr1(), +// commonDto.addr2(), +// commonDto.mapX(), +// commonDto.mapY(), +// commonDto.modifiedTime(), +// commonDto.tel(), +// commonDto.mLevel(), +// commonDto.overview(), +// imageDto.imgrul(), +// imageDto.smallimgurl(), +// detailDto.usetime(), +// detailDto.restdate(), +// detailDto.parking(), +// detailDto.usefee() +// ); +// +// middleDataList.add(dto); +// } +// +// middleRepository.saveAll( +// middleDataList.stream() +// .map(MiddleData::new) +// .toList() +// ); +// +// return middleDataList; +// } +// +//// public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) { +//// +//// RestClient client = RestClient.create(); +//// +//// URI uri = UriComponentsBuilder +//// .fromHttpUrl(url) +//// .queryParam("serviceKey", this.key) +//// .queryParam("MobileApp", "AppTest") +//// .queryParam("MobileOS", "ETC") +//// .queryParam("pageNo", 1) +//// .queryParam("numOfRows", 10) +//// .queryParam("_type", "json") +//// .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) +//// .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) +//// .build(true) +//// .toUri(); +//// +//// try { +//// return client.get().uri(uri).retrieve().body(JsonNode.class); +//// +//// } catch (Exception e) { +////// // key 변경 후 retry +////// if (this.index + 1 < ApiKeys.split(",").length) { +////// this.index++; +////// this.key = ApiKeys.split(",")[this.index]; +////// return fetchTourApi(url, contentId, contentTypeId); +////// } +//// return null; +//// } +//// } +// +// private JsonNode getItem(JsonNode root) { +// return root.path("response").path("body") +// .path("items").path("item").get(0); +// } +// +// private CommonItemDto parseCommon(JsonNode item) { +// if (item == null) { +// return new CommonItemDto(null, null, null, null, null, null, null, null, null); +// } +// +// return new CommonItemDto( +// item.path("title").asText(), +// item.path("addr1").asText(), +// item.path("addr2").asText(), +// item.path("mapx").asText(), +// item.path("mapy").asText(), +// item.path("modifiedtime").asText(), +// item.path("tel").asText(), +// item.path("mlevel").asText(), +// item.path("overview").asText() +// ); +// } +// +// private ImageItemDto parseImage(JsonNode item) { +// if (item == null) return new ImageItemDto(null, null); +// +// return new ImageItemDto( +// item.path("originimgurl").asText(null), +// item.path("smallimageurl").asText(null) +// ); +// } +// +// private DetailItemDto parseDetail(JsonNode item, String typeId) { +// if (item == null) return new DetailItemDto(null, null, null, null); +// +// return switch (typeId) { +// case "12" -> new DetailItemDto( +// null, +// item.path("parking").asText(), +// item.path("restdate").asText(), +// item.path("usetime").asText() +// ); +// case "14" -> new DetailItemDto( +// item.path("usefee").asText(), +// item.path("parkingculture").asText(), +// item.path("restdateculture").asText(), +// item.path("usetimeculture").asText() +// ); +// case "15" -> new DetailItemDto( +// item.path("usetimefestival").asText(), +// null, null, null +// ); +// case "28" -> new DetailItemDto( +// item.path("usefeeleports").asText(), +// item.path("parkingleports").asText(), +// item.path("restdateleports").asText(), +// item.path("usetimeleports").asText() +// ); +// case "38" -> new DetailItemDto( +// null, +// item.path("parkingshopping").asText(), +// null, +// null +// ); +// default -> new DetailItemDto(null, null, null, null); +// }; +// } +//} diff --git a/src/main/java/com/earseo/core/service/master/CategoryGroup.java b/src/main/java/com/earseo/core/service/master/CategoryGroup.java new file mode 100644 index 0000000..231a9b2 --- /dev/null +++ b/src/main/java/com/earseo/core/service/master/CategoryGroup.java @@ -0,0 +1,20 @@ +package com.earseo.core.service.master; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public enum CategoryGroup { + + NA("NA", "자연", "Nature"), + CU("CU", "문화", "Culture"), + AR("AR", "예술", "Art"), + HI("HI", "역사", "History"), + LS("LS", "레포츠", "Leisure/Sports"), + SH("SH", "쇼핑", "Shopping"); + + private final String code; + private final String koName; + private final String enName; +} diff --git a/src/main/java/com/earseo/core/service/master/DetailType.java b/src/main/java/com/earseo/core/service/master/DetailType.java new file mode 100644 index 0000000..f1d1e28 --- /dev/null +++ b/src/main/java/com/earseo/core/service/master/DetailType.java @@ -0,0 +1,28 @@ +package com.earseo.core.service.master; + +import com.earseo.core.dto.tourApi.*; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Arrays; + +@Getter +@RequiredArgsConstructor +public enum DetailType { + SIGHT("12", SightDetailResponse.class), + CULTURE("14", CultureDetailResponse.class), + FESTIVAL("15", FestivalDetailResponse.class), + LEPORTS("28", LeportsDetailResponse.class), + SHOPPING("38", ShoppingDetailResponse.class); + + private final String id; + private final Class responseType; + + public static DetailType from(String contentTypeId){ + return Arrays.stream(values()) + .filter(t-> t.id.equals(contentTypeId)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "컨턴츠타입 매칭 에러 : " + contentTypeId)); + } +} diff --git a/src/main/java/com/earseo/core/service/master/ExcludeRule.java b/src/main/java/com/earseo/core/service/master/ExcludeRule.java new file mode 100644 index 0000000..d3e725e --- /dev/null +++ b/src/main/java/com/earseo/core/service/master/ExcludeRule.java @@ -0,0 +1,31 @@ +package com.earseo.core.service.master; + +import com.earseo.core.dto.tourApi.AreaResponse; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Set; + +@Getter +@RequiredArgsConstructor +public enum ExcludeRule { + + CONTENT_TYPE_ID("25", "32", "39"), + CAT3("A04011000"); + + private final Set rules; + + ExcludeRule(String... rules) { + this.rules = Set.of(rules); + } + + public static boolean shouldExclude(AreaResponse.Item item) { + return CONTENT_TYPE_ID.rules.contains(String.valueOf(item.contenttypeid())) + || CAT3.rules.contains(String.valueOf(item.cat3())); + } + + public static boolean isInclude(AreaResponse.Item item) { + return !shouldExclude(item); + } + +} diff --git a/src/main/java/com/earseo/core/service/master/SubCategoryGroup.java b/src/main/java/com/earseo/core/service/master/SubCategoryGroup.java new file mode 100644 index 0000000..8905216 --- /dev/null +++ b/src/main/java/com/earseo/core/service/master/SubCategoryGroup.java @@ -0,0 +1,62 @@ +package com.earseo.core.service.master; + +import com.earseo.core.entity.Category; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +@Getter +@RequiredArgsConstructor +public enum SubCategoryGroup { + + NA01(CategoryGroup.NA, "NA01", "자연관광지", "Natural Sites", List.of("A0101")), + NA02(CategoryGroup.NA, "NA02", "관광자원", "Natural Resources", List.of("A0102")), + + CU01(CategoryGroup.CU, "CU01", "휴양관광지", "Recreational Sites", List.of("A0202")), + CU02(CategoryGroup.CU, "CU02", "체험관광지", "Experience Programs", List.of("A0203")), + CU03(CategoryGroup.CU, "CU03", "산업관광지", "Industrial Sites", List.of("A0204")), + CU04(CategoryGroup.CU, "CU04", "문화시설", "Cultural Facilities", List.of("A0206")), + + AR01(CategoryGroup.AR, "AR01", "축제", "Festivals", List.of("A0207")), + AR02(CategoryGroup.AR, "AR02", "공연/행사", "Events/Performances", List.of("A0208")), + + HI01(CategoryGroup.HI, "HI01", "역사관광지", "Historical Sites", List.of("A0201")), + HI02(CategoryGroup.HI, "HI02", "건축/조형물", "Architectural Sights", List.of("A0205")), + + LS01(CategoryGroup.LS, "LS01", "레포츠소개", "Introduction", List.of("A0301")), + LS02(CategoryGroup.LS, "LS02", "육상 레포츠", "Land Sports", List.of("A0302")), + LS03(CategoryGroup.LS, "LS03", "수상 레포츠", "Water Sports", List.of("A0303")), + LS04(CategoryGroup.LS, "LS04", "항공 레포츠", "Sky Sports", List.of("A0304")), + LS05(CategoryGroup.LS, "LS05", "복합 레포츠", "Others", List.of("A0305")), + + SH01(CategoryGroup.SH, "SH01", "쇼핑", "Shopping", List.of("A0401")); + + private final CategoryGroup categoryGroup; + private final String code; + private final String koName; + private final String enName; + private final List originPrefixes; + + public static Optional from(Category category) { + return Arrays.stream(values()) + .filter(st -> + st.originPrefixes.stream() + .anyMatch(prefix -> category.getCode().startsWith(prefix)) + ) + .findFirst(); + } + public static SubCategoryGroup fromCode(String code) { + return Arrays.stream(values()) + .filter(st -> + st.originPrefixes.stream() + .anyMatch(prefix -> code.startsWith(prefix)) + ) + .findFirst() + .orElseThrow(() -> + new IllegalArgumentException("카테고리 파싱 에러 : " + code) + ); + } +} diff --git a/src/main/java/com/earseo/core/service/master/TourApiPath.java b/src/main/java/com/earseo/core/service/master/TourApiPath.java new file mode 100644 index 0000000..9bb70d2 --- /dev/null +++ b/src/main/java/com/earseo/core/service/master/TourApiPath.java @@ -0,0 +1,19 @@ +package com.earseo.core.service.master; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +@AllArgsConstructor +public enum TourApiPath { + KoArea("/B551011/KorService2/areaBasedList2"), + EnArea("/B551011/EngService2/areaBasedList2"), + KoCommon("/B551011/KorService2/detailCommon2"), + EnCommon("/B551011/EngService2/detailCommon2"), + KoDetail("/B551011/KorService2/detailIntro2"), + EnDetail("/B551011/EngService2/detailIntro2"); + + private String path; +} From 414f9a2f81a190ddc99a21ac496032dd9a4f9c0e Mon Sep 17 00:00:00 2001 From: haribonyam Date: Sun, 18 Jan 2026 16:40:08 +0900 Subject: [PATCH 02/10] =?UTF-8?q?[#9]=20feat:=20=EB=8B=A4=EA=B5=AD?= =?UTF-8?q?=EC=96=B4=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EC=88=98=EC=A7=91=20?= =?UTF-8?q?-=20tour=20api=20=EC=97=90=EC=84=9C=20=EC=A0=9C=EA=B3=B5?= =?UTF-8?q?=ED=95=98=EB=8A=94=20=EC=98=81=EB=AC=B8=20=EB=8D=B0=EC=9D=B4?= =?UTF-8?q?=ED=84=B0=20=EC=88=98=EC=A7=91=20=EB=A1=9C=EC=A7=81=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=20-=20tour=20api=20rule=EC=97=90=20=EB=A7=9E=EA=B2=8C?= =?UTF-8?q?=20=EC=98=88=EC=99=B8=20=ED=83=80=EC=9E=85=20=EC=B6=94=EA=B0=80?= =?UTF-8?q?=20-=20=EA=B8=B0=EC=A1=B4=20=EC=88=98=EC=A7=91=20=EB=A1=9C?= =?UTF-8?q?=EC=A7=81=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/earseo/core/entity/EnMaster.java | 14 +- .../core/service/MasterDataService.java | 325 +----------------- .../core/service/master/DetailType.java | 15 +- .../core/service/master/ExcludeRule.java | 2 + 4 files changed, 25 insertions(+), 331 deletions(-) diff --git a/src/main/java/com/earseo/core/entity/EnMaster.java b/src/main/java/com/earseo/core/entity/EnMaster.java index 61784eb..606f812 100644 --- a/src/main/java/com/earseo/core/entity/EnMaster.java +++ b/src/main/java/com/earseo/core/entity/EnMaster.java @@ -1,8 +1,5 @@ package com.earseo.core.entity; -import com.earseo.core.dto.etl.MasterItemDto; -import com.earseo.core.service.master.CategoryGroup; -import com.earseo.core.service.master.SubCategoryGroup; import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; @@ -13,9 +10,9 @@ @NoArgsConstructor @AllArgsConstructor @Getter @Builder -public class EnMaster { @Id - -@GeneratedValue(strategy = GenerationType.IDENTITY) +public class EnMaster { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @Column(name = "content_id") @@ -84,9 +81,4 @@ public class EnMaster { @Id @Column(name = "use_fee", columnDefinition = "TEXT") private String usefee; - public EnMaster from(MasterItemDto masterItemDto){ - return EnMaster.builder() - .build(); - } - } \ No newline at end of file diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index 01647ce..d690d4e 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -9,23 +9,18 @@ import com.earseo.core.entity.*; import com.earseo.core.repository.*; import com.earseo.core.service.master.*; -import com.fasterxml.jackson.core.StreamWriteConstraints; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import jakarta.annotation.PostConstruct; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.locationtech.jts.geom.GeometryFactory; import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.ClassPathResource; import org.springframework.stereotype.Service; import org.springframework.web.client.RestClient; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -74,10 +69,6 @@ private void rotateKey() { keyIndex = (keyIndex + 1) % keyList.size(); } - public List getTourApiArea(String tourApiPrefix) { - return fetchTourApiArea(tourApiPrefix); - } - private T executeWithRetry(Supplier apiCall) { int retryCount = 0; while (retryCount < keyList.size() * 2) { @@ -106,6 +97,9 @@ private T executeWithRetry(Supplier apiCall) { throw new RuntimeException("TOUR API 호출 에러"); } + public List getTourApiArea(String tourApiPrefix) { + return fetchTourApiArea(tourApiPrefix); + } public void createMasterTable(List areaItemList, String commonPath, String detailPath, String lang) throws IOException { int batchSize = BATCH_SIZE; @@ -178,10 +172,15 @@ public List processBatch(List batch, String commonPa for (AreaItemDto areaItemDto : batch) { DetailItemDto detailItemDto = fetchTourApiDetail(areaItemDto, detailPath); - if (detailItemDto == null) continue; + if (detailItemDto == null){ + log.info("디테일 비었음 : {} , {}", areaItemDto.contentTypeId(), areaItemDto.contentId()); + continue;} String overview = fetchTourApiCommon(areaItemDto, commonPath); - if (overview == null) continue; + if (overview == null){ + log.info("common 비었음 : {} , {}", areaItemDto.contentTypeId(), areaItemDto.contentId()); + continue; + } SubCategoryGroup subCat = SubCategoryGroup.fromCode(areaItemDto.catCode()); CategoryGroup cat = subCat.getCategoryGroup(); @@ -280,7 +279,7 @@ public DetailItemDto fetchTourApiDetail(AreaItemDto areaItemDto, String prefix) .uri(uriBuilder -> uriBuilder .path(prefix) .queryParam("contentId", areaItemDto.contentId()) - .queryParam("contentTypeId", detailType.getId()) + .queryParam("contentTypeId", areaItemDto.contentTypeId()) .queryParam("serviceKey", getValidKey()) .queryParam("MobileApp", "AppTest") .queryParam("MobileOS", "ETC") @@ -290,7 +289,6 @@ public DetailItemDto fetchTourApiDetail(AreaItemDto areaItemDto, String prefix) .retrieve() .body(detailType.getResponseType()) ); - return (response == null) ? null : response.getDetailItemDto(); } @@ -317,11 +315,6 @@ public String fetchTourApiCommon(AreaItemDto areaItemDto, String prefix) { return response.response().body().items().item().get(0).overview(); } - public void createMaster() { - - //ist middleItemDtos = fetchTourApiCommon(areaItemDtos); - } - @Transactional public void createCategory() { List categories = categoryRepository.findAll(); @@ -356,298 +349,4 @@ private String parseSeoulAddr(String addr) { return null; } -} - -// -// public List getRawInfo() { -// try { -// InputStream rawJson = new ClassPathResource("TourAPI_seoul.json").getInputStream(); -// List rawJsonDtos = objectMapper.readValue( -// rawJson, -// new TypeReference>() { -// } -// ); -// -// List filtered = rawJsonDtos.stream() -// .filter(dto -> !dto.contentTypeId().equals("25")) -// .filter(dto -> !dto.contentTypeId().equals("32")) -// .filter(dto -> !dto.contentTypeId().equals("39")) -// .filter(dto -> !dto.cat3().equals("A04011000")) -// .toList(); -// -// return filtered; -// -// } catch (Exception e) { -// return List.of(); -// } -// } -//} -// @Transactional -// public void createMasterTable() throws IOException { -// List middleDataList = middleRepository.findAll(); -// List masterItemDtos = new ArrayList<>(); -// List masters = new ArrayList<>(); -// -// for (MiddleData middle : middleDataList) { -// String addr3 = null; -// if (middle.getAddr1() != null && middle.getAddr1().startsWith("서울특별시")) { -// addr3 = "서울시 " + middle.getAddr1().split(" ")[1]; -// } -// -// Point geom = null; -// try { -// if (middle.getMapX() != null && middle.getMapY() != null) { -// double x = Double.parseDouble(middle.getMapX()); -// double y = Double.parseDouble(middle.getMapY()); -// geom = geometryFactory.createPoint(new org.locationtech.jts.geom.Coordinate(x, y)); -// } -// } catch (NumberFormatException e) { -// geom = null; -// } -// -// Master master = Master.builder() -// .contentId(middle.getContentId()) -// .contentTypeId(middle.getContentTypeId()) -// .cat1(middle.getCat1()) -// .cat2(middle.getCat2()) -// .cat3(middle.getCat3()) -// .ocat1(middle.getCat1()) -// .ocat2(middle.getCat2()) -// .ocat3(middle.getCat3()) -// .outl(middle.getOutl()) -// .title(middle.getTitle()) -// .addr1(middle.getAddr1()) -// .addr2(middle.getAddr2()) -// .addr3(addr3) -// .mapX(middle.getMapX() != null ? Double.valueOf(middle.getMapX()) : null) -// .mapY(middle.getMapY() != null ? Double.valueOf(middle.getMapY()) : null) -// .modifiedtime(middle.getModifiedtime()) -// .tel(middle.getTel()) -// .mLevel( -// (middle.getMLevel() != null && !middle.getMLevel().isBlank()) ? Integer.parseInt(middle.getMLevel().trim()) : null -// ) -// .overview(middle.getOverview()) -// .originImgUrl(middle.getOriginImgUrl()) -// .smallImgUrl(middle.getSmallImgUrl()) -// .usetime(middle.getUsetime()) -// .restdate(middle.getRestdate()) -// .parking(middle.getParking()) -// .usefee(middle.getUsefee()) -// .geom(geom) -// .build(); -// -// masters.add(master); -// -// MasterItemDto dto = new MasterItemDto( -// master.getContentId(), master.getContentTypeId(), master.getCat1(), master.getCat2(), master.getCat3(), -// master.getOcat1(), master.getOcat2(), master.getOcat3(), master.getOutl(), master.getTitle(), -// master.getAddr1(), master.getAddr2(), master.getAddr3(), master.getMapX(), master.getMapY(), -// master.getModifiedtime(), master.getTel(), master.getMLevel(), master.getOverview(), master.getOriginImgUrl(), -// master.getSmallImgUrl(), master.getUsetime(), master.getRestdate(), master.getParking(), master.getUsefee() -// ); -// -// masterItemDtos.add(dto); -// } -// -// File jsonFile = new File("master_data.json"); -// -// masterRepository.saveAll(masters); -// objectMapper -// .getFactory() -// .setStreamWriteConstraints( -// StreamWriteConstraints.builder() -// .maxNestingDepth(3000) -// .build() -// ); -// -// objectMapper.enable(SerializationFeature.INDENT_OUTPUT); -// objectMapper.writeValue(jsonFile, masterItemDtos); // S3 저장으로 리팩토링 예정 -// -// String date = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); -// String s3Key = "master/master_data_" + date + ".json"; -// -// amazonS3.putObject( -// new PutObjectRequest( -// bucketName, -// s3Key, -// jsonFile -// ) -// ); -// } -// -// @Transactional -// public List getMiddleData(List filteredData) { -// -//// this.key = ApiKeys.split(",")[0]; -//// this.index = 0; -// -// List middleDataList = new ArrayList<>(); -// -// for (FilteredDataDto filtered : filteredData) { -// -// String contentId = filtered.contentId(); -// String contentTypeId = filtered.contentTypeId(); -// -// JsonNode commonNode = fetchTourApi( -// "https://apis.data.go.kr/B551011/KorService2/detailCommon2", -// contentId, -// null -// ); -// -// JsonNode detailNode = fetchTourApi( -// "https://apis.data.go.kr/B551011/KorService2/detailIntro2", -// contentId, -// contentTypeId -// ); -// -// JsonNode imageNode = fetchTourApi( -// "https://apis.data.go.kr/B551011/KorService2/detailImage2", -// contentId, -// null -// ); -// -// if (commonNode == null || detailNode == null || imageNode == null) continue; -// -// JsonNode commonItem = getItem(commonNode); -// JsonNode detailItem = getItem(detailNode); -// JsonNode imageItem = getItem(imageNode); -// -// CommonItemDto commonDto = parseCommon(commonItem); -// DetailItemDto detailDto = parseDetail(detailItem, contentTypeId); -// ImageItemDto imageDto = parseImage(imageItem); -// -// MiddleDataDto dto = new MiddleDataDto( -// contentId, -// contentTypeId, -// filtered.cat1(), -// filtered.cat2(), -// filtered.cat3(), -// filtered.outl(), -// commonDto.title(), -// commonDto.addr1(), -// commonDto.addr2(), -// commonDto.mapX(), -// commonDto.mapY(), -// commonDto.modifiedTime(), -// commonDto.tel(), -// commonDto.mLevel(), -// commonDto.overview(), -// imageDto.imgrul(), -// imageDto.smallimgurl(), -// detailDto.usetime(), -// detailDto.restdate(), -// detailDto.parking(), -// detailDto.usefee() -// ); -// -// middleDataList.add(dto); -// } -// -// middleRepository.saveAll( -// middleDataList.stream() -// .map(MiddleData::new) -// .toList() -// ); -// -// return middleDataList; -// } -// -//// public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) { -//// -//// RestClient client = RestClient.create(); -//// -//// URI uri = UriComponentsBuilder -//// .fromHttpUrl(url) -//// .queryParam("serviceKey", this.key) -//// .queryParam("MobileApp", "AppTest") -//// .queryParam("MobileOS", "ETC") -//// .queryParam("pageNo", 1) -//// .queryParam("numOfRows", 10) -//// .queryParam("_type", "json") -//// .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) -//// .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) -//// .build(true) -//// .toUri(); -//// -//// try { -//// return client.get().uri(uri).retrieve().body(JsonNode.class); -//// -//// } catch (Exception e) { -////// // key 변경 후 retry -////// if (this.index + 1 < ApiKeys.split(",").length) { -////// this.index++; -////// this.key = ApiKeys.split(",")[this.index]; -////// return fetchTourApi(url, contentId, contentTypeId); -////// } -//// return null; -//// } -//// } -// -// private JsonNode getItem(JsonNode root) { -// return root.path("response").path("body") -// .path("items").path("item").get(0); -// } -// -// private CommonItemDto parseCommon(JsonNode item) { -// if (item == null) { -// return new CommonItemDto(null, null, null, null, null, null, null, null, null); -// } -// -// return new CommonItemDto( -// item.path("title").asText(), -// item.path("addr1").asText(), -// item.path("addr2").asText(), -// item.path("mapx").asText(), -// item.path("mapy").asText(), -// item.path("modifiedtime").asText(), -// item.path("tel").asText(), -// item.path("mlevel").asText(), -// item.path("overview").asText() -// ); -// } -// -// private ImageItemDto parseImage(JsonNode item) { -// if (item == null) return new ImageItemDto(null, null); -// -// return new ImageItemDto( -// item.path("originimgurl").asText(null), -// item.path("smallimageurl").asText(null) -// ); -// } -// -// private DetailItemDto parseDetail(JsonNode item, String typeId) { -// if (item == null) return new DetailItemDto(null, null, null, null); -// -// return switch (typeId) { -// case "12" -> new DetailItemDto( -// null, -// item.path("parking").asText(), -// item.path("restdate").asText(), -// item.path("usetime").asText() -// ); -// case "14" -> new DetailItemDto( -// item.path("usefee").asText(), -// item.path("parkingculture").asText(), -// item.path("restdateculture").asText(), -// item.path("usetimeculture").asText() -// ); -// case "15" -> new DetailItemDto( -// item.path("usetimefestival").asText(), -// null, null, null -// ); -// case "28" -> new DetailItemDto( -// item.path("usefeeleports").asText(), -// item.path("parkingleports").asText(), -// item.path("restdateleports").asText(), -// item.path("usetimeleports").asText() -// ); -// case "38" -> new DetailItemDto( -// null, -// item.path("parkingshopping").asText(), -// null, -// null -// ); -// default -> new DetailItemDto(null, null, null, null); -// }; -// } -//} +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/service/master/DetailType.java b/src/main/java/com/earseo/core/service/master/DetailType.java index f1d1e28..e310ccb 100644 --- a/src/main/java/com/earseo/core/service/master/DetailType.java +++ b/src/main/java/com/earseo/core/service/master/DetailType.java @@ -5,22 +5,23 @@ import lombok.RequiredArgsConstructor; import java.util.Arrays; +import java.util.Set; @Getter @RequiredArgsConstructor public enum DetailType { - SIGHT("12", SightDetailResponse.class), - CULTURE("14", CultureDetailResponse.class), - FESTIVAL("15", FestivalDetailResponse.class), - LEPORTS("28", LeportsDetailResponse.class), - SHOPPING("38", ShoppingDetailResponse.class); + SIGHT(Set.of("12","76"), SightDetailResponse.class), + CULTURE(Set.of("14","78"), CultureDetailResponse.class), + FESTIVAL(Set.of("15","85"), FestivalDetailResponse.class), + LEPORTS(Set.of("28", "75"), LeportsDetailResponse.class), + SHOPPING(Set.of("38", "79"), ShoppingDetailResponse.class); - private final String id; + private final Set id; private final Class responseType; public static DetailType from(String contentTypeId){ return Arrays.stream(values()) - .filter(t-> t.id.equals(contentTypeId)) + .filter(t-> t.id.contains(contentTypeId)) .findFirst() .orElseThrow(() -> new IllegalArgumentException( "컨턴츠타입 매칭 에러 : " + contentTypeId)); diff --git a/src/main/java/com/earseo/core/service/master/ExcludeRule.java b/src/main/java/com/earseo/core/service/master/ExcludeRule.java index d3e725e..1564cd8 100644 --- a/src/main/java/com/earseo/core/service/master/ExcludeRule.java +++ b/src/main/java/com/earseo/core/service/master/ExcludeRule.java @@ -11,6 +11,7 @@ public enum ExcludeRule { CONTENT_TYPE_ID("25", "32", "39"), + CONTENT_TYPE_ID_EN("80","82","77"), CAT3("A04011000"); private final Set rules; @@ -21,6 +22,7 @@ public enum ExcludeRule { public static boolean shouldExclude(AreaResponse.Item item) { return CONTENT_TYPE_ID.rules.contains(String.valueOf(item.contenttypeid())) + || CONTENT_TYPE_ID_EN.rules.contains(String.valueOf(item.contenttypeid())) || CAT3.rules.contains(String.valueOf(item.cat3())); } From 1241dba21a6c69b9ac41aa9dc06088779275a62c Mon Sep 17 00:00:00 2001 From: yoonho Date: Thu, 22 Jan 2026 06:49:20 +0900 Subject: [PATCH 03/10] =?UTF-8?q?[#9]=20feat:=20=EB=8F=84=EC=8A=A8?= =?UTF-8?q?=ED=8A=B8=20=EB=8B=A4=EA=B5=AD=EC=96=B4=20=EC=A7=80=EC=9B=90=20?= =?UTF-8?q?-=20=EB=A7=88=EC=8A=A4=ED=84=B0=20=ED=85=8C=EC=9D=B4=EB=B8=94?= =?UTF-8?q?=20=EB=B3=80=EA=B2=BD=EC=A0=90=20=EC=A0=81=EC=9A=A9=20-=20?= =?UTF-8?q?=EC=98=81=EB=AC=B8=20=EB=8F=84=EC=8A=A8=ED=8A=B8=20=EC=83=9D?= =?UTF-8?q?=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/controller/DocentController.java | 11 ++++--- .../com/earseo/core/dto/etl/JoinItemDto.java | 10 ++++++ .../core/repository/OdiiDataRepository.java | 22 +++++++++---- .../earseo/core/service/DocentService.java | 32 +++++++++++-------- 4 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 src/main/java/com/earseo/core/dto/etl/JoinItemDto.java diff --git a/src/main/java/com/earseo/core/controller/DocentController.java b/src/main/java/com/earseo/core/controller/DocentController.java index 819180d..bf621e1 100644 --- a/src/main/java/com/earseo/core/controller/DocentController.java +++ b/src/main/java/com/earseo/core/controller/DocentController.java @@ -5,6 +5,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; @RestController @@ -14,10 +15,12 @@ public class DocentController { private final DocentService docentService; @PostMapping("/admin/core/docent") - public ResponseEntity> initDocent() { - docentService.initDocent(); - docentService.getDocent(); - docentService.getDocentJson(); + public ResponseEntity> initDocent( + @RequestBody String lang + ) { + docentService.initDocent(lang); + docentService.getDocent(lang); + docentService.getDocentJson(lang); return ResponseEntity.ok(BaseResponse.ok(null)); } } diff --git a/src/main/java/com/earseo/core/dto/etl/JoinItemDto.java b/src/main/java/com/earseo/core/dto/etl/JoinItemDto.java new file mode 100644 index 0000000..a532d77 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/JoinItemDto.java @@ -0,0 +1,10 @@ +package com.earseo.core.dto.etl; + +public record JoinItemDto( + Long id, + String contentId, + String title, + String overview, + String script +) { +} diff --git a/src/main/java/com/earseo/core/repository/OdiiDataRepository.java b/src/main/java/com/earseo/core/repository/OdiiDataRepository.java index be1d781..1699b4f 100644 --- a/src/main/java/com/earseo/core/repository/OdiiDataRepository.java +++ b/src/main/java/com/earseo/core/repository/OdiiDataRepository.java @@ -10,12 +10,22 @@ public interface OdiiDataRepository extends JpaRepository { @Query(value = """ - SELECT DISTINCT m.id, m.content_id, m.title, MIN(o.script), m.outl + SELECT DISTINCT km.id, km.content_id, km.title, MIN(o.script), km.overview FROM odii_data o - RIGHT JOIN master m ON m.title = o.title - WHERE m.title IS NOT NULL - GROUP BY m.id, m.content_id, m.title, m.outl - ORDER BY m.id + RIGHT JOIN ko_master km ON km.title = o.title + WHERE km.title IS NOT NULL + GROUP BY km.id, km.content_id, km.title, km.overview + ORDER BY km.id """, nativeQuery = true) - List joinWithMaster(); + List joinWithMasterKo(); + + @Query(value = """ + SELECT DISTINCT em.id, em.content_id, em.title, MIN(o.script), em.overview + FROM odii_data o + RIGHT JOIN en_master em ON em.title = o.title + WHERE em.title IS NOT NULL + GROUP BY em.id, em.content_id, em.title, em.overview + ORDER BY em.id + """, nativeQuery = true) + List joinWithMasterEn(); } diff --git a/src/main/java/com/earseo/core/service/DocentService.java b/src/main/java/com/earseo/core/service/DocentService.java index b3c6e44..a3fc266 100644 --- a/src/main/java/com/earseo/core/service/DocentService.java +++ b/src/main/java/com/earseo/core/service/DocentService.java @@ -60,13 +60,13 @@ public class DocentService { private int index; @Transactional - public void initDocent() { + public void initDocent(String lang) { int pageNo = 1; int numOfRows = 100; while (true) { - JsonNode jsonNode = fetchOdiiApi(pageNo, numOfRows); + JsonNode jsonNode = fetchOdiiApi(pageNo, numOfRows, lang); JsonNode body = jsonNode.path("response").path("body"); if (body.path("numOfRows").asInt() == 0) { @@ -76,7 +76,6 @@ public void initDocent() { List odiiDataList = new ArrayList<>(); if (body.path("items").path("item").isArray()) { - System.out.println("hello"); for (JsonNode item : body.path("items").path("item")) { odiiDataList.add(OdiiData.builder().title(item.path("title").asText()).script(item.path("script").asText()).build()); } @@ -87,15 +86,20 @@ public void initDocent() { } } - public void getDocent() { - List joinItems = odiiDataRepository.joinWithMaster(); + public void getDocent(String lang) { + List joinItems; + if (lang.equals("en")) { + joinItems = odiiDataRepository.joinWithMasterEn(); + } else { + joinItems = odiiDataRepository.joinWithMasterKo(); + } int chunkSize = 10; for (int i = 0; i < joinItems.size(); i += chunkSize) { List chunk = joinItems.subList(i, Math.min(i + chunkSize, joinItems.size())); try { - processChunk(chunk); + processChunk(chunk, lang); } catch (Exception e) { return; } @@ -103,12 +107,12 @@ public void getDocent() { } @Transactional(propagation = Propagation.REQUIRES_NEW) - public void processChunk(List chunk) { + public void processChunk(List chunk, String lang) { List docents = new ArrayList<>(); for (JoinItemDto data : chunk) { log.info("current item id : " + data.id()); - String source = data.outl(); + String source = data.overview(); if (data.script() != null) { source = data.script(); @@ -117,7 +121,7 @@ public void processChunk(List chunk) { String prompt = String.format(DOCENT_SCRIPT.message, data.title(), source); try { String script = chatClient.prompt(prompt).call().content(); - String docentUrl = getDocentUrl(data.contentId(), script, "ko"); + String docentUrl = getDocentUrl(data.contentId(), script, lang); Docent docent = Docent.builder() .contentId(data.contentId()) @@ -135,7 +139,7 @@ public void processChunk(List chunk) { docentRepository.saveAll(docents); } - public String getDocentJson(){ + public String getDocentJson(String lang){ try{ List docents = docentRepository.findAll(); @@ -152,7 +156,7 @@ public String getDocentJson(){ objectMapper.enable(SerializationFeature.INDENT_OUTPUT); objectMapper.writeValue(tempFile, exportData); - String s3Key = "core/docent/docent_data.json"; + String s3Key = String.format("core/docent/docent_data.json_%s_", lang); objectMapper .getFactory() @@ -233,12 +237,12 @@ public String getDocentUrl(String contentId, String script, String lang) { } } - public JsonNode fetchOdiiApi(int pageNo, int numOfRows) { + public JsonNode fetchOdiiApi(int pageNo, int numOfRows, String lang) { this.key = ApiKeys.split(",")[0]; this.index = 0; RestClient client = RestClient.create(); - URI uri = UriComponentsBuilder.fromHttpUrl("https://apis.data.go.kr/B551011/Odii/storyBasedList").queryParam("serviceKey", this.key).queryParam("MobileApp", "AppTest").queryParam("MobileOS", "ETC").queryParam("pageNo", pageNo).queryParam("numOfRows", numOfRows).queryParam("_type", "json").queryParam("langCode", "ko").build(true).toUri(); + URI uri = UriComponentsBuilder.fromHttpUrl("https://apis.data.go.kr/B551011/Odii/storyBasedList").queryParam("serviceKey", this.key).queryParam("MobileApp", "AppTest").queryParam("MobileOS", "ETC").queryParam("pageNo", pageNo).queryParam("numOfRows", numOfRows).queryParam("_type", "json").queryParam("langCode", lang).build(true).toUri(); try { return client.get().uri(uri).retrieve().body(JsonNode.class); } catch (Exception e) { @@ -246,7 +250,7 @@ public JsonNode fetchOdiiApi(int pageNo, int numOfRows) { if (this.index + 1 < ApiKeys.split(",").length) { this.index++; this.key = ApiKeys.split(",")[this.index]; - return fetchOdiiApi(pageNo, numOfRows); + return fetchOdiiApi(pageNo, numOfRows, lang); } return null; } From 0a0c8c41f01582519cf42924adb97090b45bd5f5 Mon Sep 17 00:00:00 2001 From: yoonho Date: Thu, 22 Jan 2026 14:18:26 +0900 Subject: [PATCH 04/10] =?UTF-8?q?[#9]=20refactor:=20=EB=8F=84=EC=8A=A8?= =?UTF-8?q?=ED=8A=B8=20json=20=EC=83=9D=EC=84=B1=20=EC=88=98=EC=A0=95=20-?= =?UTF-8?q?=20=EC=96=B8=EC=96=B4=EC=99=80=20=EC=83=81=EA=B4=80=EC=97=86?= =?UTF-8?q?=EA=B2=8C=20=ED=86=B5=EC=9D=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/earseo/core/controller/DocentController.java | 2 +- src/main/java/com/earseo/core/service/DocentService.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/earseo/core/controller/DocentController.java b/src/main/java/com/earseo/core/controller/DocentController.java index bf621e1..4b92039 100644 --- a/src/main/java/com/earseo/core/controller/DocentController.java +++ b/src/main/java/com/earseo/core/controller/DocentController.java @@ -20,7 +20,7 @@ public ResponseEntity> initDocent( ) { docentService.initDocent(lang); docentService.getDocent(lang); - docentService.getDocentJson(lang); + docentService.getDocentJson(); return ResponseEntity.ok(BaseResponse.ok(null)); } } diff --git a/src/main/java/com/earseo/core/service/DocentService.java b/src/main/java/com/earseo/core/service/DocentService.java index a3fc266..3bd83d4 100644 --- a/src/main/java/com/earseo/core/service/DocentService.java +++ b/src/main/java/com/earseo/core/service/DocentService.java @@ -139,7 +139,7 @@ public void processChunk(List chunk, String lang) { docentRepository.saveAll(docents); } - public String getDocentJson(String lang){ + public String getDocentJson(){ try{ List docents = docentRepository.findAll(); @@ -156,7 +156,7 @@ public String getDocentJson(String lang){ objectMapper.enable(SerializationFeature.INDENT_OUTPUT); objectMapper.writeValue(tempFile, exportData); - String s3Key = String.format("core/docent/docent_data.json_%s_", lang); + String s3Key = "core/docent/docent_data.json"; objectMapper .getFactory() From 4838e5e6f2a7b28a5f1ab198f7d98b0f72ae3c50 Mon Sep 17 00:00:00 2001 From: yoonho Date: Thu, 22 Jan 2026 15:24:36 +0900 Subject: [PATCH 05/10] =?UTF-8?q?[#9]=20fix:=20=EB=8B=A4=EA=B5=AD=EC=96=B4?= =?UTF-8?q?=20=EC=84=B8=EB=B6=80=20=EC=B9=B4=ED=85=8C=EA=B3=A0=EB=A6=AC=20?= =?UTF-8?q?=EC=A0=81=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/earseo/core/service/MasterDataService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index d690d4e..d7cc0f5 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -193,9 +193,9 @@ public List processBatch(List batch, String commonPa cat2Code = subCat.getCode(); } else { cat1 = cat.getEnName(); - cat2 = cat.getEnName(); + cat2 = subCat.getEnName(); cat1Code = cat.getCode(); - cat2Code = cat.getCode(); + cat2Code = subCat.getCode(); } MasterItemDto masterItemDto = MasterItemDto.builder() From 592d748dbfa3f0ca8a46c37136bd8f5350fd89c0 Mon Sep 17 00:00:00 2001 From: yoonho Date: Thu, 22 Jan 2026 16:24:07 +0900 Subject: [PATCH 06/10] =?UTF-8?q?[#9]=20feat:=20=EC=98=81=EB=AC=B8=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EC=A3=BC=EC=86=8C=20=ED=8C=8C?= =?UTF-8?q?=EC=8B=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/earseo/core/service/MasterDataService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index d7cc0f5..fbb55f7 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -347,6 +347,11 @@ private String parseSeoulAddr(String addr) { return "서울시 " + matcher.group(1); } + String[] originAddr = addr.split(","); + if(originAddr.length > 1){ + return originAddr[originAddr.length-2]+", "+originAddr[originAddr.length-1].trim(); + } + return null; } } \ No newline at end of file From 19734bac797b4f3968d3d664f57a7d6b36d67fde Mon Sep 17 00:00:00 2001 From: yoonho Date: Thu, 22 Jan 2026 18:04:09 +0900 Subject: [PATCH 07/10] =?UTF-8?q?[#9]=20feat:=20=EA=B4=80=EA=B4=91?= =?UTF-8?q?=EC=A7=80=20=ED=85=8C=EB=A7=88=20=EB=AA=A9=EB=A1=9D=20=EC=A1=B0?= =?UTF-8?q?=ED=9A=8C=20API=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/controller/MasterDataController.java | 6 ++++++ .../core/dto/response/ThemeListResponse.java | 6 ++++++ .../core/dto/response/ThemeResponse.java | 19 +++++++++++++++++++ .../core/service/MasterDataService.java | 11 +++++++++++ 4 files changed, 42 insertions(+) create mode 100644 src/main/java/com/earseo/core/dto/response/ThemeListResponse.java create mode 100644 src/main/java/com/earseo/core/dto/response/ThemeResponse.java diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java index 995f138..1428add 100644 --- a/src/main/java/com/earseo/core/controller/MasterDataController.java +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -2,6 +2,7 @@ import com.earseo.core.common.BaseResponse; import com.earseo.core.dto.etl.AreaItemDto; +import com.earseo.core.dto.response.ThemeListResponse; import com.earseo.core.service.MasterDataService; import com.earseo.core.service.master.TourApiPath; import lombok.RequiredArgsConstructor; @@ -37,4 +38,9 @@ public ResponseEntity> init(){ masterDataService.createCategory(); return ResponseEntity.ok(BaseResponse.ok(null)); } + + @GetMapping("/core/theme") + public ResponseEntity> createTableCategory(){ + return ResponseEntity.ok(BaseResponse.ok(masterDataService.getAllThemes())); + } } diff --git a/src/main/java/com/earseo/core/dto/response/ThemeListResponse.java b/src/main/java/com/earseo/core/dto/response/ThemeListResponse.java new file mode 100644 index 0000000..afee071 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/response/ThemeListResponse.java @@ -0,0 +1,6 @@ +package com.earseo.core.dto.response; + +import java.util.List; + +public record ThemeListResponse(List themeList) { +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/dto/response/ThemeResponse.java b/src/main/java/com/earseo/core/dto/response/ThemeResponse.java new file mode 100644 index 0000000..9a70226 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/response/ThemeResponse.java @@ -0,0 +1,19 @@ +package com.earseo.core.dto.response; + +import com.earseo.core.service.master.CategoryGroup; +import com.earseo.core.service.master.SubCategoryGroup; + +public record ThemeResponse( + String code, + String koName, + String enName +) { + + public static ThemeResponse fromSub(SubCategoryGroup subCategoryGroup) { + return new ThemeResponse(subCategoryGroup.getCode(), subCategoryGroup.getKoName(), subCategoryGroup.getEnName()); + } + + public static ThemeResponse from(CategoryGroup categoryGroup) { + return new ThemeResponse(categoryGroup.getCode(), categoryGroup.getKoName(), categoryGroup.getEnName()); + } +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index fbb55f7..bbca6f8 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -3,6 +3,8 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.PutObjectRequest; import com.earseo.core.dto.etl.*; +import com.earseo.core.dto.response.ThemeListResponse; +import com.earseo.core.dto.response.ThemeResponse; import com.earseo.core.dto.tourApi.AreaResponse; import com.earseo.core.dto.tourApi.CommonResponse; import com.earseo.core.dto.tourApi.DetailResponse; @@ -354,4 +356,13 @@ private String parseSeoulAddr(String addr) { return null; } + + public ThemeListResponse getAllThemes() { + List themeResponses = new ArrayList<>(); + + themeResponses.addAll(Arrays.stream(SubCategoryGroup.values()).map(ThemeResponse::fromSub).toList()); + themeResponses.addAll(Arrays.stream(CategoryGroup.values()).map(ThemeResponse::from).toList()); + + return new ThemeListResponse(themeResponses); + } } \ No newline at end of file From bb72457319c0b781a802aed947f92bbc0d6d29e2 Mon Sep 17 00:00:00 2001 From: yoonho Date: Tue, 27 Jan 2026 00:46:49 +0900 Subject: [PATCH 08/10] =?UTF-8?q?[#9]=20feat:=20swagger=20cors=20=EC=84=A4?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/earseo/core/common/config/SwaggerConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/earseo/core/common/config/SwaggerConfig.java b/src/main/java/com/earseo/core/common/config/SwaggerConfig.java index 7480e31..17ab4a3 100644 --- a/src/main/java/com/earseo/core/common/config/SwaggerConfig.java +++ b/src/main/java/com/earseo/core/common/config/SwaggerConfig.java @@ -3,6 +3,7 @@ import io.swagger.v3.oas.models.Components; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.servers.Server; import org.springdoc.core.models.GroupedOpenApi; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -18,6 +19,7 @@ public class SwaggerConfig { @Bean public OpenAPI defaultOpenAPI(@Value("${spring.application.name}") String appName) { return new OpenAPI() + .addServersItem(new Server().url("/")) .components(new Components()) .info(new Info() .title(appName + " API") From f30764cb90e2ad72aa13104d1b5397e6858e1b2b Mon Sep 17 00:00:00 2001 From: yoonho Date: Tue, 27 Jan 2026 00:48:07 +0900 Subject: [PATCH 09/10] =?UTF-8?q?[#9]=20refactor:=20api=20path=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/earseo/core/controller/DocentController.java | 2 +- .../com/earseo/core/controller/MasterDataController.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/earseo/core/controller/DocentController.java b/src/main/java/com/earseo/core/controller/DocentController.java index 4b92039..a1fbdee 100644 --- a/src/main/java/com/earseo/core/controller/DocentController.java +++ b/src/main/java/com/earseo/core/controller/DocentController.java @@ -14,7 +14,7 @@ public class DocentController { private final DocentService docentService; - @PostMapping("/admin/core/docent") + @PostMapping("/api/admin/core/docent") public ResponseEntity> initDocent( @RequestBody String lang ) { diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java index 1428add..5c0b02b 100644 --- a/src/main/java/com/earseo/core/controller/MasterDataController.java +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -19,27 +19,27 @@ public class MasterDataController { private final MasterDataService masterDataService; - @GetMapping("/admin/core/master/ko") + @GetMapping("/api/admin/core/master/ko") public ResponseEntity> createTableKo() throws IOException { List list = masterDataService.getTourApiArea(TourApiPath.KoArea.getPath()); masterDataService.createMasterTable(list, TourApiPath.KoCommon.getPath(), TourApiPath.KoDetail.getPath(), "ko"); return ResponseEntity.ok(BaseResponse.ok(null)); } - @GetMapping("/admin/core/master/en") + @GetMapping("/api/admin/core/master/en") public ResponseEntity> createTableEn() throws IOException { List list = masterDataService.getTourApiArea(TourApiPath.EnArea.getPath()); masterDataService.createMasterTable(list, TourApiPath.EnCommon.getPath(), TourApiPath.EnDetail.getPath(), "en"); return ResponseEntity.ok(BaseResponse.ok(null)); } - @GetMapping("/admin/core/init") + @GetMapping("/api/admin/core/init") public ResponseEntity> init(){ masterDataService.createCategory(); return ResponseEntity.ok(BaseResponse.ok(null)); } - @GetMapping("/core/theme") + @GetMapping("/api/core/theme") public ResponseEntity> createTableCategory(){ return ResponseEntity.ok(BaseResponse.ok(masterDataService.getAllThemes())); } From 0f7a1243c15fd1a85cb1906d0fdbea53b91b34bb Mon Sep 17 00:00:00 2001 From: yoonho Date: Tue, 27 Jan 2026 21:54:26 +0900 Subject: [PATCH 10/10] =?UTF-8?q?[#9]=20chore:=20v0.1.2=20=EB=B0=B0?= =?UTF-8?q?=ED=8F=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- k8s/helm-value.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/helm-value.yaml b/k8s/helm-value.yaml index cec13e8..32ff051 100644 --- a/k8s/helm-value.yaml +++ b/k8s/helm-value.yaml @@ -1,2 +1,2 @@ image: - tag: v0.1.1 + tag: v0.1.2