diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 5a81f3ca..df5053e9 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -11,6 +11,10 @@ beekeeper-api + + 2.6.1 + + org.springframework.boot @@ -20,6 +24,36 @@ org.springframework.boot spring-boot-actuator + + com.expediagroup + beekeeper-core + ${project.version} + + + net.kaczmarzyk + specification-arg-resolver + ${specification-arg-resolver.version} + + + org.json + json + 20080701 + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.projectlombok + lombok + + + org.projectlombok + lombok + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java new file mode 100644 index 00000000..c2a5fbd3 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +@SpringBootApplication +@EnableConfigurationProperties +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +@ComponentScan(basePackages = { + "com.expediagroup.beekeeper.api.conf", + "com.expediagroup.beekeeper.api.controller", + "com.expediagroup.beekeeper.api.service" }) +public class BeekeeperApiApplication { + + @Autowired + private ObjectMapper objectMapper; + + public static void main(String[] args) { + SpringApplication.run(BeekeeperApiApplication.class, args); + } + + @PostConstruct + public void setUp() { + objectMapper.registerModule(new JavaTimeModule()); + } +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java new file mode 100644 index 00000000..dd7f1338 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.conf; + +import java.util.List; + +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.web.method.support.HandlerMethodArgumentResolver; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import net.kaczmarzyk.spring.data.jpa.web.SpecificationArgumentResolver; + +@Configuration +@EnableJpaRepositories +public class JPAConfiguration implements WebMvcConfigurer { + + @Override + public void addArgumentResolvers(List argumentResolvers) { + argumentResolvers.add(new SpecificationArgumentResolver()); + } +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java new file mode 100644 index 00000000..5bb33fa3 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.controller; + + +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + +import static com.expediagroup.beekeeper.api.response.HousekeepingMetadataResponse.convertToHouseKeepingMetadataResponsePage; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import net.kaczmarzyk.spring.data.jpa.domain.EqualIgnoreCase; +import net.kaczmarzyk.spring.data.jpa.domain.GreaterThan; +import net.kaczmarzyk.spring.data.jpa.domain.LessThan; +import net.kaczmarzyk.spring.data.jpa.web.annotation.And; +import net.kaczmarzyk.spring.data.jpa.web.annotation.Spec; + +import com.expediagroup.beekeeper.api.response.HousekeepingMetadataResponse; +import com.expediagroup.beekeeper.api.service.BeekeeperService; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + +@RequestMapping("/api/v1") +@RestController +public class BeekeeperController { + private final BeekeeperService beekeeperService; + + @Autowired + public BeekeeperController(HousekeepingMetadataService housekeepingMetadataService, + BeekeeperService beekeeperService) { + this.beekeeperService = beekeeperService; + } + + @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE) + public ResponseEntity> getAll( + @And({ + @Spec(path = "tableName", params = "table_name", spec = EqualIgnoreCase.class), + @Spec(path = "databaseName", params = "database_name", spec = EqualIgnoreCase.class), + @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), + @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), + @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), + @Spec(path = "cleanupTimestamp", params = "deleted_after", spec = GreaterThan.class), + @Spec(path = "creationTimestamp", params = "registered_before", spec = LessThan.class), + @Spec(path = "creationTimestamp", params = "registered_after", spec = GreaterThan.class) + }) + Specification spec, Pageable pageable) { + return ResponseEntity.ok(beekeeperService.getAllTables(spec, pageable)); + } + +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java new file mode 100644 index 00000000..e69c58b8 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java @@ -0,0 +1,103 @@ +package com.expediagroup.beekeeper.api.response; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.persistence.Column; +import javax.persistence.Table; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; + +import lombok.Builder; +import lombok.Value; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + +@Value +@Builder +@Table(name = "housekeeping_metadata") +public class HousekeepingMetadataResponse { + + @Column(name = "database_name", nullable = false) + String databaseName; + + @Column(name = "table_name", nullable = false) + String tableName; + + @Column(name = "path", nullable = false) + String path; + + @Column(name = "lifecycles", nullable = false) + List lifecycles; + + public static HousekeepingMetadataResponse convertToHouseKeepingMetadataResponse( + HousekeepingMetadata housekeepingMetadata) { + List lifecyclesList = new ArrayList<>(); + Lifecycle lifecycle = Lifecycle.builder() + .lifecycleEventType(housekeepingMetadata.getLifecycleType()) + .configuration(Map.of( + "beekeeper.unreferenced.data.retention.period",housekeepingMetadata.getCleanupDelay().toString(), + "clean up timestamp", housekeepingMetadata.getCleanupTimestamp().toString() + )) + .build() + ; + lifecyclesList.add(lifecycle); + + return HousekeepingMetadataResponse.builder() + .databaseName(housekeepingMetadata.getDatabaseName()) + .tableName(housekeepingMetadata.getTableName()) + .path(housekeepingMetadata.getPath()) + .lifecycles(lifecyclesList) + .build(); + } + + public static Page convertToHouseKeepingMetadataResponsePage(List housekeepingMetadataList){ + List housekeepingMetadataResponseList = new ArrayList<>(); + for (HousekeepingMetadata housekeepingMetadata : housekeepingMetadataList) { + HousekeepingMetadataResponse housekeepingMetadataResponse = convertToHouseKeepingMetadataResponse(housekeepingMetadata); + int repeatedTablePosition = checkIfTableExists(housekeepingMetadataResponseList, housekeepingMetadata); + + if(repeatedTablePosition!=-1){ + housekeepingMetadataResponse = housekeepingMetadataResponseList.get(repeatedTablePosition); + housekeepingMetadataResponseList.remove(repeatedTablePosition); + Lifecycle lifecycle = Lifecycle.builder() + .lifecycleEventType(housekeepingMetadata.getLifecycleType()) + .configuration(Map.of( + "beekeeper.unreferenced.data.retention.period",housekeepingMetadata.getCleanupDelay().toString(), + "clean up timestamp", housekeepingMetadata.getCleanupTimestamp().toString() + )) + .build(); + housekeepingMetadataResponse.addLifecycle(lifecycle); + } + + housekeepingMetadataResponseList.add(housekeepingMetadataResponse); + } + return new PageImpl<>(housekeepingMetadataResponseList); + } + + public static int checkIfTableExists( + List housekeepingMetadataResponseList, HousekeepingMetadata housekeepingMetadata){ + int count = -1; + int positionOfRepeatedTable = -1; + String tableName1 = housekeepingMetadata.getTableName(); + String databaseName1 = housekeepingMetadata.getDatabaseName(); + if(!housekeepingMetadataResponseList.isEmpty()) { + for (HousekeepingMetadataResponse table : housekeepingMetadataResponseList) { + count++; + String tableName2 = table.getTableName(); + String databaseName2 = table.getDatabaseName(); + if (tableName1.equals(tableName2) && databaseName1.equals(databaseName2)) { + positionOfRepeatedTable = count; + } + } + } + return positionOfRepeatedTable; + } + + public void addLifecycle(Lifecycle lifecycle){ + lifecycles.add(lifecycle); + } + +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/Lifecycle.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/Lifecycle.java new file mode 100644 index 00000000..dea3c9a2 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/Lifecycle.java @@ -0,0 +1,17 @@ +package com.expediagroup.beekeeper.api.response; + +import java.util.Map; + +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class Lifecycle { + + String lifecycleEventType; + Map configuration; + + + +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/BeekeeperService.java new file mode 100644 index 00000000..69245c73 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/BeekeeperService.java @@ -0,0 +1,26 @@ +package com.expediagroup.beekeeper.api.service; + +import static com.expediagroup.beekeeper.api.response.HousekeepingMetadataResponse.convertToHouseKeepingMetadataResponsePage; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Service; + +import com.expediagroup.beekeeper.api.response.HousekeepingMetadataResponse; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + +@Service +public class BeekeeperService { + + private final HousekeepingMetadataService housekeepingMetadataService; + + @Autowired + public BeekeeperService( + HousekeepingMetadataService housekeepingMetadataService) {this.housekeepingMetadataService = housekeepingMetadataService;} + + public Page getAllTables(Specification spec, Pageable pageable) { + return convertToHouseKeepingMetadataResponsePage(housekeepingMetadataService.getAll(spec, pageable).getContent()); + } +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingEntityService.java similarity index 53% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingEntityService.java index e0bd2172..09d14f27 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingEntityService.java @@ -13,11 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api; +package com.expediagroup.beekeeper.api.service; -import org.springframework.web.bind.annotation.RestController; -@RestController -public class BeekeeperController { - +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; + +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; + +public interface HousekeepingEntityService { + + Page getAll(Specification spec, Pageable pageable); + } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java new file mode 100644 index 00000000..61553d14 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.service; + + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Service; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; + +@Service +public class HousekeepingMetadataService implements HousekeepingEntityService { + + private final HousekeepingMetadataRepository housekeepingMetadataRepository; + + @Autowired + public HousekeepingMetadataService(HousekeepingMetadataRepository housekeepingMetadataRepository) { + this.housekeepingMetadataRepository = housekeepingMetadataRepository; + } + + public Page getAll(Specification spec, Pageable pageable) { + return housekeepingMetadataRepository.findAll(spec, pageable); + } + +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingPathService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingPathService.java new file mode 100644 index 00000000..a165fe58 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingPathService.java @@ -0,0 +1,28 @@ +package com.expediagroup.beekeeper.api.service; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Service; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; + +@Service +public class HousekeepingPathService implements HousekeepingEntityService{ + + private final HousekeepingPathRepository housekeepingPathRepository; + + @Autowired + public HousekeepingPathService(HousekeepingPathRepository housekeepingPathRepository) { + this.housekeepingPathRepository = housekeepingPathRepository; + } + + @Override + public Page getAll(Specification spec, Pageable pageable) { + return housekeepingPathRepository.findAll(spec, pageable); + } +} diff --git a/beekeeper-api/src/main/resources/beekeeper-api-application.yml b/beekeeper-api/src/main/resources/beekeeper-api-application.yml new file mode 100644 index 00000000..dc42a41a --- /dev/null +++ b/beekeeper-api/src/main/resources/beekeeper-api-application.yml @@ -0,0 +1,13 @@ +management.endpoints.web.exposure.include: health,info,prometheus + +server: + port: 9008 + +spring: + jpa: + database: default + hibernate: + ddl-auto: validate + driver-class-name: org.h2.Driver + properties.hibernate: + dialect: org.hibernate.dialect.MySQL8Dialect diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApi.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java similarity index 69% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApi.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java index 22bbf7f1..8711afce 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApi.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java @@ -15,14 +15,15 @@ */ package com.expediagroup.beekeeper.api; -import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; @SpringBootApplication -public class BeekeeperApi { - - public static void main(String[] args) { - SpringApplication.run(BeekeeperApi.class, args); - } +@EnableConfigurationProperties +@ComponentScan(basePackages = { + "com.expediagroup.beekeeper.api.conf", + "com.expediagroup.beekeeper.api.controller"}) +public class TestApplication { } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java new file mode 100644 index 00000000..de293522 --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -0,0 +1,139 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.controller; + + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.MediaType; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.result.MockMvcResultHandlers; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.expediagroup.beekeeper.api.TestApplication; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + +@WebMvcTest(BeekeeperController.class) +@ContextConfiguration(classes = TestApplication.class) +public class BeekeeperControllerTest { + + @Autowired + private MockMvc mockMvc; + @Autowired + private ObjectMapper objectMapper; + + @Mock + private Specification spec; + + @Mock + private Pageable pageable; + + @MockBean + private HousekeepingMetadataService housekeepingMetadataService; + + @Test + public void testGetAllWhenTablesValid() throws Exception { + HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); + + when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataService); + } + + @Test + public void testGetAllWhenNoTables() throws Exception { + Page tables = new PageImpl<>(List.of()); + + when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataService); + } + + @Test + public void testControllerWhenWrongUrl() throws Exception { + Page tables = new PageImpl<>(List.of()); + + when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tablessssss")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isNotFound()); + } + + @Test + public void testPaging() throws Exception { + int pageNumber = 5; + int pageSize = 10; + HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); + + when(housekeepingMetadataService.getAll(any(), eq(PageRequest.of(pageNumber, pageSize)))).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables") + .param("page", String.valueOf(pageNumber)) + .param("size", String.valueOf(pageSize))) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataService); + } +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java new file mode 100644 index 00000000..c7df8654 --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; + +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; + +@ExtendWith(MockitoExtension.class) +public class HouseKeepingMetadataServiceTest { + + private HousekeepingMetadataService housekeepingMetadataService; + + @Mock + private HousekeepingMetadataRepository housekeepingMetadataRepository; + @Mock + private Specification spec; + @Mock + private Pageable pageable; + + @BeforeEach + public void beforeEach() { + housekeepingMetadataService = new HousekeepingMetadataService(housekeepingMetadataRepository); + } + + @Test + public void testGetAllWhenTablesValid(){ + HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + + + Page tables = new PageImpl<>(List.of(table1, table2)); + when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); + Page result = housekeepingMetadataService.getAll(spec, pageable); + + assertThat(tables).isEqualTo(result); + verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); + verifyNoMoreInteractions(housekeepingMetadataRepository); + } + +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java new file mode 100644 index 00000000..07a0dacb --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.util; + + +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZoneId; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + +public class DummyHousekeepingMetadataGenerator { + + private static final String DEFAULT_DB_NAME = "randomDatabase"; + private static final String DEFAULT_TABLE_NAME = "randomTable"; + private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + private static final Duration CLEANUP_DELAY = Duration.parse("P3D"); + + public static HousekeepingMetadata generateDummyHousekeepingMetadata() { + return generateDummyHousekeepingMetadata(DEFAULT_DB_NAME, DEFAULT_TABLE_NAME); + } + + public static HousekeepingMetadata generateDummyHousekeepingMetadata(String tableName, String databaseName) { + return HousekeepingMetadata.builder() + .path("s3://some/path/") + .databaseName(databaseName) + .tableName(tableName) + .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupDelay(Duration.parse("P3D")) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); + } +} diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java index 17a47689..d82399e7 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java @@ -69,7 +69,7 @@ void setUp() { .forEach(object -> amazonS3.deleteObject(bucket, object.getKey())); s3Client = new S3Client(amazonS3, dryRunEnabled); s3DryRunPathCleaner = new S3PathCleaner(s3Client, new S3SentinelFilesCleaner(s3Client), bytesDeletedReporter); - housekeepingPath = new HousekeepingPath.Builder() + housekeepingPath = HousekeepingPath.builder() .path(absolutePath) .tableName(tableName) .databaseName(databaseName) diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java index 9e2ffb8d..059c55f0 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java @@ -91,7 +91,7 @@ void setUp() { s3Client = new S3Client(amazonS3, dryRunEnabled); s3SentinelFilesCleaner = new S3SentinelFilesCleaner(s3Client); s3PathCleaner = new S3PathCleaner(s3Client, s3SentinelFilesCleaner, bytesDeletedReporter); - housekeepingPath = new HousekeepingPath.Builder() + housekeepingPath = HousekeepingPath.builder() .path(absolutePath) .tableName(tableName) .databaseName(databaseName) diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 2b128307..4e4978e8 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -11,8 +11,8 @@ beekeeper-core - 1.1.4 - 1.3.3 + 1.6.5 + 1.6.5 @@ -20,6 +20,10 @@ ch.qos.logback logback-core + + org.projectlombok + lombok + org.apache.commons diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java index eec8a180..a9435703 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,6 @@ */ package com.expediagroup.beekeeper.core.model; -import static java.lang.String.format; - import java.time.Duration; import java.time.LocalDateTime; @@ -32,9 +30,16 @@ import org.hibernate.annotations.UpdateTimestamp; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.monitoring.MetricTag; +@Data +@NoArgsConstructor @Entity @Table(name = "housekeeping_metadata") public class HousekeepingMetadata implements HousekeepingEntity { @@ -59,13 +64,16 @@ public class HousekeepingMetadata implements HousekeepingEntity { @Enumerated(EnumType.STRING) private HousekeepingStatus housekeepingStatus; + @EqualsAndHashCode.Exclude @Column(name = "creation_timestamp", nullable = false, updatable = false) private LocalDateTime creationTimestamp; + @EqualsAndHashCode.Exclude @Column(name = "modified_timestamp") @UpdateTimestamp private LocalDateTime modifiedTimestamp; + @EqualsAndHashCode.Exclude @Column(name = "cleanup_timestamp", nullable = false) private LocalDateTime cleanupTimestamp; @@ -78,16 +86,19 @@ public class HousekeepingMetadata implements HousekeepingEntity { @Column(name = "client_id") private String clientId; + @Column(name = "lifecycle_type", nullable = false) private String lifecycleType; - public HousekeepingMetadata() { - + public void setCleanupDelay(Duration cleanupDelay) { + this.cleanupDelay = cleanupDelay; + cleanupTimestamp = creationTimestamp.plus(cleanupDelay); } - private HousekeepingMetadata(Long id, String path, String databaseName, String tableName, + @Builder + public HousekeepingMetadata(Long id, String path, String databaseName, String tableName, String partitionName, HousekeepingStatus housekeepingStatus, LocalDateTime creationTimestamp, - LocalDateTime modifiedTimestamp, LocalDateTime cleanupTimestamp, Duration cleanupDelay, int cleanupAttempts, + LocalDateTime modifiedTimestamp, Duration cleanupDelay, int cleanupAttempts, String lifecycleType, String clientId) { this.id = id; this.path = path; @@ -97,227 +108,25 @@ private HousekeepingMetadata(Long id, String path, String databaseName, String t this.housekeepingStatus = housekeepingStatus; this.creationTimestamp = creationTimestamp; this.modifiedTimestamp = modifiedTimestamp; - this.cleanupTimestamp = cleanupTimestamp; this.cleanupDelay = cleanupDelay; + this.cleanupTimestamp = configureCleanupTimestamp(); this.cleanupAttempts = cleanupAttempts; this.lifecycleType = lifecycleType; this.clientId = clientId; } - @Override - public String getLifecycleType() { - return lifecycleType; - } - - public void setLifecycleType(String lifecycleType) { - this.lifecycleType = lifecycleType; - } - - @Override - public Long getId() { - return id; - } - - @Override - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - - @Override - public String getDatabaseName() { - return databaseName; - } - - public void setDatabaseName(String databaseName) { - this.databaseName = databaseName; - } - - @Override - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - /** - * @return The full name of the partition in hive e.g. event_date=2020-01-01/event_hour=0 - */ - public String getPartitionName() { - return partitionName; - } - - public void setPartitionName(String partitionName) { - this.partitionName = partitionName; - } - - @Override - public HousekeepingStatus getHousekeepingStatus() { - return housekeepingStatus; - } - - public void setHousekeepingStatus(HousekeepingStatus housekeepingStatus) { - this.housekeepingStatus = housekeepingStatus; - } - - @Override - public LocalDateTime getCreationTimestamp() { - return creationTimestamp; - } - - @Override - public LocalDateTime getModifiedTimestamp() { - return modifiedTimestamp; - } - - @Override - public LocalDateTime getCleanupTimestamp() { - return cleanupTimestamp; - } - - public void setCleanupTimestamp(LocalDateTime cleanupTimestamp) { - this.cleanupTimestamp = cleanupTimestamp; - } - - @Override - public int getCleanupAttempts() { - return cleanupAttempts; - } - - public void setCleanupAttempts(int cleanupAttempts) { - this.cleanupAttempts = cleanupAttempts; - } - - @Override - public String getClientId() { - return clientId; - } - - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @Override - public Duration getCleanupDelay() { - return cleanupDelay; - } - - public void setCleanupDelay(Duration cleanupDelay) { - this.cleanupDelay = cleanupDelay; - cleanupTimestamp = creationTimestamp.plus(cleanupDelay); + private LocalDateTime configureCleanupTimestamp() { + if (creationTimestamp == null) { + throw new BeekeeperException("Table requires a creation timestamp"); + } + if (cleanupDelay == null) { + throw new BeekeeperException("Table requires a cleanup delay"); + } + return creationTimestamp.plus(cleanupDelay); } @Override public MetricTag getMetricTag() { return new MetricTag("table", String.join(".", databaseName, tableName)); } - - @Override - public String toString() { - return format( - "%s(path=%s, databaseName=%s, tableName=%s, partitionName=%s, housekeepingStatus=%s, creationTimestamp=%s, modifiedTimestamp=%s, cleanupTimestamp=%s, cleanupDelay=%s, cleanupAttempts=%s, clientId=%s, lifecycleType=%s)", - HousekeepingMetadata.class.getSimpleName(), path, databaseName, tableName, partitionName, housekeepingStatus, - creationTimestamp, modifiedTimestamp, cleanupTimestamp, cleanupDelay, cleanupAttempts, clientId, lifecycleType); - } - - public static final class Builder { - - private Long id; - private String path; - private String databaseName; - private String tableName; - private String partitionName; - private HousekeepingStatus housekeepingStatus; - private LocalDateTime creationTimestamp; - private LocalDateTime modifiedTimestamp; - private Duration cleanupDelay; - private int cleanupAttempts; - private String lifecycleType; - private String clientId; - - public Builder() { } - - public HousekeepingMetadata.Builder id(Long id) { - this.id = id; - return this; - } - - public HousekeepingMetadata.Builder path(String path) { - this.path = path; - return this; - } - - public HousekeepingMetadata.Builder databaseName(String databaseName) { - this.databaseName = databaseName; - return this; - } - - public HousekeepingMetadata.Builder tableName(String tableName) { - this.tableName = tableName; - return this; - } - - public HousekeepingMetadata.Builder partitionName(String partitionName) { - this.partitionName = partitionName; - return this; - } - - public HousekeepingMetadata.Builder housekeepingStatus(HousekeepingStatus housekeepingStatus) { - this.housekeepingStatus = housekeepingStatus; - return this; - } - - public HousekeepingMetadata.Builder creationTimestamp(LocalDateTime creationTimestamp) { - this.creationTimestamp = creationTimestamp; - return this; - } - - public HousekeepingMetadata.Builder modifiedTimestamp(LocalDateTime modifiedTimestamp) { - this.modifiedTimestamp = modifiedTimestamp; - return this; - } - - public HousekeepingMetadata.Builder cleanupDelay(Duration cleanupDelay) { - this.cleanupDelay = cleanupDelay; - return this; - } - - public HousekeepingMetadata.Builder cleanupAttempts(int cleanupAttempts) { - this.cleanupAttempts = cleanupAttempts; - return this; - } - - public HousekeepingMetadata.Builder lifecycleType(String lifecycleType) { - this.lifecycleType = lifecycleType; - return this; - } - - public HousekeepingMetadata.Builder clientId(String clientId) { - this.clientId = clientId; - return this; - } - - public HousekeepingMetadata build() { - LocalDateTime cleanupTimestamp = configureCleanupTimestamp(); - - return new HousekeepingMetadata(id, path, databaseName, tableName, partitionName, housekeepingStatus, - creationTimestamp, modifiedTimestamp, cleanupTimestamp, cleanupDelay, cleanupAttempts, lifecycleType, - clientId); - } - - private LocalDateTime configureCleanupTimestamp() { - if (creationTimestamp == null) { - throw new BeekeeperException("Table requires a creation timestamp"); - } - if (cleanupDelay == null) { - throw new BeekeeperException("Table requires a cleanup delay"); - } - return creationTimestamp.plus(cleanupDelay); - } - } } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java index fd9a8e97..236721d0 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,9 +32,15 @@ import org.hibernate.annotations.UpdateTimestamp; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.monitoring.MetricTag; +@Data +@NoArgsConstructor @Entity @Table(name = "housekeeping_path") public class HousekeepingPath implements HousekeepingEntity { @@ -78,14 +84,12 @@ public class HousekeepingPath implements HousekeepingEntity { @Column(name = "lifecycle_type", nullable = false) private String lifecycleType; - public HousekeepingPath() { - - } - - private HousekeepingPath(Long id, String path, String databaseName, String tableName, + @Builder + public HousekeepingPath(Long id, String path, String databaseName, String tableName, HousekeepingStatus housekeepingStatus, LocalDateTime creationTimestamp, LocalDateTime modifiedTimestamp, LocalDateTime cleanupTimestamp, Duration cleanupDelay, int cleanupAttempts, String lifecycleType, String clientId) { + this.id = id; this.path = path; this.databaseName = databaseName; @@ -93,101 +97,12 @@ private HousekeepingPath(Long id, String path, String databaseName, String table this.housekeepingStatus = housekeepingStatus; this.creationTimestamp = creationTimestamp; this.modifiedTimestamp = modifiedTimestamp; - this.cleanupTimestamp = cleanupTimestamp; this.cleanupDelay = cleanupDelay; + this.cleanupTimestamp = configureCleanupTimestamp(); this.cleanupAttempts = cleanupAttempts; this.lifecycleType = lifecycleType; this.clientId = clientId; - } - - @Override - public String getLifecycleType() { - return lifecycleType; - } - - public void setLifecycleType(String lifecycleType) { - this.lifecycleType = lifecycleType; - } - - @Override - public Long getId() { - return id; - } - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - - @Override - public String getDatabaseName() { - return databaseName; - } - - public void setDatabaseName(String databaseName) { - this.databaseName = databaseName; - } - - @Override - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public HousekeepingStatus getHousekeepingStatus() { - return housekeepingStatus; - } - - public void setHousekeepingStatus(HousekeepingStatus housekeepingStatus) { - this.housekeepingStatus = housekeepingStatus; - } - - @Override - public LocalDateTime getCreationTimestamp() { - return creationTimestamp; - } - - @Override - public LocalDateTime getModifiedTimestamp() { - return modifiedTimestamp; - } - - @Override - public LocalDateTime getCleanupTimestamp() { - return cleanupTimestamp; - } - - public void setCleanupTimestamp(LocalDateTime cleanupTimestamp) { - this.cleanupTimestamp = cleanupTimestamp; - } - - @Override - public int getCleanupAttempts() { - return cleanupAttempts; - } - - public void setCleanupAttempts(int cleanupAttempts) { - this.cleanupAttempts = cleanupAttempts; - } - - @Override - public String getClientId() { - return clientId; - } - - public void setClientId(String clientId) { - this.clientId = clientId; - } - @Override - public Duration getCleanupDelay() { - return cleanupDelay; } public void setCleanupDelay(Duration cleanupDelay) { @@ -208,81 +123,6 @@ public String toString() { modifiedTimestamp, cleanupTimestamp, cleanupDelay, cleanupAttempts, clientId, lifecycleType); } - public static final class Builder { - - private Long id; - private String path; - private String databaseName; - private String tableName; - private HousekeepingStatus housekeepingStatus; - private LocalDateTime creationTimestamp; - private LocalDateTime modifiedTimestamp; - private LocalDateTime cleanupTimestamp; - private Duration cleanupDelay; - private int cleanupAttempts; - private String clientId; - private String lifecycleType; - - public Builder() { } - - public Builder id(Long id) { - this.id = id; - return this; - } - - public Builder path(String path) { - this.path = path; - return this; - } - - public Builder housekeepingStatus(HousekeepingStatus housekeepingStatus) { - this.housekeepingStatus = housekeepingStatus; - return this; - } - - public Builder databaseName(String databaseName) { - this.databaseName = databaseName; - return this; - } - - public Builder tableName(String tableName) { - this.tableName = tableName; - return this; - } - - public Builder creationTimestamp(LocalDateTime creationTimestamp) { - this.creationTimestamp = creationTimestamp; - return this; - } - - public Builder modifiedTimestamp(LocalDateTime modifiedTimestamp) { - this.modifiedTimestamp = modifiedTimestamp; - return this; - } - - public Builder cleanupDelay(Duration cleanupDelay) { - this.cleanupDelay = cleanupDelay; - return this; - } - - public Builder cleanupAttempts(int cleanupAttempts) { - this.cleanupAttempts = cleanupAttempts; - return this; - } - - public Builder clientId(String clientId) { - this.clientId = clientId; - return this; - } - - public HousekeepingPath build() { - cleanupTimestamp = configureCleanupTimestamp(); - - return new HousekeepingPath(id, path, databaseName, tableName, housekeepingStatus, - creationTimestamp, modifiedTimestamp, cleanupTimestamp, cleanupDelay, cleanupAttempts, lifecycleType, - clientId); - } - private LocalDateTime configureCleanupTimestamp() { if (creationTimestamp == null) { throw new BeekeeperException("Path requires a creation timestamp"); @@ -293,9 +133,5 @@ private LocalDateTime configureCleanupTimestamp() { return creationTimestamp.plus(cleanupDelay); } - public Builder lifecycleType(String lifecycleType) { - this.lifecycleType = lifecycleType; - return this; - } - } + } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/LifecycleConfiguration.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/LifecycleConfiguration.java new file mode 100644 index 00000000..706b43b6 --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/LifecycleConfiguration.java @@ -0,0 +1,11 @@ +package com.expediagroup.beekeeper.core.model; + +import java.util.List; + +public class LifecycleConfiguration { + + boolean removeUnreferencedData; + String UnreferencedDataRetentionPeriod; + List hiveEventWhitelist; + +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index cf516d00..057309eb 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,13 +20,15 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -public interface HousekeepingMetadataRepository extends JpaRepository { +public interface HousekeepingMetadataRepository extends PagingAndSortingRepository, + JpaSpecificationExecutor { @Query(value = "from HousekeepingMetadata t where t.cleanupTimestamp <= :instant " + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java index 65b0e983..572b8127 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java @@ -19,19 +19,25 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import com.expediagroup.beekeeper.core.model.HousekeepingPath; @Repository -public interface HousekeepingPathRepository extends JpaRepository { +public interface HousekeepingPathRepository extends PagingAndSortingRepository + , JpaSpecificationExecutor { @Query(value = "from HousekeepingPath p where p.cleanupTimestamp <= :instant " + "and (p.housekeepingStatus = 'SCHEDULED' or p.housekeepingStatus = 'FAILED') " + "and p.modifiedTimestamp <= :instant order by p.modifiedTimestamp") Page findRecordsForCleanupByModifiedTimestamp(@Param("instant") LocalDateTime instant, Pageable pageable); + + Page findAllByDatabaseNameAndTableName(String databaseName, String tableName, + Pageable pageable); } diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/monitoring/TimedTaggableAspectTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/monitoring/TimedTaggableAspectTest.java index 5f0bb643..d893ed6a 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/monitoring/TimedTaggableAspectTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/monitoring/TimedTaggableAspectTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,7 +61,7 @@ public class TimedTaggableAspectTest { @BeforeEach public void init() { - housekeepingPath = new HousekeepingPath.Builder() + housekeepingPath = HousekeepingPath.builder() .databaseName(DATABASE) .tableName(TABLE) .creationTimestamp(LocalDateTime.now()) diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 255bc3b2..f3cbfcb3 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import com.google.common.collect.Lists; + import com.expediagroup.beekeeper.core.TestApplication; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @@ -83,7 +85,7 @@ public void typicalSave() { housekeepingMetadataRepository.save(table); - List tables = housekeepingMetadataRepository.findAll(); + List tables = Lists.newArrayList(housekeepingMetadataRepository.findAll()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata savedTable = tables.get(0); assertThat(savedTable.getPath()).isEqualTo(PATH); @@ -108,7 +110,7 @@ public void typicalUpdate() { savedTable.setCleanupAttempts(savedTable.getCleanupAttempts() + 1); housekeepingMetadataRepository.save(savedTable); - List tables = housekeepingMetadataRepository.findAll(); + List tables = Lists.newArrayList(housekeepingMetadataRepository.findAll()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata updatedTable = tables.get(0); assertThat(updatedTable.getHousekeepingStatus()).isEqualTo(DELETED); @@ -141,7 +143,7 @@ public void notNullableTableNameField() { public void timezone() { HousekeepingMetadata path = createPartitionedEntityHousekeepingTable(); housekeepingMetadataRepository.save(path); - List tables = housekeepingMetadataRepository.findAll(); + List tables = Lists.newArrayList(housekeepingMetadataRepository.findAll()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata savedTable = tables.get(0); @@ -370,7 +372,7 @@ private HousekeepingMetadata createEntityHouseKeepingTable( String databaseName, String tableName, String partitionName) { - return new HousekeepingMetadata.Builder() + return HousekeepingMetadata.builder() .path(PATH) .databaseName(databaseName) .tableName(tableName) diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java index 7a798cdb..e4c474e5 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import com.google.common.collect.Lists; + import com.expediagroup.beekeeper.core.TestApplication; import com.expediagroup.beekeeper.core.model.HousekeepingPath; @@ -77,7 +79,7 @@ public void typicalSave() { housekeepingPathRepository.save(path); - List paths = housekeepingPathRepository.findAll(); + List paths = Lists.newArrayList(housekeepingPathRepository.findAll()); assertThat(paths.size()).isEqualTo(1); HousekeepingPath savedPath = paths.get(0); assertThat(savedPath.getPath()).isEqualTo("path"); @@ -101,7 +103,7 @@ public void typicalUpdate() { savedPath.setCleanupAttempts(savedPath.getCleanupAttempts() + 1); housekeepingPathRepository.save(savedPath); - List paths = housekeepingPathRepository.findAll(); + List paths = Lists.newArrayList(housekeepingPathRepository.findAll()); assertThat(paths.size()).isEqualTo(1); HousekeepingPath updatedPath = paths.get(0); assertThat(updatedPath.getHousekeepingStatus()).isEqualTo(DELETED); @@ -120,7 +122,7 @@ public void notNullableField() { void timezone() { HousekeepingPath path = createEntityHousekeepingPath(); housekeepingPathRepository.save(path); - List paths = housekeepingPathRepository.findAll(); + List paths = Lists.newArrayList(housekeepingPathRepository.findAll()); assertThat(paths.size()).isEqualTo(1); HousekeepingPath savedPath = paths.get(0); @@ -204,7 +206,7 @@ void findRecordsForCleanupByModifiedTimestampRespectsOrder() { } private HousekeepingPath createEntityHousekeepingPath() { - return new HousekeepingPath.Builder() + return HousekeepingPath.builder() .path("path") .databaseName("database") .tableName("table") diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 47e03f33..9ac8555d 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -60,6 +60,11 @@ ${project.version} + + org.projectlombok + lombok + + org.apache.hadoop hadoop-aws @@ -75,6 +80,19 @@ + + + com.fasterxml.jackson.module + jackson-module-parameter-names + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + org.awaitility @@ -116,7 +134,23 @@ log4j-slf4j-impl org.apache.logging.log4j + + org.eclipse.jetty.aggregate + jetty-all + + + com.expediagroup + beekeeper-api + 3.1.1-SNAPSHOT + test + + + joda-time + joda-time + 2.10.9 + test + diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 39c0b5d3..54895778 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; + import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; @@ -61,6 +63,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.integration.utils.MySqlTestUtils; @@ -88,17 +91,24 @@ public abstract class BeekeeperIntegrationTestBase { // FIELDS TO INSERT INTO BEEKEEPER TABLES private Long id = 1L; - private static final String HOUSEKEEPING_PATH_FIELDS = String.join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, - TABLE_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, - CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); - private static final String HOUSEKEEPING_METADATA_FIELDS = String.join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, - TABLE_NAME_FIELD, PARTITION_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, - MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, - LIFECYCLE_TYPE_FIELD); + private static final String HOUSEKEEPING_PATH_FIELDS = String + .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, + CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, + CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); + private static final String HOUSEKEEPING_METADATA_FIELDS = String + .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, PARTITION_NAME_FIELD, + HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, + CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); private static final String LIFE_CYCLE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; - private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'" - + " AND " + MODIFIED_TIMESTAMP_FIELD + " > " + CREATION_TIMESTAMP_FIELD - + " ORDER BY " + PATH_FIELD; + private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE " + + LIFECYCLE_TYPE_FIELD + + " = '%s'" + + " AND " + + MODIFIED_TIMESTAMP_FIELD + + " > " + + CREATION_TIMESTAMP_FIELD + + " ORDER BY " + + PATH_FIELD; // MySQL DB CONTAINER AND UTILS @Container @@ -145,13 +155,13 @@ public void dropMySQLTables() throws SQLException { protected void insertUnreferencedPath(String path) throws SQLException { HousekeepingPath housekeepingPath = createHousekeepingPath(path, UNREFERENCED); housekeepingPath.setCleanupTimestamp(housekeepingPath.getCleanupTimestamp().minus(Duration.ofDays(1))); - String values = Stream.of(housekeepingPath.getId().toString(), housekeepingPath.getPath(), - housekeepingPath.getDatabaseName(), - housekeepingPath.getTableName(), housekeepingPath.getHousekeepingStatus().toString(), - housekeepingPath.getCreationTimestamp().toString(), housekeepingPath.getModifiedTimestamp().toString(), - housekeepingPath.getCleanupTimestamp().toString(), housekeepingPath.getCleanupDelay().toString(), - String.valueOf(housekeepingPath.getCleanupAttempts()), housekeepingPath.getClientId(), - housekeepingPath.getLifecycleType()) + String values = Stream + .of(housekeepingPath.getId().toString(), housekeepingPath.getPath(), housekeepingPath.getDatabaseName(), + housekeepingPath.getTableName(), housekeepingPath.getHousekeepingStatus().toString(), + housekeepingPath.getCreationTimestamp().toString(), housekeepingPath.getModifiedTimestamp().toString(), + housekeepingPath.getCleanupTimestamp().toString(), housekeepingPath.getCleanupDelay().toString(), + String.valueOf(housekeepingPath.getCleanupAttempts()), housekeepingPath.getClientId(), + housekeepingPath.getLifecycleType()) .map(s -> s == null ? null : "\"" + s + "\"") .collect(Collectors.joining(", ")); @@ -164,7 +174,7 @@ protected void insertExpiredMetadata(String path, String partitionName) throws S } protected void insertExpiredMetadata(String tableName, String path, String partitionName, String cleanupDelay) - throws SQLException { + throws SQLException { HousekeepingMetadata metadata = createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); String values = Stream .of(metadata.getId().toString(), metadata.getPath(), metadata.getDatabaseName(), metadata.getTableName(), @@ -180,6 +190,21 @@ protected void insertExpiredMetadata(String tableName, String path, String parti values); } + protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLException { + String values = Stream + .of(metadata.getId().toString(), metadata.getPath(), metadata.getDatabaseName(), metadata.getTableName(), + metadata.getPartitionName(), metadata.getHousekeepingStatus().toString(), + metadata.getCreationTimestamp().toString(), metadata.getModifiedTimestamp().toString(), + metadata.getCleanupTimestamp().toString(), metadata.getCleanupDelay().toString(), + String.valueOf(metadata.getCleanupAttempts()), metadata.getClientId(), metadata.getLifecycleType()) + .map(s -> s == null ? null : "\"" + s + "\"") + .collect(Collectors.joining(", ")); + + mySQLTestUtils + .insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, HOUSEKEEPING_METADATA_FIELDS, + values); + } + protected int getUnreferencedPathsRowCount() throws SQLException { return mySQLTestUtils .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, @@ -225,7 +250,7 @@ protected List getExpiredMetadata() throws SQLException { } private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType lifecycleEventType) { - return new HousekeepingPath.Builder() + return HousekeepingPath.builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -240,13 +265,13 @@ private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType .build(); } - private HousekeepingMetadata createHousekeepingMetadata( + public HousekeepingMetadata createHousekeepingMetadata( String tableName, String path, String partitionName, LifecycleEventType lifecycleEventType, String cleanupDelay) { - return new HousekeepingMetadata.Builder() + return HousekeepingMetadata.builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -261,5 +286,4 @@ private HousekeepingMetadata createHousekeepingMetadata( .clientId(CLIENT_ID_FIELD) .build(); } - } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 1cb03a25..6329e4fa 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java new file mode 100644 index 00000000..ae242b0a --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -0,0 +1,347 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.integration.api; + +import static java.lang.String.format; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.http.HttpStatus.OK; + +import java.io.IOException; +import java.net.http.HttpResponse; +import java.sql.SQLException; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.List; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.util.SocketUtils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; + +import com.expediagroup.beekeeper.api.BeekeeperApiApplication; +import com.expediagroup.beekeeper.api.response.HousekeepingMetadataResponse; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; +import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.integration.BeekeeperIntegrationTestBase; +import com.expediagroup.beekeeper.integration.utils.BeekeeperApiTestClient; +import com.expediagroup.beekeeper.integration.utils.RestResponsePage; + + +public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { + + @Bean + @Primary + public ObjectMapper geObjMapper() { + return new ObjectMapper() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } + + private static final Logger log = LoggerFactory.getLogger(BeekeeperApiIntegrationTest.class); + + // APP CONTEXT AND TEST CLIENT + protected static ConfigurableApplicationContext context; + protected BeekeeperApiTestClient testClient; + + protected final ObjectMapper mapper = geObjMapper(); + + @BeforeEach + public void beforeEach() { + + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + int port = SocketUtils.findAvailableTcpPort(); + String[] args = new String[] { + "--server.port=" + port}; + final String url = format("http://localhost:%d", port); + log.info("Starting to run Beekeeper API on: {} and args: {}", url, args); + context = SpringApplication.run(BeekeeperApiApplication.class, args); + testClient = new BeekeeperApiTestClient(url); + } + + @AfterEach + public final void afterEach() { + log.info("Stopping Beekeeper API"); + if (context != null) { + context.close(); + context = null; + } + } + + @Test + public void testGetTablesWhenNoTables() throws InterruptedException, IOException { + HttpResponse response = testClient.getTables(); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + assertThat(responsePage.getContent()).isEqualTo(List.of()); + } + + @Test + public void testGetTablesWhenTablesValid() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName1","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + + HttpResponse response = testClient.getTables(); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(testMetadata1.equals(result.get(0))); + assertTrue(testMetadata2.equals(result.get(1))); + assertThat(result.size()).isEqualTo(2); + + } + + @Test + public void testGetTablesWhenTableNameFilter() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName1","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata metadataWithTableName = createHousekeepingMetadata("bobs_table","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(metadataWithTableName); + + HttpResponse response = testClient.getTablesWithTableNameFilter("bobs_table"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(metadataWithTableName.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void testGetTablesWhenDatabaseNameFilter() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName1","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata metadataWithDatabaseName = createHousekeepingMetadata("bobs_table","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + metadataWithDatabaseName.setDatabaseName("bobs_database"); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(metadataWithDatabaseName); + + HttpResponse response = testClient.getTablesWithDatabaseNameFilter("bobs_database"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertEquals(metadataWithDatabaseName,result.get(0)); + assertTrue(metadataWithDatabaseName.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void testGetTablesWhenHousekeepingStatusFilter() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName1","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata metadataWithHousekeepingStatus = createHousekeepingMetadata("bobs_table","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + metadataWithHousekeepingStatus.setHousekeepingStatus(HousekeepingStatus.FAILED); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(metadataWithHousekeepingStatus); + + HttpResponse response = testClient.getTablesWithHousekeepingStatusFilter("FAILED"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(metadataWithHousekeepingStatus.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void testGetTablesWhenLifecycleEventTypeFilter() throws SQLException, InterruptedException, IOException { + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName3","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata unreferencedMetadata = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(unreferencedMetadata); + + + HttpResponse response = testClient.getTablesWithLifecycleEventTypeFilter("UNREFERENCED"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(unreferencedMetadata.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void testGetTablesWhenDeletedBeforeFilter() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName3","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata metadataWithCleanupTimestamp = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + metadataWithCleanupTimestamp.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(metadataWithCleanupTimestamp); + + HttpResponse response = testClient.getTablesWithDeletedBeforeFilter("2000-05-05T10:41:20"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(metadataWithCleanupTimestamp.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void testGetTablesWhenDeletedAfterFilter() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName3","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata metadataWithCleanupTimestamp = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + testMetadata1.setCleanupTimestamp(LocalDateTime.parse("2020-05-05T10:41:20")); + testMetadata2.setCleanupTimestamp(LocalDateTime.parse("2020-05-05T10:41:20")); + metadataWithCleanupTimestamp.setCleanupTimestamp(LocalDateTime.parse("2020-06-05T10:41:20")); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(metadataWithCleanupTimestamp); + + HttpResponse response = testClient.getTablesWithDeletedAfterFilter("2020-05-06T10:41:20"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(metadataWithCleanupTimestamp.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void testGetTablesWhenRegisteredBeforeFilter() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName3","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata metadataWithCreationTimestamp = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + testMetadata1.setCreationTimestamp(LocalDateTime.parse("2020-05-05T10:41:20")); + testMetadata2.setCreationTimestamp(LocalDateTime.parse("2020-05-05T10:41:20")); + metadataWithCreationTimestamp.setCreationTimestamp(LocalDateTime.parse("2020-04-05T10:41:20")); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(metadataWithCreationTimestamp); + + HttpResponse response = testClient.getTablesWithRegisteredBeforeFilter("2020-05-04T10:41:20"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(metadataWithCreationTimestamp.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void testGetTablesWhenRegisteredAfterFilter() throws SQLException, InterruptedException, IOException { + + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName3","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata metadataWithCreationTimestamp = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + testMetadata1.setCreationTimestamp(LocalDateTime.parse("2020-05-05T10:41:20")); + testMetadata2.setCreationTimestamp(LocalDateTime.parse("2020-05-05T10:41:20")); + metadataWithCreationTimestamp.setCreationTimestamp(LocalDateTime.parse("2020-06-05T10:41:20")); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(metadataWithCreationTimestamp); + + HttpResponse response = testClient.getTablesWithRegisteredAfterFilter("2020-05-06T10:41:20"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + assertTrue(metadataWithCreationTimestamp.equals(result.get(0))); + assertThat(result.size()).isEqualTo(1); + } + + @Test + public void TESTING() throws SQLException, InterruptedException, IOException { + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("myTableName2","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("myTableName3","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata testMetadata3 = createHousekeepingMetadata("myTableName3","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + HousekeepingMetadata unreferencedMetadata = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=A",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + HousekeepingMetadata unreferencedMetadata2 = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=B",LifecycleEventType.EXPIRED,Duration.parse("P3D").toString()); + HousekeepingMetadata unreferencedMetadata3 = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=C",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + HousekeepingMetadata unreferencedMetadata4 = createHousekeepingMetadata("myTableName","s3://some/path/","event_date=2020-01-01/event_hour=0/event_type=C",LifecycleEventType.UNREFERENCED,Duration.parse("P3D").toString()); + testMetadata1.setCleanupDelay(Duration.parse("P4D")); + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(testMetadata3); + insertExpiredMetadata(unreferencedMetadata); + insertExpiredMetadata(unreferencedMetadata2); + insertExpiredMetadata(unreferencedMetadata3); + insertExpiredMetadata(unreferencedMetadata4); + + Thread.sleep(10000000L); + + HttpResponse response = testClient.getTables(); + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + } + +} diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java new file mode 100644 index 00000000..c8234ad8 --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.integration.utils; + +import static java.net.http.HttpRequest.newBuilder; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; + + +public class BeekeeperApiTestClient { + + private static final String PREFIX = "/api/v1"; + private static final String TABLES_PATH = PREFIX + "/tables"; + + private final String tablesUrl; + private final HttpClient httpClient; + + public BeekeeperApiTestClient(String baseUrl) { + this.tablesUrl = baseUrl + TABLES_PATH; + this.httpClient = HttpClient.newHttpClient(); + } + + public HttpResponse getTables() throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithTableNameFilter(String tableName) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?table_name="+tableName)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithDatabaseNameFilter(String databaseName) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?database_name="+databaseName)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithHousekeepingStatusFilter(String housekeepingStatus) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?housekeeping_status="+housekeepingStatus)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithLifecycleEventTypeFilter(String lifecycleEventType) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?lifecycle_type="+lifecycleEventType)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithDeletedBeforeFilter(String timestamp) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?deleted_before="+timestamp)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithDeletedAfterFilter(String timestamp) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?deleted_after="+timestamp)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithRegisteredBeforeFilter(String timestamp) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?registered_before="+timestamp)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + + public HttpResponse getTablesWithRegisteredAfterFilter(String timestamp) throws IOException, InterruptedException { + HttpRequest request = newBuilder().uri(URI.create(tablesUrl+"?registered_after="+timestamp)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + +} diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index 614b884a..564c8b5f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; public class MySqlTestUtils { @@ -50,11 +51,13 @@ public int getTableRowCount(String database, String table, String additionalFilt return getTableRowCount(format(SELECT_TABLE, database, table, additionalFilters)); } - private int getTableRowCount(String statement) throws SQLException { - ResultSet resultSet = getTableRows(statement); + private int getTableRowCount(String statementString) throws SQLException { + Statement statement; + statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, + ResultSet.CONCUR_READ_ONLY); + ResultSet resultSet = statement.executeQuery(statementString); resultSet.last(); - int rowsInTable = resultSet.getRow(); - return rowsInTable; + return resultSet.getRow(); } public ResultSet getTableRows(String database, String table, String additionalFilters) throws SQLException { diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java new file mode 100644 index 00000000..71ef4239 --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.integration.utils; +import java.util.List; + +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; + + +public class RestResponsePage extends PageImpl { + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public RestResponsePage( + @JsonProperty("content") List content, + @JsonProperty("number") int number, + @JsonProperty("size") int size, + @JsonProperty("totalElements") Long totalElements, + @JsonProperty("pageable") JsonNode pageable, + @JsonProperty("last") boolean last, + @JsonProperty("totalPages") int totalPages, + @JsonProperty("sort") JsonNode sort, + @JsonProperty("first") boolean first, + @JsonProperty("numberOfElements") int numberOfElements, + @JsonProperty("empty") boolean empty) { + super(content, PageRequest.of(number, size), totalElements); + } +} diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java index a0416d04..260bfbb7 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ public class ResultSetToHousekeepingEntityMapper { public static HousekeepingPath mapToHousekeepingPath(ResultSet resultSet) throws SQLException { - return new HousekeepingPath.Builder() + return HousekeepingPath.builder() .id(resultSet.getLong(ID_FIELD)) .path(resultSet.getString(PATH_FIELD)) .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) @@ -56,19 +56,20 @@ public static HousekeepingPath mapToHousekeepingPath(ResultSet resultSet) throws } public static HousekeepingMetadata mapToHousekeepingMetadata(ResultSet resultSet) throws SQLException { - return new HousekeepingMetadata.Builder() - .id(resultSet.getLong(ID_FIELD)) - .path(resultSet.getString(PATH_FIELD)) - .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) - .tableName(resultSet.getString(TABLE_NAME_FIELD)) - .partitionName(resultSet.getString(PARTITION_NAME_FIELD)) - .housekeepingStatus(HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) - .creationTimestamp(Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) - .modifiedTimestamp(Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) - .cleanupDelay(Duration.parse(resultSet.getString(CLEANUP_DELAY_FIELD))) - .cleanupAttempts(resultSet.getInt(CLEANUP_ATTEMPTS_FIELD)) - .clientId(resultSet.getString(CLIENT_ID_FIELD)) - .lifecycleType(resultSet.getString(LIFECYCLE_TYPE_FIELD)) - .build(); + + return HousekeepingMetadata.builder() + .id(resultSet.getLong(ID_FIELD)) + .path(resultSet.getString(PATH_FIELD)) + .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) + .tableName(resultSet.getString(TABLE_NAME_FIELD)) + .partitionName(resultSet.getString(PARTITION_NAME_FIELD)) + .housekeepingStatus(HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) + .creationTimestamp(Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) + .modifiedTimestamp(Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) + .cleanupDelay(Duration.parse(resultSet.getString(CLEANUP_DELAY_FIELD))) + .cleanupAttempts(resultSet.getInt(CLEANUP_ATTEMPTS_FIELD)) + .clientId(resultSet.getString(CLIENT_ID_FIELD)) + .lifecycleType(resultSet.getString(LIFECYCLE_TYPE_FIELD)) + .build(); } } diff --git a/beekeeper-integration-tests/src/test/resources/logback-test.xml b/beekeeper-integration-tests/src/test/resources/logback-test.xml index 42a3e8ea..0b0174af 100644 --- a/beekeeper-integration-tests/src/test/resources/logback-test.xml +++ b/beekeeper-integration-tests/src/test/resources/logback-test.xml @@ -1,6 +1,6 @@