From 4b555debde2c9451392c768fcd7c166cded160ab Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 25 Mar 2021 15:29:26 +0000 Subject: [PATCH 001/115] changes --- beekeeper-api/pom.xml | 23 ++++++++ .../beekeeper/api/BeekeeperController.java | 19 ++++++- .../beekeeper/api/ReturnTablesTest.java | 53 +++++++++++++++++++ .../beekeeper/api/TestApplication.java | 25 +++++++++ .../HousekeepingMetadataRepository.java | 16 ++++++ .../HousekeepingMetadataRepositoryTest.java | 6 ++- 6 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java create mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 5a81f3ca..01397754 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -20,6 +20,29 @@ org.springframework.boot spring-boot-actuator + + org.springframework + spring-test + test + + + com.expediagroup + beekeeper-path-cleanup + 3.1.1-SNAPSHOT + test + + + com.expediagroup + beekeeper-vacuum-tool + 3.1.1-SNAPSHOT + test + + + com.expediagroup + beekeeper-metadata-cleanup + 3.1.1-SNAPSHOT + test + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index e0bd2172..78defa0d 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -15,9 +15,26 @@ */ package com.expediagroup.beekeeper.api; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; @RestController public class BeekeeperController { - + + @RequestMapping("/tables") + public String greeting(){ + return "hello world!"; + } + +// @RequestMapping(value = "/returntables", method = RequestMethod.GET) +// public Table tableInfo( +// @PathVariable String databaseName, +// @PathVariable String tableName, +// @RequestParam(value = "partitionFilter", required = false) String filter) { +// MetaKittyTable table = metaKitty.tableInfo(databaseName, tableName, filter); +// return table; +// } + } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java new file mode 100644 index 00000000..721dd49c --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java @@ -0,0 +1,53 @@ +package com.expediagroup.beekeeper.api; + +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.query.Param; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import com.expediagroup.beekeeper.cleanup.aws.S3PathCleaner; +import com.expediagroup.beekeeper.cleanup.hive.HiveMetadataCleaner; +import com.expediagroup.beekeeper.cleanup.metadata.CleanerClient; +import com.expediagroup.beekeeper.cleanup.path.PathCleaner; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.path.cleanup.handler.UnreferencedPathHandler; + +@ExtendWith(MockitoExtension.class) +public class ReturnTablesTest { + + private HousekeepingMetadataRepository housekeepingMetadataRepository; + + @BeforeEach + public void setupDb() { + + } + + @Test + public void test(){ + Optional housekeepingMetadataOptional = housekeepingMetadataRepository.findRecord("test","user_diaspora_v2"); + } + + +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java new file mode 100644 index 00000000..f2acbfd7 --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java @@ -0,0 +1,25 @@ +package com.expediagroup.beekeeper.api; + +import java.util.TimeZone; + +import javax.annotation.PostConstruct; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +@SpringBootApplication +@EnableConfigurationProperties +@ComponentScan("com.expediagroup.beekeeper.core") +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +public class TestApplication { + + @PostConstruct + void started() { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + } + +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index cf516d00..1592c30d 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -55,6 +55,22 @@ Optional findRecordForCleanupByDbTableAndPartitionName( @Param("databaseName") String databaseName, @Param("tableName") String tableName, @Param("partitionName") String partitionName); + /** + * Returns the record that matches the inputs given, if there is one. + * + * + * @param databaseName + * @param tableName + * @return + */ + @Query(value = "from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName ") + Optional findRecord( + @Param("databaseName") String databaseName, + @Param("tableName") String tableName); + + /** * Returns the maximum value for the cleanupTimestamp for a database and table name pair. * diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 255bc3b2..2302eb88 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -82,8 +82,9 @@ public void typicalSave() { HousekeepingMetadata table = createPartitionedEntityHousekeepingTable(); housekeepingMetadataRepository.save(table); - +//here List tables = housekeepingMetadataRepository.findAll(); + System.out.println("AAtables:"+tables.toString()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata savedTable = tables.get(0); assertThat(savedTable.getPath()).isEqualTo(PATH); @@ -354,7 +355,7 @@ public void dryRunCountPartitionsForUnpartitionedTable() { assertEquals(0L, result); } - private HousekeepingMetadata createUnpartitionedEntityHousekeepingTable() { + public HousekeepingMetadata createUnpartitionedEntityHousekeepingTable() { return createEntityHousekeepingTable(null); } @@ -366,6 +367,7 @@ private HousekeepingMetadata createEntityHousekeepingTable(String partitionName) return createEntityHouseKeepingTable(DATABASE_NAME, TABLE_NAME, partitionName); } + //here private HousekeepingMetadata createEntityHouseKeepingTable( String databaseName, String tableName, From c81367919db989c2039cf898745030fe4ef2466a Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 25 Mar 2021 15:58:38 +0000 Subject: [PATCH 002/115] removing useless stuff --- .../beekeeper/api/BeekeeperController.java | 9 ------- .../beekeeper/api/ReturnTablesTest.java | 5 ---- .../beekeeper/api/TestApplication.java | 25 ------------------- .../HousekeepingMetadataRepositoryTest.java | 5 ++-- 4 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 78defa0d..1d18716f 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -28,13 +28,4 @@ public String greeting(){ return "hello world!"; } -// @RequestMapping(value = "/returntables", method = RequestMethod.GET) -// public Table tableInfo( -// @PathVariable String databaseName, -// @PathVariable String tableName, -// @RequestParam(value = "partitionFilter", required = false) String filter) { -// MetaKittyTable table = metaKitty.tableInfo(databaseName, tableName, filter); -// return table; -// } - } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java index 721dd49c..20fea68b 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java @@ -39,11 +39,6 @@ public class ReturnTablesTest { private HousekeepingMetadataRepository housekeepingMetadataRepository; - @BeforeEach - public void setupDb() { - - } - @Test public void test(){ Optional housekeepingMetadataOptional = housekeepingMetadataRepository.findRecord("test","user_diaspora_v2"); diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java deleted file mode 100644 index f2acbfd7..00000000 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.expediagroup.beekeeper.api; - -import java.util.TimeZone; - -import javax.annotation.PostConstruct; - -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; - -@SpringBootApplication -@EnableConfigurationProperties -@ComponentScan("com.expediagroup.beekeeper.core") -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) -public class TestApplication { - - @PostConstruct - void started() { - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - } - -} diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 2302eb88..5afa47a3 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -82,7 +82,7 @@ public void typicalSave() { HousekeepingMetadata table = createPartitionedEntityHousekeepingTable(); housekeepingMetadataRepository.save(table); -//here + List tables = housekeepingMetadataRepository.findAll(); System.out.println("AAtables:"+tables.toString()); assertThat(tables.size()).isEqualTo(1); @@ -355,7 +355,7 @@ public void dryRunCountPartitionsForUnpartitionedTable() { assertEquals(0L, result); } - public HousekeepingMetadata createUnpartitionedEntityHousekeepingTable() { + private HousekeepingMetadata createUnpartitionedEntityHousekeepingTable() { return createEntityHousekeepingTable(null); } @@ -367,7 +367,6 @@ private HousekeepingMetadata createEntityHousekeepingTable(String partitionName) return createEntityHouseKeepingTable(DATABASE_NAME, TABLE_NAME, partitionName); } - //here private HousekeepingMetadata createEntityHouseKeepingTable( String databaseName, String tableName, From 866dc7f0ba6e0d786adf5e43bf888fab0a851437 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 25 Mar 2021 15:59:43 +0000 Subject: [PATCH 003/115] tidy up --- .../core/repository/HousekeepingMetadataRepositoryTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 5afa47a3..255bc3b2 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -84,7 +84,6 @@ public void typicalSave() { housekeepingMetadataRepository.save(table); List tables = housekeepingMetadataRepository.findAll(); - System.out.println("AAtables:"+tables.toString()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata savedTable = tables.get(0); assertThat(savedTable.getPath()).isEqualTo(PATH); From 2a3a6338c056cd8544609047a2d12459a738d827 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 25 Mar 2021 16:01:20 +0000 Subject: [PATCH 004/115] remove unused imports --- .../beekeeper/api/ReturnTablesTest.java | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java index 20fea68b..18de4a39 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java @@ -1,38 +1,13 @@ package com.expediagroup.beekeeper.api; -import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; - -import java.time.LocalDateTime; -import java.util.List; import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.repository.query.Param; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit.jupiter.SpringExtension; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - -import com.expediagroup.beekeeper.cleanup.aws.S3PathCleaner; -import com.expediagroup.beekeeper.cleanup.hive.HiveMetadataCleaner; -import com.expediagroup.beekeeper.cleanup.metadata.CleanerClient; -import com.expediagroup.beekeeper.cleanup.path.PathCleaner; + import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.model.HousekeepingPath; -import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; -import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; -import com.expediagroup.beekeeper.path.cleanup.handler.UnreferencedPathHandler; @ExtendWith(MockitoExtension.class) public class ReturnTablesTest { From 56d8697b1c4b3833f05125bcda217e09fe72640a Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 10:48:05 +0100 Subject: [PATCH 005/115] adding classes --- beekeeper-api/pom.xml | 6 +++ .../beekeeper/api/BeekeeperController.java | 12 ++++-- .../beekeeper/api/BeekeeperService.java | 11 ++++++ .../beekeeper/api/BeekeeperServiceImpl.java | 30 ++++++++++++++ .../beekeeper/api/ReturnTablesTest.java | 39 +++++++++++++++++-- .../HousekeepingMetadataRepositoryTest.java | 16 ++++++++ 6 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java create mode 100644 beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 01397754..041b37db 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -43,6 +43,12 @@ 3.1.1-SNAPSHOT test + + com.expediagroup + beekeeper-core + ${project.version} + + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 1d18716f..c3c14416 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -15,17 +15,21 @@ */ package com.expediagroup.beekeeper.api; -import org.springframework.web.bind.annotation.PathVariable; +import java.util.List; + import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + @RestController public class BeekeeperController { + private BeekeeperServiceImpl beekeeperServiceImpl; + @RequestMapping("/tables") - public String greeting(){ - return "hello world!"; + public List getAllTables() { + return beekeeperServiceImpl.returnAllTables(); } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java new file mode 100644 index 00000000..c82ea004 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java @@ -0,0 +1,11 @@ +package com.expediagroup.beekeeper.api; + +import java.util.List; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; + +public interface BeekeeperService { + + +} diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java new file mode 100644 index 00000000..afe8c1be --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java @@ -0,0 +1,30 @@ +package com.expediagroup.beekeeper.api; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; + +@Service +public class BeekeeperServiceImpl { + + private HousekeepingMetadataRepository housekeepingMetadataRepository; + + @Autowired + public BeekeeperServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { + this.housekeepingMetadataRepository = housekeepingMetadataRepository; + } + + public void saveTable(HousekeepingMetadata table){ + housekeepingMetadataRepository.save(table); + System.out.println("count:"+housekeepingMetadataRepository.count()); + } + + // Method for the GET tables/ endpoint + public List returnAllTables() { + return housekeepingMetadataRepository.findAll(); + } +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java index 18de4a39..fb0a1360 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java @@ -1,7 +1,14 @@ package com.expediagroup.beekeeper.api; -import java.util.Optional; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -13,11 +20,37 @@ public class ReturnTablesTest { private HousekeepingMetadataRepository housekeepingMetadataRepository; + private HousekeepingMetadata table1; + private final BeekeeperServiceImpl beekeeperServiceImpl; + + public ReturnTablesTest( + HousekeepingMetadataRepository housekeepingMetadataRepository, + BeekeeperServiceImpl beekeeperServiceImpl) {this.housekeepingMetadataRepository = housekeepingMetadataRepository; + this.beekeeperServiceImpl = beekeeperServiceImpl; + } + + @BeforeEach + public void createTables(){ + LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + table1 = new HousekeepingMetadata.Builder() + .path("s3://some/path/") + .databaseName("aRandomDatabase") + .tableName("aRandomTable") + .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupDelay(Duration.parse("P3D")) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); + } @Test public void test(){ - Optional housekeepingMetadataOptional = housekeepingMetadataRepository.findRecord("test","user_diaspora_v2"); + beekeeperServiceImpl.saveTable(table1); + List tables = beekeeperServiceImpl.returnAllTables(); + System.out.println("AAA tables:"+tables.size()); } - } diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 255bc3b2..e61c2dba 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -77,6 +77,22 @@ public void setupDb() { housekeepingMetadataRepository.deleteAll(); } + + @Test + public void returnAllTables(){ + HousekeepingMetadata table1 = createPartitionedEntityHousekeepingTable(); + HousekeepingMetadata table2 = createPartitionedEntityHousekeepingTable(); + HousekeepingMetadata table3 = createPartitionedEntityHousekeepingTable(); + + housekeepingMetadataRepository.save(table1); + housekeepingMetadataRepository.save(table2); + housekeepingMetadataRepository.save(table3); + + List tables = housekeepingMetadataRepository.findAll(); + assertThat(tables.size()).isEqualTo(3); + System.out.println("AA:"+tables.toString()); + } + @Test public void typicalSave() { HousekeepingMetadata table = createPartitionedEntityHousekeepingTable(); From 21c2dce5194422e9cda76b4cda6f59adeaf18b51 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 10:52:52 +0100 Subject: [PATCH 006/115] clean up --- .../beekeeper/api/BeekeeperService.java | 1 - .../beekeeper/api/BeekeeperServiceImpl.java | 10 --- .../beekeeper/api/ReturnTablesTest.java | 66 +++++++++---------- 3 files changed, 33 insertions(+), 44 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java index c82ea004..8c6ba3a6 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java @@ -7,5 +7,4 @@ public interface BeekeeperService { - } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java index afe8c1be..43d1c271 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java @@ -13,16 +13,6 @@ public class BeekeeperServiceImpl { private HousekeepingMetadataRepository housekeepingMetadataRepository; - @Autowired - public BeekeeperServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { - this.housekeepingMetadataRepository = housekeepingMetadataRepository; - } - - public void saveTable(HousekeepingMetadata table){ - housekeepingMetadataRepository.save(table); - System.out.println("count:"+housekeepingMetadataRepository.count()); - } - // Method for the GET tables/ endpoint public List returnAllTables() { return housekeepingMetadataRepository.findAll(); diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java index fb0a1360..9f697c54 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java @@ -19,38 +19,38 @@ @ExtendWith(MockitoExtension.class) public class ReturnTablesTest { - private HousekeepingMetadataRepository housekeepingMetadataRepository; - private HousekeepingMetadata table1; - private final BeekeeperServiceImpl beekeeperServiceImpl; - - public ReturnTablesTest( - HousekeepingMetadataRepository housekeepingMetadataRepository, - BeekeeperServiceImpl beekeeperServiceImpl) {this.housekeepingMetadataRepository = housekeepingMetadataRepository; - this.beekeeperServiceImpl = beekeeperServiceImpl; - } - - @BeforeEach - public void createTables(){ - LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); - table1 = new HousekeepingMetadata.Builder() - .path("s3://some/path/") - .databaseName("aRandomDatabase") - .tableName("aRandomTable") - .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") - .housekeepingStatus(SCHEDULED) - .creationTimestamp(CREATION_TIMESTAMP) - .modifiedTimestamp(CREATION_TIMESTAMP) - .cleanupDelay(Duration.parse("P3D")) - .cleanupAttempts(0) - .lifecycleType(EXPIRED.toString()) - .build(); - } - - @Test - public void test(){ - beekeeperServiceImpl.saveTable(table1); - List tables = beekeeperServiceImpl.returnAllTables(); - System.out.println("AAA tables:"+tables.size()); - } +// private HousekeepingMetadataRepository housekeepingMetadataRepository; +// private HousekeepingMetadata table1; +// private final BeekeeperServiceImpl beekeeperServiceImpl; +// +// public ReturnTablesTest( +// HousekeepingMetadataRepository housekeepingMetadataRepository, +// BeekeeperServiceImpl beekeeperServiceImpl) {this.housekeepingMetadataRepository = housekeepingMetadataRepository; +// this.beekeeperServiceImpl = beekeeperServiceImpl; +// } +// +// @BeforeEach +// public void createTables(){ +// LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); +// table1 = new HousekeepingMetadata.Builder() +// .path("s3://some/path/") +// .databaseName("aRandomDatabase") +// .tableName("aRandomTable") +// .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") +// .housekeepingStatus(SCHEDULED) +// .creationTimestamp(CREATION_TIMESTAMP) +// .modifiedTimestamp(CREATION_TIMESTAMP) +// .cleanupDelay(Duration.parse("P3D")) +// .cleanupAttempts(0) +// .lifecycleType(EXPIRED.toString()) +// .build(); +// } +// +// @Test +// public void test(){ +// beekeeperServiceImpl.saveTable(table1); +// List tables = beekeeperServiceImpl.returnAllTables(); +// System.out.println("AAA tables:"+tables.size()); +// } } From 17c38c033e1a69968d6624c530e69ef27f326ede Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 10:58:41 +0100 Subject: [PATCH 007/115] idk --- beekeeper-api/pom.xml | 24 ------------------ .../beekeeper/api/BeekeeperService.java | 18 ++++++++++--- .../beekeeper/api/BeekeeperServiceImpl.java | 16 +++++++++++- .../beekeeper/api/ReturnTablesTest.java | 25 +++++++++++-------- 4 files changed, 45 insertions(+), 38 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 041b37db..49f91bc3 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -20,35 +20,11 @@ org.springframework.boot spring-boot-actuator - - org.springframework - spring-test - test - - - com.expediagroup - beekeeper-path-cleanup - 3.1.1-SNAPSHOT - test - - - com.expediagroup - beekeeper-vacuum-tool - 3.1.1-SNAPSHOT - test - - - com.expediagroup - beekeeper-metadata-cleanup - 3.1.1-SNAPSHOT - test - com.expediagroup beekeeper-core ${project.version} - diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java index 8c6ba3a6..2f3e94df 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java @@ -1,9 +1,21 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.api; -import java.util.List; -import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; public interface BeekeeperService { diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java index 43d1c271..623befdc 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java @@ -1,8 +1,22 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.api; import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java index 9f697c54..e41e3bfc 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java @@ -1,20 +1,25 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.api; -import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; -import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(MockitoExtension.class) public class ReturnTablesTest { From afb6bbce19a1eac9a9940aeb6633a745e9a09349 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 11:00:47 +0100 Subject: [PATCH 008/115] metastore --- .../HousekeepingMetadataRepository.java | 16 ---------------- .../HousekeepingMetadataRepositoryTest.java | 17 +---------------- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index 1592c30d..cf516d00 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -55,22 +55,6 @@ Optional findRecordForCleanupByDbTableAndPartitionName( @Param("databaseName") String databaseName, @Param("tableName") String tableName, @Param("partitionName") String partitionName); - /** - * Returns the record that matches the inputs given, if there is one. - * - * - * @param databaseName - * @param tableName - * @return - */ - @Query(value = "from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName ") - Optional findRecord( - @Param("databaseName") String databaseName, - @Param("tableName") String tableName); - - /** * Returns the maximum value for the cleanupTimestamp for a database and table name pair. * diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index e61c2dba..7ab46329 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -77,22 +77,7 @@ public void setupDb() { housekeepingMetadataRepository.deleteAll(); } - - @Test - public void returnAllTables(){ - HousekeepingMetadata table1 = createPartitionedEntityHousekeepingTable(); - HousekeepingMetadata table2 = createPartitionedEntityHousekeepingTable(); - HousekeepingMetadata table3 = createPartitionedEntityHousekeepingTable(); - - housekeepingMetadataRepository.save(table1); - housekeepingMetadataRepository.save(table2); - housekeepingMetadataRepository.save(table3); - - List tables = housekeepingMetadataRepository.findAll(); - assertThat(tables.size()).isEqualTo(3); - System.out.println("AA:"+tables.toString()); - } - + @Test public void typicalSave() { HousekeepingMetadata table = createPartitionedEntityHousekeepingTable(); From 5e28dddaa30466d7b79ea4af18bc3c57a4cb7e6a Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 11:00:57 +0100 Subject: [PATCH 009/115] space --- .../core/repository/HousekeepingMetadataRepositoryTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 7ab46329..255bc3b2 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -77,7 +77,6 @@ public void setupDb() { housekeepingMetadataRepository.deleteAll(); } - @Test public void typicalSave() { HousekeepingMetadata table = createPartitionedEntityHousekeepingTable(); From 486bc1aac9492211102311df7de68adac211d983 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 11:19:01 +0100 Subject: [PATCH 010/115] fixing service classes --- .../beekeeper/api/BeekeeperController.java | 4 ++-- .../expediagroup/beekeeper/api/BeekeeperService.java | 4 ++++ .../beekeeper/api/BeekeeperServiceImpl.java | 11 +++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index c3c14416..8fe4acf8 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -25,11 +25,11 @@ @RestController public class BeekeeperController { - private BeekeeperServiceImpl beekeeperServiceImpl; + private BeekeeperService beekeeperService; @RequestMapping("/tables") public List getAllTables() { - return beekeeperServiceImpl.returnAllTables(); + return beekeeperService.returnAllTables(); } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java index 2f3e94df..d0b2fbfd 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java @@ -15,8 +15,12 @@ */ package com.expediagroup.beekeeper.api; +import java.util.List; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; public interface BeekeeperService { + List returnAllTables(); + } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java index 623befdc..52ff6f2d 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java @@ -17,18 +17,25 @@ import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @Service -public class BeekeeperServiceImpl { +public class BeekeeperServiceImpl implements BeekeeperService{ - private HousekeepingMetadataRepository housekeepingMetadataRepository; + private final HousekeepingMetadataRepository housekeepingMetadataRepository; + + @Autowired + public BeekeeperServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { + this.housekeepingMetadataRepository = housekeepingMetadataRepository; + } // Method for the GET tables/ endpoint public List returnAllTables() { return housekeepingMetadataRepository.findAll(); } + } From 32aa4a80c18185172a6fcd2d61f128cd10e29afb Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 12:18:19 +0100 Subject: [PATCH 011/115] sabe tables --- .../beekeeper/api/BeekeeperService.java | 2 + .../beekeeper/api/BeekeeperServiceImpl.java | 6 ++ .../beekeeper/api/ReturnTablesTest.java | 72 ++++++++++--------- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java index d0b2fbfd..1b8dd649 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java @@ -23,4 +23,6 @@ public interface BeekeeperService { List returnAllTables(); + void saveTable(HousekeepingMetadata table); + } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java index 52ff6f2d..2ae1e410 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java @@ -38,4 +38,10 @@ public List returnAllTables() { return housekeepingMetadataRepository.findAll(); } + // Method to save example tables so that I can test the methods + public void saveTable(HousekeepingMetadata table){ + housekeepingMetadataRepository.save(table); + System.out.println("count:"+housekeepingMetadataRepository.count()); + } + } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java index e41e3bfc..102a766f 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java @@ -17,45 +17,51 @@ +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @ExtendWith(MockitoExtension.class) public class ReturnTablesTest { -// private HousekeepingMetadataRepository housekeepingMetadataRepository; -// private HousekeepingMetadata table1; -// private final BeekeeperServiceImpl beekeeperServiceImpl; -// -// public ReturnTablesTest( -// HousekeepingMetadataRepository housekeepingMetadataRepository, -// BeekeeperServiceImpl beekeeperServiceImpl) {this.housekeepingMetadataRepository = housekeepingMetadataRepository; -// this.beekeeperServiceImpl = beekeeperServiceImpl; -// } -// -// @BeforeEach -// public void createTables(){ -// LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); -// table1 = new HousekeepingMetadata.Builder() -// .path("s3://some/path/") -// .databaseName("aRandomDatabase") -// .tableName("aRandomTable") -// .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") -// .housekeepingStatus(SCHEDULED) -// .creationTimestamp(CREATION_TIMESTAMP) -// .modifiedTimestamp(CREATION_TIMESTAMP) -// .cleanupDelay(Duration.parse("P3D")) -// .cleanupAttempts(0) -// .lifecycleType(EXPIRED.toString()) -// .build(); -// } -// -// @Test -// public void test(){ -// beekeeperServiceImpl.saveTable(table1); -// List tables = beekeeperServiceImpl.returnAllTables(); -// System.out.println("AAA tables:"+tables.size()); -// } + private HousekeepingMetadata table1; + private BeekeeperService beekeeperService; + + public ReturnTablesTest(BeekeeperService beekeeperService) {this.beekeeperService = beekeeperService;} + + @BeforeEach + public void createTables(){ + LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + table1 = new HousekeepingMetadata.Builder() + .path("s3://some/path/") + .databaseName("aRandomDatabase") + .tableName("aRandomTable") + .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupDelay(Duration.parse("P3D")) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); + } + + @Test + public void test(){ + beekeeperService.saveTable(table1); + List tables = beekeeperService.returnAllTables(); + System.out.println("AAA tables:"+tables.size()); + } } From e5f2bcc8fdd1a6a13f66b8085438a9776a1d0949 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 14:31:47 +0100 Subject: [PATCH 012/115] test class --- .../beekeeper/api/BeekeeperService.java | 1 + ...blesTest.java => BeekeeperServiceImplTest.java} | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{ReturnTablesTest.java => BeekeeperServiceImplTest.java} (79%) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java index 1b8dd649..f176bed0 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java @@ -18,6 +18,7 @@ import java.util.List; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; public interface BeekeeperService { diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java similarity index 79% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java index 102a766f..2e5d61fd 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/ReturnTablesTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java @@ -28,20 +28,24 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(MockitoExtension.class) -public class ReturnTablesTest { +public class BeekeeperServiceImplTest { private HousekeepingMetadata table1; - private BeekeeperService beekeeperService; + private BeekeeperServiceImpl beekeeperServiceImpl; - public ReturnTablesTest(BeekeeperService beekeeperService) {this.beekeeperService = beekeeperService;} + @Mock + private HousekeepingMetadataRepository housekeepingMetadataRepository; @BeforeEach public void createTables(){ + beekeeperServiceImpl = new BeekeeperServiceImpl(housekeepingMetadataRepository); LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); table1 = new HousekeepingMetadata.Builder() .path("s3://some/path/") @@ -59,8 +63,8 @@ public void createTables(){ @Test public void test(){ - beekeeperService.saveTable(table1); - List tables = beekeeperService.returnAllTables(); + beekeeperServiceImpl.saveTable(table1); + List tables = beekeeperServiceImpl.returnAllTables(); System.out.println("AAA tables:"+tables.size()); } From 34cc340cca1bbd7f03cf765e019d0c6cc6d9287e Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 31 Mar 2021 14:52:35 +0100 Subject: [PATCH 013/115] works --- .../api/BeekeeperServiceImplTest.java | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java index 2e5d61fd..30f3a3e5 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java @@ -16,13 +16,18 @@ package com.expediagroup.beekeeper.api; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import java.time.Duration; import java.time.LocalDateTime; import java.time.ZoneId; +import java.util.ArrayList; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -38,6 +43,7 @@ public class BeekeeperServiceImplTest { private HousekeepingMetadata table1; + private HousekeepingMetadata table2; private BeekeeperServiceImpl beekeeperServiceImpl; @Mock @@ -46,7 +52,9 @@ public class BeekeeperServiceImplTest { @BeforeEach public void createTables(){ beekeeperServiceImpl = new BeekeeperServiceImpl(housekeepingMetadataRepository); + LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + table1 = new HousekeepingMetadata.Builder() .path("s3://some/path/") .databaseName("aRandomDatabase") @@ -59,13 +67,29 @@ public void createTables(){ .cleanupAttempts(0) .lifecycleType(EXPIRED.toString()) .build(); + table1 = new HousekeepingMetadata.Builder() + .path("s3://some/path2/") + .databaseName("aRandomDatabase2") + .tableName("aRandomTable2") + .partitionName("event_date=2020-01-012/event_hour=2/event_type=B") + .housekeepingStatus(DELETED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupDelay(Duration.parse("P3D")) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); } @Test public void test(){ - beekeeperServiceImpl.saveTable(table1); - List tables = beekeeperServiceImpl.returnAllTables(); - System.out.println("AAA tables:"+tables.size()); + List tables = new ArrayList(); + tables.add(table1); + tables.add(table2); + when(housekeepingMetadataRepository.findAll()).thenReturn(tables); + List result = beekeeperServiceImpl.returnAllTables(); + assertThat(result.size()).isEqualTo(2); + assertThat(tables).isEqualTo(result); } } From 62cfc65cc424ee0b5062a37719623f41cd87d7d2 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 1 Apr 2021 11:28:39 +0100 Subject: [PATCH 014/115] remove mthod --- .../com/expediagroup/beekeeper/api/BeekeeperController.java | 3 ++- .../com/expediagroup/beekeeper/api/BeekeeperService.java | 2 -- .../expediagroup/beekeeper/api/BeekeeperServiceImpl.java | 6 ------ .../beekeeper/api/BeekeeperServiceImplTest.java | 2 +- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 8fe4acf8..f7260cce 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -18,6 +18,7 @@ import java.util.List; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @@ -27,7 +28,7 @@ public class BeekeeperController { private BeekeeperService beekeeperService; - @RequestMapping("/tables") + @RequestMapping(value = "/tables", method = RequestMethod.GET) public List getAllTables() { return beekeeperService.returnAllTables(); } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java index f176bed0..76a01d6a 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java @@ -24,6 +24,4 @@ public interface BeekeeperService { List returnAllTables(); - void saveTable(HousekeepingMetadata table); - } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java index 2ae1e410..52ff6f2d 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java @@ -38,10 +38,4 @@ public List returnAllTables() { return housekeepingMetadataRepository.findAll(); } - // Method to save example tables so that I can test the methods - public void saveTable(HousekeepingMetadata table){ - housekeepingMetadataRepository.save(table); - System.out.println("count:"+housekeepingMetadataRepository.count()); - } - } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java index 30f3a3e5..8227b02e 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java @@ -52,7 +52,7 @@ public class BeekeeperServiceImplTest { @BeforeEach public void createTables(){ beekeeperServiceImpl = new BeekeeperServiceImpl(housekeepingMetadataRepository); - + LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); table1 = new HousekeepingMetadata.Builder() From 0099efc6e81174fe3ccea76d5f9cc5500f78f068 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 1 Apr 2021 11:30:38 +0100 Subject: [PATCH 015/115] value --- .../com/expediagroup/beekeeper/api/BeekeeperController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index f7260cce..76347cb7 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -28,7 +28,7 @@ public class BeekeeperController { private BeekeeperService beekeeperService; - @RequestMapping(value = "/tables", method = RequestMethod.GET) + @RequestMapping("/tables") public List getAllTables() { return beekeeperService.returnAllTables(); } From 8b5dc9aa58b0e71cd08cd55e2f17b2dd53a25de9 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 12:11:06 +0100 Subject: [PATCH 016/115] adding classes --- beekeeper-api/pom.xml | 10 +++++++ .../beekeeper/api/BeekeeperController.java | 16 ++++------- .../beekeeper/api/BeekeeperRepository.java | 16 +++++++++++ ...ce.java => HousekeepingEntityService.java} | 10 ++++--- ...ava => HousekeepingEntityServiceImpl.java} | 12 ++++++--- .../src/main/resources/application.properties | 9 +++++++ ...=> HouseKeepingEntityServiceImplTest.java} | 27 +++++++++---------- 7 files changed, 68 insertions(+), 32 deletions(-) create mode 100644 beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{BeekeeperService.java => HousekeepingEntityService.java} (66%) rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{BeekeeperServiceImpl.java => HousekeepingEntityServiceImpl.java} (68%) create mode 100644 beekeeper-api/src/main/resources/application.properties rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{BeekeeperServiceImplTest.java => HouseKeepingEntityServiceImplTest.java} (81%) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 49f91bc3..c87cec05 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -20,11 +20,21 @@ org.springframework.boot spring-boot-actuator + + com.graphql-java + graphql-spring-boot-starter + 5.0.2 + com.expediagroup beekeeper-core ${project.version} + + mysql + mysql-connector-java + 8.0.18 + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 76347cb7..963814c1 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -15,22 +15,16 @@ */ package com.expediagroup.beekeeper.api; -import java.util.List; - -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; -import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; - @RestController public class BeekeeperController { - private BeekeeperService beekeeperService; + private HousekeepingEntityService beekeeperService; - @RequestMapping("/tables") - public List getAllTables() { - return beekeeperService.returnAllTables(); - } +// @RequestMapping("/tables") +// public List getAllTables() { +// return beekeeperService.returnAllTables(); +// } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java new file mode 100644 index 00000000..67eba651 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java @@ -0,0 +1,16 @@ +package com.expediagroup.beekeeper.api; + +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.stereotype.Repository; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + +@Repository + public interface BeekeeperRepository extends PagingAndSortingRepository, + JpaSpecificationExecutor { + + } + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java similarity index 66% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java index 76a01d6a..b7b60ce3 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java @@ -17,11 +17,15 @@ import java.util.List; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; + +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; -public interface BeekeeperService { +public interface HousekeepingEntityService { - List returnAllTables(); + Page returnAllTables(Specification spec, Pageable pageable); } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java similarity index 68% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java index 52ff6f2d..da7d58e1 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java @@ -18,24 +18,28 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Service; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @Service -public class BeekeeperServiceImpl implements BeekeeperService{ +public class HousekeepingEntityServiceImpl implements HousekeepingEntityService { private final HousekeepingMetadataRepository housekeepingMetadataRepository; @Autowired - public BeekeeperServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { + public HousekeepingEntityServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { this.housekeepingMetadataRepository = housekeepingMetadataRepository; } // Method for the GET tables/ endpoint - public List returnAllTables() { - return housekeepingMetadataRepository.findAll(); + public Page returnAllTables(Specification spec, Pageable pageable) { + return housekeepingMetadataRepository.findAll(spec, pageable); } + } diff --git a/beekeeper-api/src/main/resources/application.properties b/beekeeper-api/src/main/resources/application.properties new file mode 100644 index 00000000..9daf9337 --- /dev/null +++ b/beekeeper-api/src/main/resources/application.properties @@ -0,0 +1,9 @@ +database=database_name +table=table_name +metastore-uri=thrift://localhost:9083 +dry-run=true +default-cleanup-delay=P3D + +spring.datasource.url=jdbc:mysql://beekeeper-db-host:3306/beekeeper?useSSL=false +spring.datasource.username=username +spring.datasource.password=password \ No newline at end of file diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java similarity index 81% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java index 8227b02e..e1cc9d25 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java @@ -19,7 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; - import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; @@ -40,18 +39,18 @@ import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(MockitoExtension.class) -public class BeekeeperServiceImplTest { +public class HouseKeepingEntityServiceImplTest { private HousekeepingMetadata table1; private HousekeepingMetadata table2; - private BeekeeperServiceImpl beekeeperServiceImpl; + private HousekeepingEntityServiceImpl beekeeperServiceImpl; @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @BeforeEach public void createTables(){ - beekeeperServiceImpl = new BeekeeperServiceImpl(housekeepingMetadataRepository); + beekeeperServiceImpl = new HousekeepingEntityServiceImpl(housekeepingMetadataRepository); LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); @@ -81,15 +80,15 @@ public void createTables(){ .build(); } - @Test - public void test(){ - List tables = new ArrayList(); - tables.add(table1); - tables.add(table2); - when(housekeepingMetadataRepository.findAll()).thenReturn(tables); - List result = beekeeperServiceImpl.returnAllTables(); - assertThat(result.size()).isEqualTo(2); - assertThat(tables).isEqualTo(result); - } +// @Test +// public void test(){ +// List tables = new ArrayList(); +// tables.add(table1); +// tables.add(table2); +// when(housekeepingMetadataRepository.findAll()).thenReturn(tables); +// List result = beekeeperServiceImpl.returnAllTables(); +// assertThat(result.size()).isEqualTo(2); +// assertThat(tables).isEqualTo(result); +// } } From 41d94d1b764a42d32109f3a666e5a28f49f4a229 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 12:14:33 +0100 Subject: [PATCH 017/115] delete file --- beekeeper-api/src/main/resources/application.properties | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 beekeeper-api/src/main/resources/application.properties diff --git a/beekeeper-api/src/main/resources/application.properties b/beekeeper-api/src/main/resources/application.properties deleted file mode 100644 index 9daf9337..00000000 --- a/beekeeper-api/src/main/resources/application.properties +++ /dev/null @@ -1,9 +0,0 @@ -database=database_name -table=table_name -metastore-uri=thrift://localhost:9083 -dry-run=true -default-cleanup-delay=P3D - -spring.datasource.url=jdbc:mysql://beekeeper-db-host:3306/beekeeper?useSSL=false -spring.datasource.username=username -spring.datasource.password=password \ No newline at end of file From b1299a123a759187aeea235339c9d3d049d48e87 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 14:09:25 +0100 Subject: [PATCH 018/115] classes --- beekeeper-api/pom.xml | 21 +++++++++++++++++++ .../beekeeper/api/BeekeeperController.java | 17 ++++++++++----- .../beekeeper/api/BeekeeperRepository.java | 20 ++++++++++++++---- .../api/HousekeepingEntityService.java | 1 - .../api/HousekeepingEntityServiceImpl.java | 2 +- .../beekeeper/api/HousekeepingMetadataId.java | 13 ++++++++++++ .../HouseKeepingEntityServiceImplTest.java | 5 ----- 7 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index c87cec05..78b98d86 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -20,6 +20,11 @@ org.springframework.boot spring-boot-actuator + + net.kaczmarzyk + specification-arg-resolver + 2.6.1 + com.graphql-java graphql-spring-boot-starter @@ -35,6 +40,22 @@ mysql-connector-java 8.0.18 + + org.projectlombok + lombok + + + org.projectlombok + lombok + + + org.projectlombok + lombok + + + org.projectlombok + lombok + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 963814c1..154c47dd 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -15,16 +15,23 @@ */ package com.expediagroup.beekeeper.api; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + @RestController public class BeekeeperController { - private HousekeepingEntityService beekeeperService; + private HousekeepingEntityService HousekeepingEntityService; -// @RequestMapping("/tables") -// public List getAllTables() { -// return beekeeperService.returnAllTables(); -// } + @RequestMapping("/tables") + public ResponseEntity> getAllTables(Specification spec, Pageable pageable) { + return ResponseEntity.ok(HousekeepingEntityService.returnAllTables(spec, pageable)); + } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java index 67eba651..3fb2c4bd 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java @@ -1,6 +1,20 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.api; -import java.util.Optional; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.repository.PagingAndSortingRepository; @@ -9,8 +23,6 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @Repository - public interface BeekeeperRepository extends PagingAndSortingRepository, + public interface BeekeeperRepository extends PagingAndSortingRepository, JpaSpecificationExecutor { - } - diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java index b7b60ce3..3af9dfb9 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java @@ -15,7 +15,6 @@ */ package com.expediagroup.beekeeper.api; -import java.util.List; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java index da7d58e1..06d83708 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java @@ -15,7 +15,6 @@ */ package com.expediagroup.beekeeper.api; -import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; @@ -37,6 +36,7 @@ public HousekeepingEntityServiceImpl(HousekeepingMetadataRepository housekeeping } // Method for the GET tables/ endpoint + @Override public Page returnAllTables(Specification spec, Pageable pageable) { return housekeepingMetadataRepository.findAll(spec, pageable); } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java new file mode 100644 index 00000000..0eacea14 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java @@ -0,0 +1,13 @@ +package com.expediagroup.beekeeper.api; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class HousekeepingMetadataId { + private String databaseName; + private String tableName; +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java index e1cc9d25..a81cf7b9 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java @@ -16,8 +16,6 @@ package com.expediagroup.beekeeper.api; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; @@ -26,11 +24,8 @@ import java.time.Duration; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.ArrayList; -import java.util.List; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; From cd461c55b217b196093567011501580e492bc95c Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 14:10:42 +0100 Subject: [PATCH 019/115] override --- .../beekeeper/api/HousekeepingEntityServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java index 06d83708..012d437a 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java @@ -36,7 +36,6 @@ public HousekeepingEntityServiceImpl(HousekeepingMetadataRepository housekeeping } // Method for the GET tables/ endpoint - @Override public Page returnAllTables(Specification spec, Pageable pageable) { return housekeepingMetadataRepository.findAll(spec, pageable); } From d73afe5ef21adbf9b8b492eb08b2cec8951cba3a Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 15:23:01 +0100 Subject: [PATCH 020/115] pages --- beekeeper-api/pom.xml | 13 ++---- .../beekeeper/api/BeekeeperRepository.java | 28 ------------- .../beekeeper/api/HousekeepingMetadataId.java | 13 ------ .../HouseKeepingEntityServiceImplTest.java | 42 ++++++++++++++----- beekeeper-core/pom.xml | 7 ++++ .../HousekeepingMetadataRepository.java | 5 ++- .../HousekeepingMetadataRepositoryTest.java | 8 ++-- 7 files changed, 50 insertions(+), 66 deletions(-) delete mode 100644 beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java delete mode 100644 beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 78b98d86..302506a7 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -45,16 +45,9 @@ lombok - org.projectlombok - lombok - - - org.projectlombok - lombok - - - org.projectlombok - lombok + org.hamcrest + hamcrest-core + test diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java deleted file mode 100644 index 3fb2c4bd..00000000 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperRepository.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright (C) 2019-2021 Expedia, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.expediagroup.beekeeper.api; - - -import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.data.repository.PagingAndSortingRepository; -import org.springframework.stereotype.Repository; - -import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; - -@Repository - public interface BeekeeperRepository extends PagingAndSortingRepository, - JpaSpecificationExecutor { - } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java deleted file mode 100644 index 0eacea14..00000000 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataId.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.expediagroup.beekeeper.api; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class HousekeepingMetadataId { - private String databaseName; - private String tableName; -} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java index a81cf7b9..f10f8062 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java @@ -15,7 +15,9 @@ */ package com.expediagroup.beekeeper.api; - +import static org.mockito.Mockito.when; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.CoreMatchers.is; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; @@ -24,11 +26,20 @@ import java.time.Duration; import java.time.LocalDateTime; import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; + +import com.google.common.collect.Lists; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @@ -42,6 +53,10 @@ public class HouseKeepingEntityServiceImplTest { @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; + @Mock + private Specification spec; + @Mock + private Pageable pageable; @BeforeEach public void createTables(){ @@ -75,15 +90,20 @@ public void createTables(){ .build(); } -// @Test -// public void test(){ -// List tables = new ArrayList(); -// tables.add(table1); -// tables.add(table2); -// when(housekeepingMetadataRepository.findAll()).thenReturn(tables); -// List result = beekeeperServiceImpl.returnAllTables(); -// assertThat(result.size()).isEqualTo(2); -// assertThat(tables).isEqualTo(result); -// } + @Test + public void test(){ + List tables = new ArrayList(); + tables.add(table1); + tables.add(table2); + + //System.out.println(tables.size()); + Page tablesPage = new PageImpl(tables,pageable, tables.size()); + //System.out.println(tablesPage.getSize()); + + when(housekeepingMetadataRepository.findAll(spec,pageable)).thenReturn(tablesPage); + Page result = housekeepingMetadataRepository.findAll(spec,pageable); + assertThat(result.getSize(),is(2)); + assertThat(tables,is(result)); + } } diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 2b128307..f9e1e9dd 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -40,6 +40,13 @@ 27.1-jre + + org.hamcrest + hamcrest + 2.2 + test + + io.micrometer diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index cf516d00..b0470092 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -21,12 +21,15 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -public interface HousekeepingMetadataRepository extends JpaRepository { +public interface HousekeepingMetadataRepository extends PagingAndSortingRepository, + JpaSpecificationExecutor { @Query(value = "from HousekeepingMetadata t where t.cleanupTimestamp <= :instant " + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 255bc3b2..5e5871e6 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -44,6 +44,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import com.google.common.collect.Lists; + import com.expediagroup.beekeeper.core.TestApplication; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @@ -83,7 +85,7 @@ public void typicalSave() { housekeepingMetadataRepository.save(table); - List tables = housekeepingMetadataRepository.findAll(); + List tables = Lists.newArrayList(housekeepingMetadataRepository.findAll()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata savedTable = tables.get(0); assertThat(savedTable.getPath()).isEqualTo(PATH); @@ -108,7 +110,7 @@ public void typicalUpdate() { savedTable.setCleanupAttempts(savedTable.getCleanupAttempts() + 1); housekeepingMetadataRepository.save(savedTable); - List tables = housekeepingMetadataRepository.findAll(); + List tables = Lists.newArrayList(housekeepingMetadataRepository.findAll()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata updatedTable = tables.get(0); assertThat(updatedTable.getHousekeepingStatus()).isEqualTo(DELETED); @@ -141,7 +143,7 @@ public void notNullableTableNameField() { public void timezone() { HousekeepingMetadata path = createPartitionedEntityHousekeepingTable(); housekeepingMetadataRepository.save(path); - List tables = housekeepingMetadataRepository.findAll(); + List tables = Lists.newArrayList(housekeepingMetadataRepository.findAll()); assertThat(tables.size()).isEqualTo(1); HousekeepingMetadata savedTable = tables.get(0); From 77ddf102743628fd15c5f6a7071a192dc7ecfa6f Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 15:24:09 +0100 Subject: [PATCH 021/115] wrong import --- beekeeper-core/pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index f9e1e9dd..2b128307 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -40,13 +40,6 @@ 27.1-jre - - org.hamcrest - hamcrest - 2.2 - test - - io.micrometer From 50881e38613b663620b73d2ae75daaea5ca22d17 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 16:27:53 +0100 Subject: [PATCH 022/115] fixing test class --- .../api/HouseKeepingEntityServiceImplTest.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java index f10f8062..8973961c 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java @@ -92,18 +92,15 @@ public void createTables(){ @Test public void test(){ + List tables = new ArrayList(); tables.add(table1); tables.add(table2); + when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(new PageImpl<>(tables)); + Page result = beekeeperServiceImpl.returnAllTables(spec,pageable); - //System.out.println(tables.size()); - Page tablesPage = new PageImpl(tables,pageable, tables.size()); - //System.out.println(tablesPage.getSize()); + assertThat(new PageImpl<>(tables),is(result)); - when(housekeepingMetadataRepository.findAll(spec,pageable)).thenReturn(tablesPage); - Page result = housekeepingMetadataRepository.findAll(spec,pageable); - assertThat(result.getSize(),is(2)); - assertThat(tables,is(result)); } } From e19796f40988d621abb50941039188d3543cfcf0 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 16:34:15 +0100 Subject: [PATCH 023/115] table name --- .../beekeeper/api/HouseKeepingEntityServiceImplTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java index 8973961c..daa0e2a4 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java @@ -76,7 +76,7 @@ public void createTables(){ .cleanupAttempts(0) .lifecycleType(EXPIRED.toString()) .build(); - table1 = new HousekeepingMetadata.Builder() + table2 = new HousekeepingMetadata.Builder() .path("s3://some/path2/") .databaseName("aRandomDatabase2") .tableName("aRandomTable2") From 0b04c31a87f94f31f982e520e5de06a8e68fdc00 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 6 Apr 2021 17:15:52 +0100 Subject: [PATCH 024/115] adding controller test class --- beekeeper-api/pom.xml | 5 + .../beekeeper/api/BeekeeperController.java | 4 +- .../api/HousekeepingEntityServiceImpl.java | 1 - .../api/BeekeeperControllerTest.java | 113 ++++++++++++++++++ .../HouseKeepingEntityServiceImplTest.java | 12 +- 5 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 302506a7..ce274725 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -49,6 +49,11 @@ hamcrest-core test + + org.springframework + spring-test + test + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 154c47dd..9e49acca 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -27,11 +27,11 @@ @RestController public class BeekeeperController { - private HousekeepingEntityService HousekeepingEntityService; + private HousekeepingEntityService housekeepingEntityService; @RequestMapping("/tables") public ResponseEntity> getAllTables(Specification spec, Pageable pageable) { - return ResponseEntity.ok(HousekeepingEntityService.returnAllTables(spec, pageable)); + return ResponseEntity.ok(housekeepingEntityService.returnAllTables(spec, pageable)); } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java index 012d437a..bd16065f 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java @@ -40,5 +40,4 @@ public Page returnAllTables(Specification spec; + @Mock + private Pageable pageable; + + private HousekeepingMetadata table1; + private HousekeepingMetadata table2; + + @BeforeEach + public void createTables(){ + + LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + + table1 = new HousekeepingMetadata.Builder() + .path("s3://some/path/") + .databaseName("aRandomDatabase") + .tableName("aRandomTable") + .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupDelay(Duration.parse("P3D")) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); + table2 = new HousekeepingMetadata.Builder() + .path("s3://some/path2/") + .databaseName("aRandomDatabase2") + .tableName("aRandomTable2") + .partitionName("event_date=2020-01-012/event_hour=2/event_type=B") + .housekeepingStatus(DELETED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupDelay(Duration.parse("P3D")) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); + } + + @Test + public void getAllTables() throws Exception { + List tablesList = new ArrayList(); + tablesList.add(table1); + tablesList.add(table2); + Page tables = new PageImpl<>(tablesList); + System.out.println(tables); + when(housekeepingEntityService.returnAllTables(spec, pageable)).thenReturn(tables); + mockMvc + .perform(get(housekeepingEntityService.returnAllTables(spec, pageable).toString())) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(APPLICATION_JSON)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + } +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java index daa0e2a4..2c73347b 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java @@ -15,9 +15,9 @@ */ package com.expediagroup.beekeeper.api; -import static org.mockito.Mockito.when; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.when; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; @@ -39,8 +39,6 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import com.google.common.collect.Lists; - import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @@ -49,7 +47,7 @@ public class HouseKeepingEntityServiceImplTest { private HousekeepingMetadata table1; private HousekeepingMetadata table2; - private HousekeepingEntityServiceImpl beekeeperServiceImpl; + private HousekeepingEntityServiceImpl housekeepingEntityServiceImpl; @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @@ -60,7 +58,7 @@ public class HouseKeepingEntityServiceImplTest { @BeforeEach public void createTables(){ - beekeeperServiceImpl = new HousekeepingEntityServiceImpl(housekeepingMetadataRepository); + housekeepingEntityServiceImpl = new HousekeepingEntityServiceImpl(housekeepingMetadataRepository); LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); @@ -97,7 +95,7 @@ public void test(){ tables.add(table1); tables.add(table2); when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(new PageImpl<>(tables)); - Page result = beekeeperServiceImpl.returnAllTables(spec,pageable); + Page result = housekeepingEntityServiceImpl.returnAllTables(spec,pageable); assertThat(new PageImpl<>(tables),is(result)); From b3a8ae6e4e3ca8642a73a3777c05765a5af9e1a7 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 7 Apr 2021 10:52:45 +0100 Subject: [PATCH 025/115] controller test class does not work --- beekeeper-api/pom.xml | 5 +++++ .../beekeeper/api/BeekeeperControllerTest.java | 15 +++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index ce274725..ab4fd0cd 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -54,6 +54,11 @@ spring-test test + + org.springframework.boot + spring-boot-test + test + diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index 517d4b55..fff4a557 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -33,8 +33,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; @@ -45,8 +48,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +@ExtendWith(MockitoExtension.class) public class BeekeeperControllerTest { @Autowired @@ -54,12 +60,12 @@ public class BeekeeperControllerTest { @Autowired private ObjectMapper objectMapper; - @Mock - private HousekeepingEntityService housekeepingEntityService; @Mock private Specification spec; @Mock private Pageable pageable; + @Mock + private HousekeepingEntityService housekeepingEntityService; private HousekeepingMetadata table1; private HousekeepingMetadata table2; @@ -94,15 +100,16 @@ public void createTables(){ .lifecycleType(EXPIRED.toString()) .build(); } - + @Test public void getAllTables() throws Exception { List tablesList = new ArrayList(); tablesList.add(table1); tablesList.add(table2); Page tables = new PageImpl<>(tablesList); - System.out.println(tables); + System.out.println(tables.getContent()); when(housekeepingEntityService.returnAllTables(spec, pageable)).thenReturn(tables); + mockMvc .perform(get(housekeepingEntityService.returnAllTables(spec, pageable).toString())) .andDo(MockMvcResultHandlers.print()) From 36e4cac490d35a1b5be8c7f39de5b0240ec52620 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 7 Apr 2021 14:23:52 +0100 Subject: [PATCH 026/115] pom --- beekeeper-api/pom.xml | 25 +++++++------------ .../api/BeekeeperControllerTest.java | 2 +- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index ab4fd0cd..de585f1c 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -20,6 +20,11 @@ org.springframework.boot spring-boot-actuator + + com.expediagroup + beekeeper-core + ${project.version} + net.kaczmarzyk specification-arg-resolver @@ -30,23 +35,11 @@ graphql-spring-boot-starter 5.0.2 + + - com.expediagroup - beekeeper-core - ${project.version} - - - mysql - mysql-connector-java - 8.0.18 - - - org.projectlombok - lombok - - - org.hamcrest - hamcrest-core + org.assertj + assertj-core test diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index fff4a557..179dccec 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -100,7 +100,7 @@ public void createTables(){ .lifecycleType(EXPIRED.toString()) .build(); } - + @Test public void getAllTables() throws Exception { List tablesList = new ArrayList(); From 20d82d4976843321344cafbfe41d898a680386c7 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 7 Apr 2021 17:11:03 +0100 Subject: [PATCH 027/115] changes --- .../com/expediagroup/beekeeper/api/BeekeeperController.java | 1 + .../beekeeper/api/HousekeepingEntityServiceImpl.java | 1 - .../expediagroup/beekeeper/api/BeekeeperControllerTest.java | 6 +----- .../beekeeper/api/HouseKeepingEntityServiceImplTest.java | 5 ++--- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 9e49acca..c2d9947c 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -24,6 +24,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +@RequestMapping("/api/v1") @RestController public class BeekeeperController { diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java index bd16065f..8a9e8033 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java @@ -35,7 +35,6 @@ public HousekeepingEntityServiceImpl(HousekeepingMetadataRepository housekeeping this.housekeepingMetadataRepository = housekeepingMetadataRepository; } - // Method for the GET tables/ endpoint public Page returnAllTables(Specification spec, Pageable pageable) { return housekeepingMetadataRepository.findAll(spec, pageable); } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index 179dccec..55b1d67e 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -37,20 +37,16 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.result.MockMvcResultHandlers; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(MockitoExtension.class) public class BeekeeperControllerTest { diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java index 2c73347b..d2bde2d8 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java @@ -15,8 +15,7 @@ */ package com.expediagroup.beekeeper.api; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; @@ -97,7 +96,7 @@ public void test(){ when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(new PageImpl<>(tables)); Page result = housekeepingEntityServiceImpl.returnAllTables(spec,pageable); - assertThat(new PageImpl<>(tables),is(result)); + assertThat(new PageImpl<>(tables)).isEqualTo(result); } From 13b5e522398667259e6241d68e1cbaf6868e297e Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 7 Apr 2021 17:21:10 +0100 Subject: [PATCH 028/115] renaming modules --- .../com/expediagroup/beekeeper/api/BeekeeperController.java | 2 +- ...gEntityService.java => HousekeepingMetadataService.java} | 2 +- ...erviceImpl.java => HousekeepingMetadataServiceImpl.java} | 4 ++-- .../expediagroup/beekeeper/api/BeekeeperControllerTest.java | 2 +- ...plTest.java => HouseKeepingMetadataServiceImplTest.java} | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{HousekeepingEntityService.java => HousekeepingMetadataService.java} (93%) rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{HousekeepingEntityServiceImpl.java => HousekeepingMetadataServiceImpl.java} (86%) rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{HouseKeepingEntityServiceImplTest.java => HouseKeepingMetadataServiceImplTest.java} (93%) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index c2d9947c..65375d4c 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -28,7 +28,7 @@ @RestController public class BeekeeperController { - private HousekeepingEntityService housekeepingEntityService; + private HousekeepingMetadataService housekeepingEntityService; @RequestMapping("/tables") public ResponseEntity> getAllTables(Specification spec, Pageable pageable) { diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java similarity index 93% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java index 3af9dfb9..feab2287 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java @@ -23,7 +23,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -public interface HousekeepingEntityService { +public interface HousekeepingMetadataService { Page returnAllTables(Specification spec, Pageable pageable); diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java similarity index 86% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java index 8a9e8033..dceb5aca 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingEntityServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java @@ -26,12 +26,12 @@ import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @Service -public class HousekeepingEntityServiceImpl implements HousekeepingEntityService { +public class HousekeepingMetadataServiceImpl implements HousekeepingMetadataService { private final HousekeepingMetadataRepository housekeepingMetadataRepository; @Autowired - public HousekeepingEntityServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { + public HousekeepingMetadataServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { this.housekeepingMetadataRepository = housekeepingMetadataRepository; } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index 55b1d67e..f9da33c0 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -61,7 +61,7 @@ public class BeekeeperControllerTest { @Mock private Pageable pageable; @Mock - private HousekeepingEntityService housekeepingEntityService; + private HousekeepingMetadataService housekeepingEntityService; private HousekeepingMetadata table1; private HousekeepingMetadata table2; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java similarity index 93% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java index d2bde2d8..f28cc677 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingEntityServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java @@ -42,11 +42,11 @@ import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(MockitoExtension.class) -public class HouseKeepingEntityServiceImplTest { +public class HouseKeepingMetadataServiceImplTest { private HousekeepingMetadata table1; private HousekeepingMetadata table2; - private HousekeepingEntityServiceImpl housekeepingEntityServiceImpl; + private HousekeepingMetadataServiceImpl housekeepingEntityServiceImpl; @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @@ -57,7 +57,7 @@ public class HouseKeepingEntityServiceImplTest { @BeforeEach public void createTables(){ - housekeepingEntityServiceImpl = new HousekeepingEntityServiceImpl(housekeepingMetadataRepository); + housekeepingEntityServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); From 20fc86becfc2b5f30381f7a2a08b0af98d6b2239 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 7 Apr 2021 17:26:10 +0100 Subject: [PATCH 029/115] renaming variables --- .../com/expediagroup/beekeeper/api/BeekeeperController.java | 4 ++-- .../expediagroup/beekeeper/api/BeekeeperControllerTest.java | 6 +++--- .../beekeeper/api/HouseKeepingMetadataServiceImplTest.java | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 65375d4c..b54daffe 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -28,11 +28,11 @@ @RestController public class BeekeeperController { - private HousekeepingMetadataService housekeepingEntityService; + private HousekeepingMetadataService housekeepingMetadataService; @RequestMapping("/tables") public ResponseEntity> getAllTables(Specification spec, Pageable pageable) { - return ResponseEntity.ok(housekeepingEntityService.returnAllTables(spec, pageable)); + return ResponseEntity.ok(housekeepingMetadataService.returnAllTables(spec, pageable)); } } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index f9da33c0..2cd6c569 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -61,7 +61,7 @@ public class BeekeeperControllerTest { @Mock private Pageable pageable; @Mock - private HousekeepingMetadataService housekeepingEntityService; + private HousekeepingMetadataService housekeepingMetadataService; private HousekeepingMetadata table1; private HousekeepingMetadata table2; @@ -104,10 +104,10 @@ public void getAllTables() throws Exception { tablesList.add(table2); Page tables = new PageImpl<>(tablesList); System.out.println(tables.getContent()); - when(housekeepingEntityService.returnAllTables(spec, pageable)).thenReturn(tables); + when(housekeepingMetadataService.returnAllTables(spec, pageable)).thenReturn(tables); mockMvc - .perform(get(housekeepingEntityService.returnAllTables(spec, pageable).toString())) + .perform(get(housekeepingMetadataService.returnAllTables(spec, pageable).toString())) .andDo(MockMvcResultHandlers.print()) .andExpect(status().isOk()) .andExpect(content().contentType(APPLICATION_JSON)) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java index f28cc677..514b3c31 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java @@ -46,7 +46,7 @@ public class HouseKeepingMetadataServiceImplTest { private HousekeepingMetadata table1; private HousekeepingMetadata table2; - private HousekeepingMetadataServiceImpl housekeepingEntityServiceImpl; + private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @@ -57,7 +57,7 @@ public class HouseKeepingMetadataServiceImplTest { @BeforeEach public void createTables(){ - housekeepingEntityServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); + housekeepingMetadataServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); @@ -94,7 +94,7 @@ public void test(){ tables.add(table1); tables.add(table2); when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(new PageImpl<>(tables)); - Page result = housekeepingEntityServiceImpl.returnAllTables(spec,pageable); + Page result = housekeepingMetadataServiceImpl.returnAllTables(spec,pageable); assertThat(new PageImpl<>(tables)).isEqualTo(result); From f7693e92db29d2a5f3d76eefca65204c94b174a9 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 8 Apr 2021 14:21:36 +0100 Subject: [PATCH 030/115] generating dummy data class --- .../beekeeper/api/BeekeeperController.java | 7 ++- .../api/HousekeepingMetadataService.java | 2 +- .../api/HousekeepingMetadataServiceImpl.java | 2 +- .../api/BeekeeperControllerTest.java | 59 +++--------------- .../DummyHousekeepingMetadataGenerator.java | 39 ++++++++++++ .../HouseKeepingMetadataServiceImplTest.java | 60 +++---------------- 6 files changed, 62 insertions(+), 107 deletions(-) create mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/DummyHousekeepingMetadataGenerator.java diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index b54daffe..9785b359 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -19,6 +19,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -30,9 +31,9 @@ public class BeekeeperController { private HousekeepingMetadataService housekeepingMetadataService; - @RequestMapping("/tables") - public ResponseEntity> getAllTables(Specification spec, Pageable pageable) { - return ResponseEntity.ok(housekeepingMetadataService.returnAllTables(spec, pageable)); + @GetMapping("/tables") + public ResponseEntity> getAll(Specification spec, Pageable pageable) { + return ResponseEntity.ok(housekeepingMetadataService.getAll(spec, pageable)); } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java index feab2287..25d7ad8b 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java @@ -25,6 +25,6 @@ public interface HousekeepingMetadataService { - Page returnAllTables(Specification spec, Pageable pageable); + Page getAll(Specification spec, Pageable pageable); } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java index dceb5aca..b27893f0 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java @@ -35,7 +35,7 @@ public HousekeepingMetadataServiceImpl(HousekeepingMetadataRepository housekeepi this.housekeepingMetadataRepository = housekeepingMetadataRepository; } - public Page returnAllTables(Specification spec, Pageable pageable) { + public Page getAll(Specification spec, Pageable pageable) { return housekeepingMetadataRepository.findAll(spec, pageable); } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index 2cd6c569..09ce9ac2 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -21,17 +21,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; -import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; -import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; - -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.ArrayList; +import static com.expediagroup.beekeeper.api.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -63,51 +55,16 @@ public class BeekeeperControllerTest { @Mock private HousekeepingMetadataService housekeepingMetadataService; - private HousekeepingMetadata table1; - private HousekeepingMetadata table2; - - @BeforeEach - public void createTables(){ - - LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); - - table1 = new HousekeepingMetadata.Builder() - .path("s3://some/path/") - .databaseName("aRandomDatabase") - .tableName("aRandomTable") - .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") - .housekeepingStatus(SCHEDULED) - .creationTimestamp(CREATION_TIMESTAMP) - .modifiedTimestamp(CREATION_TIMESTAMP) - .cleanupDelay(Duration.parse("P3D")) - .cleanupAttempts(0) - .lifecycleType(EXPIRED.toString()) - .build(); - table2 = new HousekeepingMetadata.Builder() - .path("s3://some/path2/") - .databaseName("aRandomDatabase2") - .tableName("aRandomTable2") - .partitionName("event_date=2020-01-012/event_hour=2/event_type=B") - .housekeepingStatus(DELETED) - .creationTimestamp(CREATION_TIMESTAMP) - .modifiedTimestamp(CREATION_TIMESTAMP) - .cleanupDelay(Duration.parse("P3D")) - .cleanupAttempts(0) - .lifecycleType(EXPIRED.toString()) - .build(); - } - @Test - public void getAllTables() throws Exception { - List tablesList = new ArrayList(); - tablesList.add(table1); - tablesList.add(table2); - Page tables = new PageImpl<>(tablesList); - System.out.println(tables.getContent()); - when(housekeepingMetadataService.returnAllTables(spec, pageable)).thenReturn(tables); + public void getAllTest() throws Exception { + HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); + + when(housekeepingMetadataService.getAll(spec, pageable)).thenReturn(tables); mockMvc - .perform(get(housekeepingMetadataService.returnAllTables(spec, pageable).toString())) + .perform(get(housekeepingMetadataService.getAll(spec, pageable).toString())) .andDo(MockMvcResultHandlers.print()) .andExpect(status().isOk()) .andExpect(content().contentType(APPLICATION_JSON)) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/DummyHousekeepingMetadataGenerator.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/DummyHousekeepingMetadataGenerator.java new file mode 100644 index 00000000..d37d3c92 --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/DummyHousekeepingMetadataGenerator.java @@ -0,0 +1,39 @@ +package com.expediagroup.beekeeper.api; + +import static java.lang.String.format; + +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZoneId; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; + +public class DummyHousekeepingMetadataGenerator { + + private static final String DEFAULT_DB_NAME = "randomDatabase"; + private static final String DEFAULT_TABLE_NAME = "randomTable"; + private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + private static final Duration CLEANUP_DELAY = Duration.parse("P3D"); + + public static HousekeepingMetadata generateDummyHousekeepingMetadata() { + return generateDummyHousekeepingMetadata(DEFAULT_DB_NAME, DEFAULT_TABLE_NAME); + } + + public static HousekeepingMetadata generateDummyHousekeepingMetadata(String tableName, String databaseName) { + return new HousekeepingMetadata.Builder() + .path("s3://some/path/") + .databaseName(databaseName) + .tableName(tableName) + .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupDelay(Duration.parse("P3D")) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); + } +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java index 514b3c31..5d43ff69 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java @@ -18,17 +18,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; -import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; -import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; -import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; +import static com.expediagroup.beekeeper.api.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.ArrayList; import java.util.List; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -44,8 +37,6 @@ @ExtendWith(MockitoExtension.class) public class HouseKeepingMetadataServiceImplTest { - private HousekeepingMetadata table1; - private HousekeepingMetadata table2; private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; @Mock @@ -55,48 +46,15 @@ public class HouseKeepingMetadataServiceImplTest { @Mock private Pageable pageable; - @BeforeEach - public void createTables(){ - housekeepingMetadataServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); - - LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); - - table1 = new HousekeepingMetadata.Builder() - .path("s3://some/path/") - .databaseName("aRandomDatabase") - .tableName("aRandomTable") - .partitionName("event_date=2020-01-01/event_hour=0/event_type=A") - .housekeepingStatus(SCHEDULED) - .creationTimestamp(CREATION_TIMESTAMP) - .modifiedTimestamp(CREATION_TIMESTAMP) - .cleanupDelay(Duration.parse("P3D")) - .cleanupAttempts(0) - .lifecycleType(EXPIRED.toString()) - .build(); - table2 = new HousekeepingMetadata.Builder() - .path("s3://some/path2/") - .databaseName("aRandomDatabase2") - .tableName("aRandomTable2") - .partitionName("event_date=2020-01-012/event_hour=2/event_type=B") - .housekeepingStatus(DELETED) - .creationTimestamp(CREATION_TIMESTAMP) - .modifiedTimestamp(CREATION_TIMESTAMP) - .cleanupDelay(Duration.parse("P3D")) - .cleanupAttempts(0) - .lifecycleType(EXPIRED.toString()) - .build(); - } - @Test - public void test(){ - - List tables = new ArrayList(); - tables.add(table1); - tables.add(table2); - when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(new PageImpl<>(tables)); - Page result = housekeepingMetadataServiceImpl.returnAllTables(spec,pageable); - - assertThat(new PageImpl<>(tables)).isEqualTo(result); + public void getAllTest(){ + HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); + when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); + Page result = housekeepingMetadataServiceImpl.getAll(spec,pageable); + + assertThat(tables).isEqualTo(result); } From 021d949b44e46c8a629e80ac2e09de7c419b1366 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 8 Apr 2021 14:51:41 +0100 Subject: [PATCH 031/115] fixing test class --- .../api/HouseKeepingMetadataServiceImplTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java index 5d43ff69..c1cbc2ce 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java @@ -22,6 +22,7 @@ import java.util.List; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -37,8 +38,6 @@ @ExtendWith(MockitoExtension.class) public class HouseKeepingMetadataServiceImplTest { - private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; - @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @Mock @@ -46,13 +45,21 @@ public class HouseKeepingMetadataServiceImplTest { @Mock private Pageable pageable; + private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + + @BeforeEach + public void beforeEach() { + housekeepingMetadataServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); + } + @Test public void getAllTest(){ HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); - Page result = housekeepingMetadataServiceImpl.getAll(spec,pageable); + Page result = housekeepingMetadataServiceImpl.getAll(spec, pageable); assertThat(tables).isEqualTo(result); From eed7b8403df377a8ad3070744190f6b12ee5322a Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 8 Apr 2021 14:59:33 +0100 Subject: [PATCH 032/115] renaming method --- .../beekeeper/api/HouseKeepingMetadataServiceImplTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java index c1cbc2ce..b6ccc6df 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java @@ -38,6 +38,8 @@ @ExtendWith(MockitoExtension.class) public class HouseKeepingMetadataServiceImplTest { + private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @Mock @@ -45,15 +47,13 @@ public class HouseKeepingMetadataServiceImplTest { @Mock private Pageable pageable; - private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; - @BeforeEach public void beforeEach() { housekeepingMetadataServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); } @Test - public void getAllTest(){ + public void testGetAllWhenTablesValid(){ HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); @@ -62,7 +62,6 @@ public void getAllTest(){ Page result = housekeepingMetadataServiceImpl.getAll(spec, pageable); assertThat(tables).isEqualTo(result); - } } From c37894046c9ca872e1f54276f77132d29816580b Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 8 Apr 2021 15:06:22 +0100 Subject: [PATCH 033/115] adding verification mocks --- .../beekeeper/api/HouseKeepingMetadataServiceImplTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java index b6ccc6df..6d6ef3ef 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java @@ -17,7 +17,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; - +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static com.expediagroup.beekeeper.api.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; @@ -62,6 +64,8 @@ public void testGetAllWhenTablesValid(){ Page result = housekeepingMetadataServiceImpl.getAll(spec, pageable); assertThat(tables).isEqualTo(result); + verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); + verifyNoMoreInteractions(housekeepingMetadataRepository); } } From 3dea0bfb7736e9f2fa53b9cb3d93bfe1cc3e8f50 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 8 Apr 2021 16:52:04 +0100 Subject: [PATCH 034/115] impl class --- .../beekeeper/api/BeekeeperController.java | 4 ++-- .../api/BeekeeperControllerTest.java | 20 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index 9785b359..fc105af9 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -29,11 +29,11 @@ @RestController public class BeekeeperController { - private HousekeepingMetadataService housekeepingMetadataService; + private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; @GetMapping("/tables") public ResponseEntity> getAll(Specification spec, Pageable pageable) { - return ResponseEntity.ok(housekeepingMetadataService.getAll(spec, pageable)); + return ResponseEntity.ok(housekeepingMetadataServiceImpl.getAll(spec, pageable)); } } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index 09ce9ac2..1ce2b64b 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -24,6 +24,7 @@ import static com.expediagroup.beekeeper.api.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -39,6 +40,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(MockitoExtension.class) public class BeekeeperControllerTest { @@ -48,23 +50,31 @@ public class BeekeeperControllerTest { @Autowired private ObjectMapper objectMapper; + @Mock + private HousekeepingMetadataRepository housekeepingMetadataRepository; @Mock private Specification spec; @Mock private Pageable pageable; - @Mock - private HousekeepingMetadataService housekeepingMetadataService; + + private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + + + @BeforeEach + public void beforeEach() { + housekeepingMetadataServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); + } @Test - public void getAllTest() throws Exception { + public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataService.getAll(spec, pageable)).thenReturn(tables); + when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); mockMvc - .perform(get(housekeepingMetadataService.getAll(spec, pageable).toString())) + .perform(get(housekeepingMetadataServiceImpl.getAll(spec, pageable).toString())) .andDo(MockMvcResultHandlers.print()) .andExpect(status().isOk()) .andExpect(content().contentType(APPLICATION_JSON)) From 72fca3f91e1525136d10ddcd8a6fa18fb764bcbe Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 9 Apr 2021 11:09:47 +0100 Subject: [PATCH 035/115] add filtering in controller --- .../beekeeper/api/BeekeeperController.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index fc105af9..e1856dc8 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -15,6 +15,8 @@ */ package com.expediagroup.beekeeper.api; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; @@ -23,6 +25,10 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import net.kaczmarzyk.spring.data.jpa.domain.EqualIgnoreCase; +import net.kaczmarzyk.spring.data.jpa.web.annotation.And; +import net.kaczmarzyk.spring.data.jpa.web.annotation.Spec; + import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @RequestMapping("/api/v1") @@ -31,8 +37,13 @@ public class BeekeeperController { private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; - @GetMapping("/tables") - public ResponseEntity> getAll(Specification spec, Pageable pageable) { + @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE, params = {"tableName", "databaseName"}) + public ResponseEntity> getAll( + @And({ + @Spec(path = "tableName", spec = EqualIgnoreCase.class), + @Spec(path = "databaseName", spec = EqualIgnoreCase.class) + }) + Specification spec, Pageable pageable) { return ResponseEntity.ok(housekeepingMetadataServiceImpl.getAll(spec, pageable)); } From a325002fcb13c806e684d3eb167be349ab95644f Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 9 Apr 2021 11:55:52 +0100 Subject: [PATCH 036/115] adding specs --- .../expediagroup/beekeeper/api/BeekeeperController.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java index e1856dc8..7e162abc 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java @@ -26,6 +26,8 @@ import org.springframework.web.bind.annotation.RestController; import net.kaczmarzyk.spring.data.jpa.domain.EqualIgnoreCase; +import net.kaczmarzyk.spring.data.jpa.domain.GreaterThan; +import net.kaczmarzyk.spring.data.jpa.domain.LessThan; import net.kaczmarzyk.spring.data.jpa.web.annotation.And; import net.kaczmarzyk.spring.data.jpa.web.annotation.Spec; @@ -37,11 +39,14 @@ public class BeekeeperController { private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; - @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE, params = {"tableName", "databaseName"}) + @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE) public ResponseEntity> getAll( @And({ @Spec(path = "tableName", spec = EqualIgnoreCase.class), - @Spec(path = "databaseName", spec = EqualIgnoreCase.class) + @Spec(path = "databaseName", spec = EqualIgnoreCase.class), + @Spec(path = "status", spec = EqualIgnoreCase.class), + @Spec(path = "deletedBefore", spec = LessThan.class), + @Spec(path = "deletedAfter", spec = GreaterThan.class) }) Specification spec, Pageable pageable) { return ResponseEntity.ok(housekeepingMetadataServiceImpl.getAll(spec, pageable)); From 8a865ec71eca75fd37caee0d34662807ca5fd392 Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 9 Apr 2021 12:01:29 +0100 Subject: [PATCH 037/115] idk --- .../expediagroup/beekeeper/api/BeekeeperControllerTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java index 1ce2b64b..c6b716b3 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java @@ -15,6 +15,9 @@ */ package com.expediagroup.beekeeper.api; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static org.springframework.http.MediaType.APPLICATION_JSON; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -79,5 +82,7 @@ public void testGetAllWhenTablesValid() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(APPLICATION_JSON)) .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); + verifyNoMoreInteractions(housekeepingMetadataRepository); } } From 4796feac54b3a9e331934706b33769f6ab33e767 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 12 Apr 2021 10:55:19 +0100 Subject: [PATCH 038/115] doesn't work --- beekeeper-api/pom.xml | 6 ++++++ .../beekeeper/api/conf/JPAConfiguration.java | 21 +++++++++++++++++++ .../{ => controller}/BeekeeperController.java | 3 ++- .../HousekeepingMetadataService.java | 2 +- .../HousekeepingMetadataServiceImpl.java | 3 ++- .../beekeeper/api/TestApplication.java | 14 +++++++++++++ .../BeekeeperControllerTest.java | 15 ++++++++----- .../HouseKeepingMetadataServiceImplTest.java | 7 ++++--- .../DummyHousekeepingMetadataGenerator.java | 18 ++++++++++++++-- 9 files changed, 76 insertions(+), 13 deletions(-) create mode 100644 beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{ => controller}/BeekeeperController.java (94%) rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{ => service}/HousekeepingMetadataService.java (95%) rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{ => service}/HousekeepingMetadataServiceImpl.java (92%) create mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{ => controller}/BeekeeperControllerTest.java (85%) rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{ => service}/HouseKeepingMetadataServiceImplTest.java (93%) rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{ => util}/DummyHousekeepingMetadataGenerator.java (68%) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index de585f1c..2dff8a8f 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -52,6 +52,12 @@ spring-boot-test test + + org.springframework.boot + spring-boot-test-autoconfigure + 2.2.3.RELEASE + test + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java new file mode 100644 index 00000000..68896d60 --- /dev/null +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java @@ -0,0 +1,21 @@ +package com.expediagroup.beekeeper.api.conf; + +import java.util.List; + +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.web.method.support.HandlerMethodArgumentResolver; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import net.kaczmarzyk.spring.data.jpa.web.SpecificationArgumentResolver; + +@Configuration +@EnableJpaRepositories +public class JPAConfiguration implements WebMvcConfigurer { + + @Override + public void addArgumentResolvers(List argumentResolvers) { + argumentResolvers.add(new SpecificationArgumentResolver()); + } +} + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java similarity index 94% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 7e162abc..f2231bdb 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api; +package com.expediagroup.beekeeper.api.controller; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; @@ -31,6 +31,7 @@ import net.kaczmarzyk.spring.data.jpa.web.annotation.And; import net.kaczmarzyk.spring.data.jpa.web.annotation.Spec; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataServiceImpl; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @RequestMapping("/api/v1") diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java similarity index 95% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java index 25d7ad8b..0c5e2dde 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api; +package com.expediagroup.beekeeper.api.service; import org.springframework.data.domain.Page; diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java similarity index 92% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java index b27893f0..357a8152 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/HousekeepingMetadataServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api; +package com.expediagroup.beekeeper.api.service; import org.springframework.beans.factory.annotation.Autowired; @@ -22,6 +22,7 @@ import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Service; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java new file mode 100644 index 00000000..1ec1afd9 --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java @@ -0,0 +1,14 @@ +package com.expediagroup.beekeeper.api; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; + +@SpringBootApplication +@EnableConfigurationProperties +@ComponentScan(basePackages = { + "com.expediagroup.beekeeper.api.conf", + "com.expediagroup.beekeeper.api.controller" }) +public class TestApplication { + +} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java similarity index 85% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index c6b716b3..695bbbd3 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api; +package com.expediagroup.beekeeper.api.controller; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -24,28 +24,33 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static com.expediagroup.beekeeper.api.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; +import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; + import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataServiceImpl; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; -@ExtendWith(MockitoExtension.class) +@ExtendWith(SpringExtension.class) +//@ExtendWith(MockitoExtension.class) +@WebMvcTest(BeekeeperController.class) public class BeekeeperControllerTest { @Autowired @@ -77,7 +82,7 @@ public void testGetAllWhenTablesValid() throws Exception { when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); mockMvc - .perform(get(housekeepingMetadataServiceImpl.getAll(spec, pageable).toString())) + .perform(get("/api/v1/tables")) .andDo(MockMvcResultHandlers.print()) .andExpect(status().isOk()) .andExpect(content().contentType(APPLICATION_JSON)) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceImplTest.java similarity index 93% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceImplTest.java index 6d6ef3ef..0cd2c987 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/HouseKeepingMetadataServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceImplTest.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api; +package com.expediagroup.beekeeper.api.service; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static com.expediagroup.beekeeper.api.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; +import static org.mockito.Mockito.when; + +import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/DummyHousekeepingMetadataGenerator.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java similarity index 68% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/DummyHousekeepingMetadataGenerator.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java index d37d3c92..e6799d57 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/DummyHousekeepingMetadataGenerator.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java @@ -1,6 +1,20 @@ -package com.expediagroup.beekeeper.api; +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.util; -import static java.lang.String.format; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; From e6cbe4e9af3f651863ae3747c4bf45603f57c60d Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 12 Apr 2021 10:56:52 +0100 Subject: [PATCH 039/115] idk whats going on --- .../core/repository/HousekeepingMetadataRepository.java | 3 +-- .../core/repository/HousekeepingMetadataRepositoryTest.java | 2 +- .../cleanup/service/PagingMetadataCleanupServiceTest.java | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index b0470092..057309eb 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 5e5871e6..ece68903 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java index b2379bbf..1b6ab9f0 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -210,7 +210,7 @@ void metadataCleanerException() { .extracting("tableName") .containsExactly(tables.get(0).getTableName(), tables.get(1).getTableName()); - List result = metadataRepository.findAll(); + List result = (List) metadataRepository.findAll(); assertThat(result.size()).isEqualTo(2); HousekeepingMetadata housekeepingMetadata1 = result.get(0); HousekeepingMetadata housekeepingMetadata2 = result.get(1); From 9294f26763a896f1c50d12da44b93f8691b4e56a Mon Sep 17 00:00:00 2001 From: vedantchokshi Date: Mon, 12 Apr 2021 11:35:58 +0100 Subject: [PATCH 040/115] Fix --- beekeeper-api/pom.xml | 39 ++++++++----------- ...rApi.java => BeekeeperApiApplication.java} | 16 ++++++-- .../beekeeper/api/conf/JPAConfiguration.java | 16 +++++++- .../HousekeepingMetadataServiceImpl.java | 1 - .../beekeeper/api/TestApplication.java | 14 ------- .../controller/BeekeeperControllerTest.java | 1 - .../HousekeepingMetadataRepository.java | 3 +- .../HousekeepingMetadataRepositoryTest.java | 2 +- pom.xml | 2 +- 9 files changed, 47 insertions(+), 47 deletions(-) rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/{BeekeeperApi.java => BeekeeperApiApplication.java} (52%) delete mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 2dff8a8f..f6113edc 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -11,6 +11,10 @@ beekeeper-api + + 2.6.1 + + org.springframework.boot @@ -28,35 +32,24 @@ net.kaczmarzyk specification-arg-resolver - 2.6.1 - - - com.graphql-java - graphql-spring-boot-starter - 5.0.2 + ${specification-arg-resolver.version} - - org.assertj - assertj-core - test - - - org.springframework - spring-test - test - - - org.springframework.boot - spring-boot-test - test - org.springframework.boot - spring-boot-test-autoconfigure - 2.2.3.RELEASE + spring-boot-starter-test test + + + junit-jupiter + org.junit.jupiter + + + com.vaadin.external.google + android-json + + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApi.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java similarity index 52% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApi.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java index 22bbf7f1..3763dc74 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApi.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java @@ -17,12 +17,22 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; @SpringBootApplication -public class BeekeeperApi { +@EnableConfigurationProperties +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +@ComponentScan(basePackages = { + "com.expediagroup.beekeeper.api.conf", + "com.expediagroup.beekeeper.api.controller", + "com.expediagroup.beekeeper.api.service" }) +public class BeekeeperApiApplication { public static void main(String[] args) { - SpringApplication.run(BeekeeperApi.class, args); + SpringApplication.run(BeekeeperApiApplication.class, args); } - } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java index 68896d60..dd7f1338 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java @@ -1,3 +1,18 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.api.conf; import java.util.List; @@ -18,4 +33,3 @@ public void addArgumentResolvers(List argumentRes argumentResolvers.add(new SpecificationArgumentResolver()); } } - diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java index 357a8152..521ba60e 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java @@ -22,7 +22,6 @@ import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Service; -import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java deleted file mode 100644 index 1ec1afd9..00000000 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.expediagroup.beekeeper.api; - -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.ComponentScan; - -@SpringBootApplication -@EnableConfigurationProperties -@ComponentScan(basePackages = { - "com.expediagroup.beekeeper.api.conf", - "com.expediagroup.beekeeper.api.controller" }) -public class TestApplication { - -} diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 695bbbd3..24ef2d1f 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -48,7 +48,6 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; -@ExtendWith(SpringExtension.class) //@ExtendWith(MockitoExtension.class) @WebMvcTest(BeekeeperController.class) public class BeekeeperControllerTest { diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index b0470092..057309eb 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index 5e5871e6..ece68903 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pom.xml b/pom.xml index 9130d6cf..26b160f2 100644 --- a/pom.xml +++ b/pom.xml @@ -21,9 +21,9 @@ beekeeper-path-cleanup beekeeper-scheduler beekeeper-scheduler-apiary + beekeeper-api beekeeper-integration-tests beekeeper-vacuum-tool - beekeeper-api From baed7edc435fe174635b5f6076a52d7bbf684e90 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 12 Apr 2021 14:34:27 +0100 Subject: [PATCH 041/115] stuff --- beekeeper-api/pom.xml | 6 ------ .../beekeeper/api/conf/JPAConfiguration.java | 16 +++++++++++++++- .../api/controller/BeekeeperController.java | 8 +++++++- .../HousekeepingMetadataServiceImpl.java | 1 - .../beekeeper/api/TestApplication.java | 18 +++++++++++++++++- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 2dff8a8f..a9d3116a 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -30,11 +30,6 @@ specification-arg-resolver 2.6.1 - - com.graphql-java - graphql-spring-boot-starter - 5.0.2 - @@ -55,7 +50,6 @@ org.springframework.boot spring-boot-test-autoconfigure - 2.2.3.RELEASE test diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java index 68896d60..dd7f1338 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/JPAConfiguration.java @@ -1,3 +1,18 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.api.conf; import java.util.List; @@ -18,4 +33,3 @@ public void addArgumentResolvers(List argumentRes argumentResolvers.add(new SpecificationArgumentResolver()); } } - diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index f2231bdb..9f373bf9 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -17,6 +17,7 @@ import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; @@ -38,7 +39,12 @@ @RestController public class BeekeeperController { - private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + private final HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + + @Autowired + public BeekeeperController(HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl) { + this.housekeepingMetadataServiceImpl = housekeepingMetadataServiceImpl; + } @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE) public ResponseEntity> getAll( diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java index 357a8152..521ba60e 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java @@ -22,7 +22,6 @@ import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Service; -import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java index 1ec1afd9..86ce8374 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java @@ -1,13 +1,29 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.api; +import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; @SpringBootApplication +@SpringBootConfiguration @EnableConfigurationProperties @ComponentScan(basePackages = { - "com.expediagroup.beekeeper.api.conf", "com.expediagroup.beekeeper.api.controller" }) public class TestApplication { From bc0dbd1b987486ab90312a7f40d3dcc0d6c2d627 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 12 Apr 2021 16:07:40 +0100 Subject: [PATCH 042/115] works without the mockmvc and verifications --- .../controller/BeekeeperControllerTest.java | 31 +++++++------------ .../api/{ => controller}/TestApplication.java | 3 +- 2 files changed, 13 insertions(+), 21 deletions(-) rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{ => controller}/TestApplication.java (91%) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 24ef2d1f..5799a0dd 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -15,14 +15,6 @@ */ package com.expediagroup.beekeeper.api.controller; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.springframework.http.MediaType.APPLICATION_JSON; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; @@ -30,17 +22,15 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; @@ -64,6 +54,7 @@ public class BeekeeperControllerTest { @Mock private Pageable pageable; + @MockBean private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; @@ -78,15 +69,15 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); +// when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); - mockMvc - .perform(get("/api/v1/tables")) - .andDo(MockMvcResultHandlers.print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(APPLICATION_JSON)) - .andExpect(content().json(objectMapper.writeValueAsString(tables))); - verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); - verifyNoMoreInteractions(housekeepingMetadataRepository); +// mockMvc +// .perform(get("/api/v1/tables")) +// .andDo(MockMvcResultHandlers.print()) +// .andExpect(status().isOk()) +// .andExpect(content().contentType(APPLICATION_JSON)) +// .andExpect(content().json(objectMapper.writeValueAsString(tables))); + // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); + // verifyNoMoreInteractions(housekeepingMetadataRepository); } } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java similarity index 91% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java index 86ce8374..c27044d6 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api; +package com.expediagroup.beekeeper.api.controller; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -24,6 +24,7 @@ @SpringBootConfiguration @EnableConfigurationProperties @ComponentScan(basePackages = { + "com.expediagroup.beekeeper.api.config", "com.expediagroup.beekeeper.api.controller" }) public class TestApplication { From 61e123eb75590466bbf6d30235139129229fcbe1 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 12 Apr 2021 16:58:14 +0100 Subject: [PATCH 043/115] reproducing error --- beekeeper-api/pom.xml | 6 ++++ .../api/controller/BeekeeperController.java | 4 +-- .../controller/BeekeeperControllerTest.java | 28 +++++++++++++++++-- .../api/controller/TestApplication.java | 2 +- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index f6113edc..7c90614d 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -51,6 +51,12 @@ + + co.cask.tephra + tephra-core + 0.6.0 + test + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 9f373bf9..fed74e68 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -15,12 +15,12 @@ */ package com.expediagroup.beekeeper.api.controller; -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -46,7 +46,7 @@ public BeekeeperController(HousekeepingMetadataServiceImpl housekeepingMetadataS this.housekeepingMetadataServiceImpl = housekeepingMetadataServiceImpl; } - @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE) + @GetMapping(path = "/tables", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> getAll( @And({ @Spec(path = "tableName", spec = EqualIgnoreCase.class), diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 5799a0dd..eda741f5 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -16,6 +16,16 @@ package com.expediagroup.beekeeper.api.controller; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.http.MediaType.APPLICATION_JSON; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; @@ -31,6 +41,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; @@ -69,7 +80,7 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); -// when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); + when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); // mockMvc // .perform(get("/api/v1/tables")) @@ -77,7 +88,18 @@ public void testGetAllWhenTablesValid() throws Exception { // .andExpect(status().isOk()) // .andExpect(content().contentType(APPLICATION_JSON)) // .andExpect(content().json(objectMapper.writeValueAsString(tables))); - // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); - // verifyNoMoreInteractions(housekeepingMetadataRepository); +// verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); +// verifyNoMoreInteractions(housekeepingMetadataRepository); + System.out.println("AAA"+MockMvcResultHandlers.print().toString()); + + mockMvc + .perform(get("/api/v1/tables").contentType(APPLICATION_JSON)) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(header().string("Content-Type", APPLICATION_JSON.toString())) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + + verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); + verifyNoMoreInteractions(housekeepingMetadataRepository); } } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java index c27044d6..7fb53875 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java @@ -24,7 +24,7 @@ @SpringBootConfiguration @EnableConfigurationProperties @ComponentScan(basePackages = { - "com.expediagroup.beekeeper.api.config", + "com.expediagroup.beekeeper.api.conf", "com.expediagroup.beekeeper.api.controller" }) public class TestApplication { From dc173f36e65d9b1a574346a57ed7d3f6a2c1edae Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 12 Apr 2021 17:01:16 +0100 Subject: [PATCH 044/115] reproducing error --- .../beekeeper/api/controller/BeekeeperController.java | 5 +++-- .../beekeeper/api/controller/BeekeeperControllerTest.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index fed74e68..5b63b46e 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -20,7 +20,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import org.springframework.http.MediaType; + +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -46,7 +47,7 @@ public BeekeeperController(HousekeepingMetadataServiceImpl housekeepingMetadataS this.housekeepingMetadataServiceImpl = housekeepingMetadataServiceImpl; } - @GetMapping(path = "/tables", produces = MediaType.APPLICATION_JSON_VALUE) + @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE) public ResponseEntity> getAll( @And({ @Spec(path = "tableName", spec = EqualIgnoreCase.class), diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index eda741f5..910cba81 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -26,6 +26,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; @@ -90,7 +91,6 @@ public void testGetAllWhenTablesValid() throws Exception { // .andExpect(content().json(objectMapper.writeValueAsString(tables))); // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); // verifyNoMoreInteractions(housekeepingMetadataRepository); - System.out.println("AAA"+MockMvcResultHandlers.print().toString()); mockMvc .perform(get("/api/v1/tables").contentType(APPLICATION_JSON)) From 598edbac8a17a52db9b8a26cf0c0970eee73ded5 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 13 Apr 2021 14:09:36 +0100 Subject: [PATCH 045/115] still doesn't work, utf8 error --- .../api/controller/BeekeeperController.java | 5 ++- .../controller/BeekeeperControllerTest.java | 37 ++++++------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 5b63b46e..79a0dca4 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -16,15 +16,16 @@ package com.expediagroup.beekeeper.api.controller; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; - -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import net.kaczmarzyk.spring.data.jpa.domain.EqualIgnoreCase; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 910cba81..b17cab37 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -22,17 +22,18 @@ import static org.mockito.Mockito.when; import static org.springframework.http.MediaType.APPLICATION_JSON; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - - +import static org.mockito.ArgumentMatchers.any; import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; @@ -41,6 +42,8 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.MediaType; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.result.MockMvcResultHandlers; @@ -50,7 +53,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; -//@ExtendWith(MockitoExtension.class) +@ExtendWith(SpringExtension.class) @WebMvcTest(BeekeeperController.class) public class BeekeeperControllerTest { @@ -59,8 +62,6 @@ public class BeekeeperControllerTest { @Autowired private ObjectMapper objectMapper; - @Mock - private HousekeepingMetadataRepository housekeepingMetadataRepository; @Mock private Specification spec; @Mock @@ -69,37 +70,21 @@ public class BeekeeperControllerTest { @MockBean private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; - - @BeforeEach - public void beforeEach() { - housekeepingMetadataServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); - } - @Test public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); - -// mockMvc -// .perform(get("/api/v1/tables")) -// .andDo(MockMvcResultHandlers.print()) -// .andExpect(status().isOk()) -// .andExpect(content().contentType(APPLICATION_JSON)) -// .andExpect(content().json(objectMapper.writeValueAsString(tables))); -// verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); -// verifyNoMoreInteractions(housekeepingMetadataRepository); + when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); mockMvc - .perform(get("/api/v1/tables").contentType(APPLICATION_JSON)) + .perform(get("/api/v1/tables")) .andDo(MockMvcResultHandlers.print()) .andExpect(status().isOk()) - .andExpect(header().string("Content-Type", APPLICATION_JSON.toString())) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) .andExpect(content().json(objectMapper.writeValueAsString(tables))); - - verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); - verifyNoMoreInteractions(housekeepingMetadataRepository); + // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); + //verifyNoMoreInteractions(housekeepingMetadataRepository); } } From 896e506af76adb42b05c7822ee7c5e312b4f5474 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 13 Apr 2021 15:06:15 +0100 Subject: [PATCH 046/115] compiles correctly without test --- .../api/controller/BeekeeperController.java | 1 - .../controller/BeekeeperControllerTest.java | 31 +++++-------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 79a0dca4..0407162f 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -25,7 +25,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import net.kaczmarzyk.spring.data.jpa.domain.EqualIgnoreCase; diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index b17cab37..7b650802 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -16,22 +16,10 @@ package com.expediagroup.beekeeper.api.controller; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.springframework.http.MediaType.APPLICATION_JSON; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.mockito.ArgumentMatchers.any; import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -42,16 +30,13 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import org.springframework.http.MediaType; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; import com.expediagroup.beekeeper.api.service.HousekeepingMetadataServiceImpl; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(SpringExtension.class) @WebMvcTest(BeekeeperController.class) @@ -76,14 +61,14 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); - - mockMvc - .perform(get("/api/v1/tables")) - .andDo(MockMvcResultHandlers.print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) - .andExpect(content().json(objectMapper.writeValueAsString(tables))); +// when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); +// +// mockMvc +// .perform(get("/api/v1/tables")) +// .andDo(MockMvcResultHandlers.print()) +// .andExpect(status().isOk()) +// .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) +// .andExpect(content().json(objectMapper.writeValueAsString(tables))); // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); //verifyNoMoreInteractions(housekeepingMetadataRepository); } From 3cc729f3c367d6580696ccbc7717d103aa94da4f Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 13 Apr 2021 15:34:25 +0100 Subject: [PATCH 047/115] compiles fine --- beekeeper-core/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 2b128307..1451d71c 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -11,8 +11,8 @@ beekeeper-core - 1.1.4 - 1.3.3 + 1.6.5 + 1.6.5 From 8de50709bf9645a5e25c7fe09c5a15f9f56b53ff Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 13 Apr 2021 16:04:18 +0100 Subject: [PATCH 048/115] compiles fine --- .../controller/BeekeeperControllerTest.java | 24 ++++++++++++------- pom.xml | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 7b650802..b454a855 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -16,6 +16,12 @@ package com.expediagroup.beekeeper.api.controller; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; @@ -30,8 +36,10 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.MediaType; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; @@ -61,14 +69,14 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); -// when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); -// -// mockMvc -// .perform(get("/api/v1/tables")) -// .andDo(MockMvcResultHandlers.print()) -// .andExpect(status().isOk()) -// .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) -// .andExpect(content().json(objectMapper.writeValueAsString(tables))); + when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); //verifyNoMoreInteractions(housekeepingMetadataRepository); } diff --git a/pom.xml b/pom.xml index 26b160f2..ca232105 100644 --- a/pom.xml +++ b/pom.xml @@ -43,10 +43,10 @@ 5.6.0 1.2.3 3.1.0 - 3.2.4 + 3.9.0 1.24 2.1.3.RELEASE - 5.1.5.RELEASE + 5.3.5 1.15.2 11-slim From b17cb110e4b0e53fb6e0762824ca850baf7e6cf4 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 13 Apr 2021 16:54:54 +0100 Subject: [PATCH 049/115] ACTUALLY compiles correctyl --- .../controller/BeekeeperControllerTest.java | 23 +++++++------------ beekeeper-core/pom.xml | 4 ++-- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index b454a855..b67232b6 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -16,11 +16,6 @@ package com.expediagroup.beekeeper.api.controller; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; @@ -36,10 +31,8 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import org.springframework.http.MediaType; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; @@ -69,14 +62,14 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); - - mockMvc - .perform(get("/api/v1/tables")) - .andDo(MockMvcResultHandlers.print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) - .andExpect(content().json(objectMapper.writeValueAsString(tables))); +// when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); +// +// mockMvc +// .perform(get("/api/v1/tables")) +// .andDo(MockMvcResultHandlers.print()) +// .andExpect(status().isOk()) +// .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) +// .andExpect(content().json(objectMapper.writeValueAsString(tables))); // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); //verifyNoMoreInteractions(housekeepingMetadataRepository); } diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 1451d71c..2b128307 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -11,8 +11,8 @@ beekeeper-core - 1.6.5 - 1.6.5 + 1.1.4 + 1.3.3 From c424e7c86d752d3330858153f8a22f0739bc4d98 Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 15 Apr 2021 11:00:16 +0100 Subject: [PATCH 050/115] reproducing error --- .../beekeeper/api/controller/BeekeeperControllerTest.java | 5 +++-- beekeeper-core/pom.xml | 4 ++-- pom.xml | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index b67232b6..5b69949c 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -17,6 +17,7 @@ + import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; @@ -70,7 +71,7 @@ public void testGetAllWhenTablesValid() throws Exception { // .andExpect(status().isOk()) // .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) // .andExpect(content().json(objectMapper.writeValueAsString(tables))); - // verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); - //verifyNoMoreInteractions(housekeepingMetadataRepository); +// verify(housekeepingMetadataServiceImpl, times(1)).getAll(spec, pageable); +// verifyNoMoreInteractions(housekeepingMetadataServiceImpl); } } diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 2b128307..1451d71c 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -11,8 +11,8 @@ beekeeper-core - 1.1.4 - 1.3.3 + 1.6.5 + 1.6.5 diff --git a/pom.xml b/pom.xml index ca232105..9048a815 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ 3.1.0 3.9.0 1.24 - 2.1.3.RELEASE + 2.4.4 5.3.5 1.15.2 11-slim From 500df9b4f226bfe7c68f93cb2f6029479079324b Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 15 Apr 2021 15:17:30 +0100 Subject: [PATCH 051/115] some of it works --- .../controller/BeekeeperControllerTest.java | 32 ++++++++++++------- beekeeper-integration-tests/pom.xml | 28 ++++++++++++++++ .../path/cleanup/BeekeeperPathCleanup.java | 6 +++- beekeeper-scheduler-apiary/pom.xml | 12 +++++++ .../apiary/BeekeeperSchedulerApiary.java | 4 +-- 5 files changed, 67 insertions(+), 15 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 5b69949c..fdeabcf3 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -15,8 +15,14 @@ */ package com.expediagroup.beekeeper.api.controller; - - +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; @@ -32,8 +38,10 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.MediaType; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; @@ -63,15 +71,15 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); -// when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); -// -// mockMvc -// .perform(get("/api/v1/tables")) -// .andDo(MockMvcResultHandlers.print()) -// .andExpect(status().isOk()) -// .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) -// .andExpect(content().json(objectMapper.writeValueAsString(tables))); -// verify(housekeepingMetadataServiceImpl, times(1)).getAll(spec, pageable); -// verifyNoMoreInteractions(housekeepingMetadataServiceImpl); + when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataServiceImpl, times(1)).getAll(spec, pageable); + verifyNoMoreInteractions(housekeepingMetadataServiceImpl); } } diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 47e03f33..89812a1b 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -52,6 +52,10 @@ org.apache.hive hive-service-rpc + + org.mortbay.jetty + jetty-util + @@ -73,6 +77,18 @@ slf4j-log4j12 org.slf4j + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-sslengine + @@ -116,6 +132,18 @@ log4j-slf4j-impl org.apache.logging.log4j + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + jetty + + + org.eclipse.jetty.aggregate + jetty-all + diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java index 1e569532..99230291 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java @@ -24,6 +24,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.PropertySource; import io.micrometer.core.instrument.MeterRegistry; @@ -38,9 +39,12 @@ public class BeekeeperPathCleanup implements ApplicationContextAware { public static void main(String[] args) { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + +// Srring additionalConfigs = + new SpringApplicationBuilder(BeekeeperPathCleanup.class) .properties( - "spring.config.additional-location:classpath:/beekeeper-path-cleanup-application.yml,${config:null}") + "spring.config.additional-location:classpath:/beekeeper-path-cleanup-application.yml,${config:/}") .build() .run(args); } diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml index 4dc0c8c3..8e867a46 100644 --- a/beekeeper-scheduler-apiary/pom.xml +++ b/beekeeper-scheduler-apiary/pom.xml @@ -50,6 +50,18 @@ org.apache.geronimo.specs geronimo-jaspic_1.0_spec + + org.mortbay.jetty + jetty + + + org.eclipse.jetty.aggregate + jetty-all + + + org.mortbay.jetty + jetty-util + diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java index d859d5be..6e5734ca 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java @@ -24,6 +24,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.PropertySource; import io.micrometer.core.instrument.MeterRegistry; @@ -42,8 +43,7 @@ public class BeekeeperSchedulerApiary implements ApplicationContextAware { public static void main(String[] args) { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); new SpringApplicationBuilder(BeekeeperSchedulerApiary.class) - .properties("spring.config.additional-location:classpath:/beekeeper-scheduler-apiary-application.yml," - + "${config:null}") + .properties("spring.config.additional-location:classpath:/beekeeper-scheduler-apiary-application.yml") .build() .run(args); } From 226642ac95bcdd77a60e2ff7fb9800d71720a104 Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 16 Apr 2021 12:37:30 +0100 Subject: [PATCH 052/115] improvement --- .../controller/BeekeeperControllerTest.java | 30 +++++++------------ .../BeekeeperIntegrationTestBase.java | 6 +++- ...cedPathSchedulerApiaryIntegrationTest.java | 6 ++-- .../integration/utils/MySqlTestUtils.java | 13 ++++++-- .../cleanup/BeekeeperMetadataCleanup.java | 2 +- .../path/cleanup/BeekeeperPathCleanup.java | 3 -- .../apiary/BeekeeperSchedulerApiary.java | 3 +- 7 files changed, 30 insertions(+), 33 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index fdeabcf3..d122b4ec 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -15,14 +15,6 @@ */ package com.expediagroup.beekeeper.api.controller; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; @@ -38,10 +30,8 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import org.springframework.http.MediaType; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; @@ -71,15 +61,15 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); - - mockMvc - .perform(get("/api/v1/tables")) - .andDo(MockMvcResultHandlers.print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) - .andExpect(content().json(objectMapper.writeValueAsString(tables))); - verify(housekeepingMetadataServiceImpl, times(1)).getAll(spec, pageable); - verifyNoMoreInteractions(housekeepingMetadataServiceImpl); +// when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); +// +// mockMvc +// .perform(get("/api/v1/tables")) +// .andDo(MockMvcResultHandlers.print()) +// .andExpect(status().isOk()) +// .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) +// .andExpect(content().json(objectMapper.writeValueAsString(tables))); +// verify(housekeepingMetadataServiceImpl, times(1)).getAll(spec, pageable); +// verifyNoMoreInteractions(housekeepingMetadataServiceImpl); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 39c0b5d3..885685f1 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -181,6 +181,10 @@ protected void insertExpiredMetadata(String tableName, String path, String parti } protected int getUnreferencedPathsRowCount() throws SQLException { + System.out.println(BEEKEEPER_DB_NAME); + System.out.println(BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME); + System.out.println(LIFE_CYCLE_FILTER); + System.out.println(UNREFERENCED); return mySQLTestUtils .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, format(LIFE_CYCLE_FILTER, UNREFERENCED)); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 1cb03a25..ed4b817d 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -144,8 +144,8 @@ public void unreferencedAlterPartitionEvent() throws SQLException, IOException, alterPartitionSqsMessage.setPartitionLocation("s3://partitionLocation2"); alterPartitionSqsMessage.setOldPartitionLocation("s3://partitionLocation"); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); - + //await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() > 1); List unreferencedPaths = getUnreferencedPaths(); assertUnreferencedPath(unreferencedPaths.get(0), "s3://partitionLocation"); assertUnreferencedPath(unreferencedPaths.get(1), "s3://unreferencedPartitionLocation"); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index 614b884a..e21c04e5 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -19,8 +19,10 @@ import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; public class MySqlTestUtils { @@ -29,6 +31,7 @@ public class MySqlTestUtils { private static final String INSERT_TO_TABLE = "INSERT INTO %s.%s (%s) VALUES (%s);"; private final Connection connection; + private Statement statement2; public MySqlTestUtils(String jdbcUrl, String username, String password) throws SQLException { connection = DriverManager.getConnection(jdbcUrl, username, password); @@ -51,9 +54,13 @@ public int getTableRowCount(String database, String table, String additionalFilt } private int getTableRowCount(String statement) throws SQLException { - ResultSet resultSet = getTableRows(statement); - resultSet.last(); - int rowsInTable = resultSet.getRow(); + //ResultSet resultSet = getTableRows(statement); + statement2 = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, + ResultSet.CONCUR_READ_ONLY); + ResultSet rs = statement2.executeQuery(statement); + System.out.println("type:"+rs.getType()); + rs.last(); + int rowsInTable = rs.getRow(); return rowsInTable; } diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java index aac1ce6a..220f299a 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java @@ -40,7 +40,7 @@ public static void main(String[] args) { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); new SpringApplicationBuilder(BeekeeperMetadataCleanup.class) .properties( - "spring.config.additional-location:classpath:/beekeeper-metadata-cleanup-application.yml,${config:null}") + "spring.config.additional-location:classpath:/beekeeper-metadata-cleanup-application.yml,${config:/}") .build() .run(args); } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java index 99230291..e95db8f2 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java @@ -39,9 +39,6 @@ public class BeekeeperPathCleanup implements ApplicationContextAware { public static void main(String[] args) { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - -// Srring additionalConfigs = - new SpringApplicationBuilder(BeekeeperPathCleanup.class) .properties( "spring.config.additional-location:classpath:/beekeeper-path-cleanup-application.yml,${config:/}") diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java index 6e5734ca..651394fa 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/BeekeeperSchedulerApiary.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.PropertySource; import io.micrometer.core.instrument.MeterRegistry; From dc7a4499ea6c9141945a41ddbd6cb84a9d2c8303 Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 16 Apr 2021 14:20:25 +0100 Subject: [PATCH 053/115] mysqlutils --- .../beekeeper/integration/utils/MySqlTestUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index e21c04e5..6bb02d93 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ import java.sql.Connection; import java.sql.DriverManager; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; From b9f0e6903d3cd34bffa7f51e83257f4b6678a8f9 Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 16 Apr 2021 16:29:49 +0100 Subject: [PATCH 054/115] i think i fixed it --- beekeeper-api/pom.xml | 5 +++ .../controller/BeekeeperControllerTest.java | 31 +++++++++++++------ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 7c90614d..c89aa91d 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -34,6 +34,11 @@ specification-arg-resolver ${specification-arg-resolver.version} + + org.json + json + 20080701 + diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index d122b4ec..5106d434 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -16,6 +16,15 @@ package com.expediagroup.beekeeper.api.controller; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; import java.util.List; @@ -30,8 +39,10 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.MediaType; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; @@ -61,15 +72,15 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); -// when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); -// -// mockMvc -// .perform(get("/api/v1/tables")) -// .andDo(MockMvcResultHandlers.print()) -// .andExpect(status().isOk()) -// .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) -// .andExpect(content().json(objectMapper.writeValueAsString(tables))); -// verify(housekeepingMetadataServiceImpl, times(1)).getAll(spec, pageable); -// verifyNoMoreInteractions(housekeepingMetadataServiceImpl); + when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataServiceImpl, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataServiceImpl); } } From 6afbab8d1ba9e40eadac8ca3c010ed1cb5a71ce1 Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 16 Apr 2021 16:43:33 +0100 Subject: [PATCH 055/115] taking out dependencies and systemoutprintln --- beekeeper-api/pom.xml | 16 ---------------- .../integration/utils/MySqlTestUtils.java | 5 +---- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index c89aa91d..189230ec 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -45,22 +45,6 @@ org.springframework.boot spring-boot-starter-test test - - - junit-jupiter - org.junit.jupiter - - - com.vaadin.external.google - android-json - - - - - co.cask.tephra - tephra-core - 0.6.0 - test diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index 6bb02d93..95fc5a23 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -53,14 +53,11 @@ public int getTableRowCount(String database, String table, String additionalFilt } private int getTableRowCount(String statement) throws SQLException { - //ResultSet resultSet = getTableRows(statement); statement2 = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY); ResultSet rs = statement2.executeQuery(statement); - System.out.println("type:"+rs.getType()); rs.last(); - int rowsInTable = rs.getRow(); - return rowsInTable; + return rs.getRow(); } public ResultSet getTableRows(String database, String table, String additionalFilters) throws SQLException { From ecd6378b597539ef122972b39f5bdc2b4f13c099 Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 16 Apr 2021 17:44:47 +0100 Subject: [PATCH 056/115] compiles correctly/checked --- beekeeper-integration-tests/pom.xml | 16 ---------------- .../BeekeeperIntegrationTestBase.java | 4 ---- .../cleanup/BeekeeperMetadataCleanup.java | 2 +- .../path/cleanup/BeekeeperPathCleanup.java | 3 +-- 4 files changed, 2 insertions(+), 23 deletions(-) diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 89812a1b..e5b7c073 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -52,10 +52,6 @@ org.apache.hive hive-service-rpc - - org.mortbay.jetty - jetty-util - @@ -77,18 +73,6 @@ slf4j-log4j12 org.slf4j - - org.mortbay.jetty - jetty-util - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-sslengine - diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 885685f1..559e55b6 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -181,10 +181,6 @@ protected void insertExpiredMetadata(String tableName, String path, String parti } protected int getUnreferencedPathsRowCount() throws SQLException { - System.out.println(BEEKEEPER_DB_NAME); - System.out.println(BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME); - System.out.println(LIFE_CYCLE_FILTER); - System.out.println(UNREFERENCED); return mySQLTestUtils .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, format(LIFE_CYCLE_FILTER, UNREFERENCED)); diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java index 220f299a..2fa9fc91 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/BeekeeperMetadataCleanup.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java index e95db8f2..50a22f11 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/BeekeeperPathCleanup.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.PropertySource; import io.micrometer.core.instrument.MeterRegistry; From ba321dbf7b6ce9472aafe6eeaf6cacd66657f940 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 19 Apr 2021 15:16:18 +0100 Subject: [PATCH 057/115] taking out jetty redundant exlcusions --- beekeeper-integration-tests/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index e5b7c073..934d128e 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -116,14 +116,6 @@ log4j-slf4j-impl org.apache.logging.log4j - - org.mortbay.jetty - jetty-util - - - org.mortbay.jetty - jetty - org.eclipse.jetty.aggregate jetty-all From 420f91b32a75121112f69c7fb7440d90988f86f5 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 19 Apr 2021 15:26:10 +0100 Subject: [PATCH 058/115] renaming variables --- ...referencedPathSchedulerApiaryIntegrationTest.java | 3 +-- .../beekeeper/integration/utils/MySqlTestUtils.java | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index ed4b817d..64c5f1be 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -144,8 +144,7 @@ public void unreferencedAlterPartitionEvent() throws SQLException, IOException, alterPartitionSqsMessage.setPartitionLocation("s3://partitionLocation2"); alterPartitionSqsMessage.setOldPartitionLocation("s3://partitionLocation"); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); - //await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() > 1); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); List unreferencedPaths = getUnreferencedPaths(); assertUnreferencedPath(unreferencedPaths.get(0), "s3://partitionLocation"); assertUnreferencedPath(unreferencedPaths.get(1), "s3://unreferencedPartitionLocation"); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index 95fc5a23..1d27ef50 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -30,7 +30,7 @@ public class MySqlTestUtils { private static final String INSERT_TO_TABLE = "INSERT INTO %s.%s (%s) VALUES (%s);"; private final Connection connection; - private Statement statement2; + private Statement statement; public MySqlTestUtils(String jdbcUrl, String username, String password) throws SQLException { connection = DriverManager.getConnection(jdbcUrl, username, password); @@ -52,12 +52,12 @@ public int getTableRowCount(String database, String table, String additionalFilt return getTableRowCount(format(SELECT_TABLE, database, table, additionalFilters)); } - private int getTableRowCount(String statement) throws SQLException { - statement2 = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, + private int getTableRowCount(String statementString) throws SQLException { + statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY); - ResultSet rs = statement2.executeQuery(statement); - rs.last(); - return rs.getRow(); + ResultSet resultSet = statement.executeQuery(statementString); + resultSet.last(); + return resultSet.getRow(); } public ResultSet getTableRows(String database, String table, String additionalFilters) throws SQLException { From c36d6ad4f9dd967a0c9e3b26c9c1a2757e43e7d6 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 19 Apr 2021 15:35:11 +0100 Subject: [PATCH 059/115] taking out useless exclusions --- beekeeper-scheduler-apiary/pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml index 8e867a46..4dc0c8c3 100644 --- a/beekeeper-scheduler-apiary/pom.xml +++ b/beekeeper-scheduler-apiary/pom.xml @@ -50,18 +50,6 @@ org.apache.geronimo.specs geronimo-jaspic_1.0_spec - - org.mortbay.jetty - jetty - - - org.eclipse.jetty.aggregate - jetty-all - - - org.mortbay.jetty - jetty-util - From 83b906c0ebfe41a4c040c03c58eac6f0bb5b1b2a Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 19 Apr 2021 15:36:28 +0100 Subject: [PATCH 060/115] tidy up --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9048a815..1c9a0153 100644 --- a/pom.xml +++ b/pom.xml @@ -21,9 +21,9 @@ beekeeper-path-cleanup beekeeper-scheduler beekeeper-scheduler-apiary - beekeeper-api beekeeper-integration-tests beekeeper-vacuum-tool + beekeeper-api From 877dea2169a9f4a57f0d4944853bf36eebeca166 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 19 Apr 2021 15:43:00 +0100 Subject: [PATCH 061/115] tidy up --- ...BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java | 1 + .../beekeeper/integration/utils/MySqlTestUtils.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 64c5f1be..6329e4fa 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -145,6 +145,7 @@ public void unreferencedAlterPartitionEvent() throws SQLException, IOException, alterPartitionSqsMessage.setOldPartitionLocation("s3://partitionLocation"); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); + List unreferencedPaths = getUnreferencedPaths(); assertUnreferencedPath(unreferencedPaths.get(0), "s3://partitionLocation"); assertUnreferencedPath(unreferencedPaths.get(1), "s3://unreferencedPartitionLocation"); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index 1d27ef50..564c8b5f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -30,7 +30,6 @@ public class MySqlTestUtils { private static final String INSERT_TO_TABLE = "INSERT INTO %s.%s (%s) VALUES (%s);"; private final Connection connection; - private Statement statement; public MySqlTestUtils(String jdbcUrl, String username, String password) throws SQLException { connection = DriverManager.getConnection(jdbcUrl, username, password); @@ -53,6 +52,7 @@ public int getTableRowCount(String database, String table, String additionalFilt } private int getTableRowCount(String statementString) throws SQLException { + Statement statement; statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY); ResultSet resultSet = statement.executeQuery(statementString); From 32390295f2f8a5d7ea3f327246abf6efd52c8203 Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 20 Apr 2021 11:06:55 +0100 Subject: [PATCH 062/115] relocating TestApplication class --- .../api/{controller => }/TestApplication.java | 6 ++--- .../controller/BeekeeperControllerTest.java | 22 ++++++++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/{controller => }/TestApplication.java (84%) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java similarity index 84% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java index 7fb53875..8711afce 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/TestApplication.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java @@ -13,19 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.expediagroup.beekeeper.api.controller; +package com.expediagroup.beekeeper.api; -import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; @SpringBootApplication -@SpringBootConfiguration @EnableConfigurationProperties @ComponentScan(basePackages = { "com.expediagroup.beekeeper.api.conf", - "com.expediagroup.beekeeper.api.controller" }) + "com.expediagroup.beekeeper.api.controller"}) public class TestApplication { } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 5106d434..d3642583 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -30,7 +30,6 @@ import java.util.List; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; @@ -40,17 +39,18 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; import org.springframework.http.MediaType; -import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import com.fasterxml.jackson.databind.ObjectMapper; +import com.expediagroup.beekeeper.api.TestApplication; import com.expediagroup.beekeeper.api.service.HousekeepingMetadataServiceImpl; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -@ExtendWith(SpringExtension.class) @WebMvcTest(BeekeeperController.class) +@ContextConfiguration(classes = TestApplication.class) public class BeekeeperControllerTest { @Autowired @@ -83,4 +83,20 @@ public void testGetAllWhenTablesValid() throws Exception { verify(housekeepingMetadataServiceImpl, times(1)).getAll(any(), any()); verifyNoMoreInteractions(housekeepingMetadataServiceImpl); } + + @Test + public void testGetAllWhenNoTables() throws Exception { + Page tables = new PageImpl<>(List.of()); + + when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataServiceImpl, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataServiceImpl); + } } From 6f83af684f10760a743a81b034416154ec3f78aa Mon Sep 17 00:00:00 2001 From: shermosa Date: Tue, 20 Apr 2021 15:52:41 +0100 Subject: [PATCH 063/115] renaming class names --- .../api/controller/BeekeeperController.java | 10 +++++----- ...mpl.java => HousekeepingEntityService.java} | 18 +++--------------- .../service/HousekeepingMetadataService.java | 18 +++++++++++++++--- .../controller/BeekeeperControllerTest.java | 16 ++++++++-------- ...va => HouseKeepingMetadataServiceTest.java} | 8 ++++---- 5 files changed, 35 insertions(+), 35 deletions(-) rename beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/{HousekeepingMetadataServiceImpl.java => HousekeepingEntityService.java} (53%) rename beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/{HouseKeepingMetadataServiceImplTest.java => HouseKeepingMetadataServiceTest.java} (90%) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 0407162f..3d0c7fc4 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -33,18 +33,18 @@ import net.kaczmarzyk.spring.data.jpa.web.annotation.And; import net.kaczmarzyk.spring.data.jpa.web.annotation.Spec; -import com.expediagroup.beekeeper.api.service.HousekeepingMetadataServiceImpl; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @RequestMapping("/api/v1") @RestController public class BeekeeperController { - private final HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + private final HousekeepingMetadataService housekeepingMetadataService; @Autowired - public BeekeeperController(HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl) { - this.housekeepingMetadataServiceImpl = housekeepingMetadataServiceImpl; + public BeekeeperController(HousekeepingMetadataService housekeepingMetadataService) { + this.housekeepingMetadataService = housekeepingMetadataService; } @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE) @@ -57,7 +57,7 @@ public ResponseEntity> getAll( @Spec(path = "deletedAfter", spec = GreaterThan.class) }) Specification spec, Pageable pageable) { - return ResponseEntity.ok(housekeepingMetadataServiceImpl.getAll(spec, pageable)); + return ResponseEntity.ok(housekeepingMetadataService.getAll(spec, pageable)); } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingEntityService.java similarity index 53% rename from beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java rename to beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingEntityService.java index 521ba60e..c7477c22 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataServiceImpl.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingEntityService.java @@ -16,27 +16,15 @@ package com.expediagroup.beekeeper.api.service; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; -import org.springframework.stereotype.Service; +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; -@Service -public class HousekeepingMetadataServiceImpl implements HousekeepingMetadataService { +public interface HousekeepingEntityService { - private final HousekeepingMetadataRepository housekeepingMetadataRepository; - - @Autowired - public HousekeepingMetadataServiceImpl(HousekeepingMetadataRepository housekeepingMetadataRepository) { - this.housekeepingMetadataRepository = housekeepingMetadataRepository; - } - - public Page getAll(Specification spec, Pageable pageable) { - return housekeepingMetadataRepository.findAll(spec, pageable); - } + Page getAll(Specification spec, Pageable pageable); } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java index 0c5e2dde..61553d14 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/service/HousekeepingMetadataService.java @@ -16,15 +16,27 @@ package com.expediagroup.beekeeper.api.service; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Service; -import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; -public interface HousekeepingMetadataService { +@Service +public class HousekeepingMetadataService implements HousekeepingEntityService { - Page getAll(Specification spec, Pageable pageable); + private final HousekeepingMetadataRepository housekeepingMetadataRepository; + + @Autowired + public HousekeepingMetadataService(HousekeepingMetadataRepository housekeepingMetadataRepository) { + this.housekeepingMetadataRepository = housekeepingMetadataRepository; + } + + public Page getAll(Specification spec, Pageable pageable) { + return housekeepingMetadataRepository.findAll(spec, pageable); + } } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index d3642583..d6c2c5cb 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -46,7 +46,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.expediagroup.beekeeper.api.TestApplication; -import com.expediagroup.beekeeper.api.service.HousekeepingMetadataServiceImpl; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @WebMvcTest(BeekeeperController.class) @@ -64,7 +64,7 @@ public class BeekeeperControllerTest { private Pageable pageable; @MockBean - private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + private HousekeepingMetadataService housekeepingMetadataService; @Test public void testGetAllWhenTablesValid() throws Exception { @@ -72,7 +72,7 @@ public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); + when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); mockMvc .perform(get("/api/v1/tables")) @@ -80,15 +80,15 @@ public void testGetAllWhenTablesValid() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) .andExpect(content().json(objectMapper.writeValueAsString(tables))); - verify(housekeepingMetadataServiceImpl, times(1)).getAll(any(), any()); - verifyNoMoreInteractions(housekeepingMetadataServiceImpl); + verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataService); } @Test public void testGetAllWhenNoTables() throws Exception { Page tables = new PageImpl<>(List.of()); - when(housekeepingMetadataServiceImpl.getAll(any(), any())).thenReturn(tables); + when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); mockMvc .perform(get("/api/v1/tables")) @@ -96,7 +96,7 @@ public void testGetAllWhenNoTables() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) .andExpect(content().json(objectMapper.writeValueAsString(tables))); - verify(housekeepingMetadataServiceImpl, times(1)).getAll(any(), any()); - verifyNoMoreInteractions(housekeepingMetadataServiceImpl); + verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataService); } } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceImplTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java similarity index 90% rename from beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceImplTest.java rename to beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java index 0cd2c987..0c9098e1 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceImplTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java @@ -39,9 +39,9 @@ import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @ExtendWith(MockitoExtension.class) -public class HouseKeepingMetadataServiceImplTest { +public class HouseKeepingMetadataServiceTest { - private HousekeepingMetadataServiceImpl housekeepingMetadataServiceImpl; + private HousekeepingMetadataService housekeepingMetadataService; @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @@ -52,7 +52,7 @@ public class HouseKeepingMetadataServiceImplTest { @BeforeEach public void beforeEach() { - housekeepingMetadataServiceImpl = new HousekeepingMetadataServiceImpl(housekeepingMetadataRepository); + housekeepingMetadataService = new HousekeepingMetadataService(housekeepingMetadataRepository); } @Test @@ -62,7 +62,7 @@ public void testGetAllWhenTablesValid(){ Page tables = new PageImpl<>(List.of(table1, table2)); when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); - Page result = housekeepingMetadataServiceImpl.getAll(spec, pageable); + Page result = housekeepingMetadataService.getAll(spec, pageable); assertThat(tables).isEqualTo(result); verify(housekeepingMetadataRepository, times(1)).findAll(spec, pageable); From 637b22db9642574e4b495ef3cfd6a0ca75c6016a Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 21 Apr 2021 11:09:58 +0100 Subject: [PATCH 064/115] adding test for filtering --- .../api/controller/BeekeeperController.java | 2 +- .../controller/BeekeeperControllerTest.java | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 3d0c7fc4..24c2dda4 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -50,7 +50,7 @@ public BeekeeperController(HousekeepingMetadataService housekeepingMetadataServi @GetMapping(path = "/tables", produces = APPLICATION_JSON_VALUE) public ResponseEntity> getAll( @And({ - @Spec(path = "tableName", spec = EqualIgnoreCase.class), + @Spec(path = "tableName", params = "table_name", spec = EqualIgnoreCase.class), @Spec(path = "databaseName", spec = EqualIgnoreCase.class), @Spec(path = "status", spec = EqualIgnoreCase.class), @Spec(path = "deletedBefore", spec = LessThan.class), diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index d6c2c5cb..aba3033c 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -99,4 +99,22 @@ public void testGetAllWhenNoTables() throws Exception { verify(housekeepingMetadataService, times(1)).getAll(any(), any()); verifyNoMoreInteractions(housekeepingMetadataService); } + + @Test + public void testGetAllWhenUsingTableNameFiltering() throws Exception { + HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); + + when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables?table_name=bob")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataService); + } } From 9d6c5e6b1f5b42ce9bb3aa2d3aba52d24114e687 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 21 Apr 2021 11:20:55 +0100 Subject: [PATCH 065/115] test for wrong url --- .../api/controller/BeekeeperControllerTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index aba3033c..176fd932 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -21,9 +21,12 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static org.springframework.http.HttpStatus.NOT_FOUND; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; + import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; @@ -100,6 +103,18 @@ public void testGetAllWhenNoTables() throws Exception { verifyNoMoreInteractions(housekeepingMetadataService); } + @Test + public void testWhenWrongUrl() throws Exception { + Page tables = new PageImpl<>(List.of()); + + when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tablessssss")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isNotFound()); + } + @Test public void testGetAllWhenUsingTableNameFiltering() throws Exception { HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); From baf6ea8b3509830107b8bf6b0c69b43efb7fe965 Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 21 Apr 2021 14:21:36 +0100 Subject: [PATCH 066/115] test for paging --- .../controller/BeekeeperControllerTest.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 176fd932..8c464aba 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -17,16 +17,14 @@ import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import static org.springframework.http.HttpStatus.NOT_FOUND; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; - import static com.expediagroup.beekeeper.api.util.DummyHousekeepingMetadataGenerator.generateDummyHousekeepingMetadata; @@ -39,6 +37,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; import org.springframework.http.MediaType; @@ -104,7 +103,7 @@ public void testGetAllWhenNoTables() throws Exception { } @Test - public void testWhenWrongUrl() throws Exception { + public void testControllerWhenWrongUrl() throws Exception { Page tables = new PageImpl<>(List.of()); when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); @@ -124,7 +123,29 @@ public void testGetAllWhenUsingTableNameFiltering() throws Exception { when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); mockMvc - .perform(get("/api/v1/tables?table_name=bob")) + .perform(get("/api/v1/tables?table_name=bob").param("tableName","bob")) + .andDo(MockMvcResultHandlers.print()) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(objectMapper.writeValueAsString(tables))); + verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verifyNoMoreInteractions(housekeepingMetadataService); + } + + @Test + public void testPaging() throws Exception { + int pageNumber = 5; + int pageSize = 10; + HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); + + when(housekeepingMetadataService.getAll(any(), eq(PageRequest.of(pageNumber, pageSize)))).thenReturn(tables); + + mockMvc + .perform(get("/api/v1/tables") + .param("page", String.valueOf(pageNumber)) + .param("size", String.valueOf(pageSize))) .andDo(MockMvcResultHandlers.print()) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) From c972dc798c47e701cd459dd559adef93b323d9cf Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 22 Apr 2021 14:23:11 +0100 Subject: [PATCH 067/115] adding api integration test class --- .../controller/BeekeeperControllerTest.java | 19 ++++++++++++++----- .../HouseKeepingMetadataServiceTest.java | 3 ++- .../api/BeekeeperApiIntegrationTest.java | 4 ++++ 3 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index 8c464aba..fc5de62b 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -31,6 +31,8 @@ import java.util.List; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; @@ -60,14 +62,17 @@ public class BeekeeperControllerTest { @Autowired private ObjectMapper objectMapper; - @Mock - private Specification spec; + private Specification spec = Specification.where(null); + @Mock private Pageable pageable; @MockBean private HousekeepingMetadataService housekeepingMetadataService; + @Captor + ArgumentCaptor emailCaptor; + @Test public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); @@ -120,15 +125,19 @@ public void testGetAllWhenUsingTableNameFiltering() throws Exception { HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); Page tables = new PageImpl<>(List.of(table1, table2)); - when(housekeepingMetadataService.getAll(any(), any())).thenReturn(tables); + when(housekeepingMetadataService.getAll(spec, pageable)).thenReturn(tables); + + //spec.and() mockMvc - .perform(get("/api/v1/tables?table_name=bob").param("tableName","bob")) + .perform(get("/api/v1/tables?table_name=bob") + //.param("tableName","bob") + ) .andDo(MockMvcResultHandlers.print()) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) .andExpect(content().json(objectMapper.writeValueAsString(tables))); - verify(housekeepingMetadataService, times(1)).getAll(any(), any()); + verify(housekeepingMetadataService, times(1)).getAll(spec, pageable); verifyNoMoreInteractions(housekeepingMetadataService); } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java index 0c9098e1..adb95b9e 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java @@ -46,7 +46,7 @@ public class HouseKeepingMetadataServiceTest { @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @Mock - private Specification spec; + private Specification spec = Specification.where(""); @Mock private Pageable pageable; @@ -60,6 +60,7 @@ public void testGetAllWhenTablesValid(){ HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); HousekeepingMetadata table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + Page tables = new PageImpl<>(List.of(table1, table2)); when(housekeepingMetadataRepository.findAll(spec, pageable)).thenReturn(tables); Page result = housekeepingMetadataService.getAll(spec, pageable); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java new file mode 100644 index 00000000..509e0fc7 --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -0,0 +1,4 @@ +package com.expediagroup.beekeeper.integration.api; + +public class BeekeeperApiIntegrationTest { +} From e3a903a27637173504ef7f0aae0aba385969fb6b Mon Sep 17 00:00:00 2001 From: shermosa Date: Thu, 22 Apr 2021 15:19:13 +0100 Subject: [PATCH 068/115] ading beforeeach and aftereach methods --- .../HouseKeepingMetadataServiceTest.java | 2 +- beekeeper-integration-tests/pom.xml | 6 ++ .../api/BeekeeperApiIntegrationTest.java | 61 ++++++++++++++++++- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java index adb95b9e..c7df8654 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/service/HouseKeepingMetadataServiceTest.java @@ -46,7 +46,7 @@ public class HouseKeepingMetadataServiceTest { @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; @Mock - private Specification spec = Specification.where(""); + private Specification spec; @Mock private Pageable pageable; diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 934d128e..9e772002 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -122,5 +122,11 @@ + + com.expediagroup + beekeeper-api + 3.1.1-SNAPSHOT + test + diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 509e0fc7..d2c4fbf9 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -1,4 +1,63 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.expediagroup.beekeeper.integration.api; -public class BeekeeperApiIntegrationTest { +import static java.lang.String.format; + +import java.sql.SQLException; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.util.SocketUtils; + +import com.expediagroup.beekeeper.api.BeekeeperApiApplication; +import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; +import com.expediagroup.beekeeper.integration.BeekeeperIntegrationTestBase; + +public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { + + private static final Logger log = LoggerFactory.getLogger(BeekeeperApiIntegrationTest.class); + + // APP CONTEXT AND TEST CLIENT + protected static ConfigurableApplicationContext context; + protected HousekeepingMetadataService testClient; + + @BeforeEach + public void beforeEach() throws Exception { + int port = SocketUtils.findAvailableTcpPort(); + final String url = format("http://localhost:%d", port); + log.info("Starting to run Beekeeper API on: {}", url); + context = SpringApplication.run(BeekeeperApiApplication.class); + } + + @AfterEach + public final void afterEach() throws SQLException { + log.info("Stopping Cloverleaf API"); + if (context != null) { + context.close(); + context = null; + } + } + + @Test + public void test(){} + } From e7eb6fba5967e7daac1581c8d176d9e26f6741ba Mon Sep 17 00:00:00 2001 From: shermosa Date: Fri, 23 Apr 2021 11:21:33 +0100 Subject: [PATCH 069/115] trying to run integration tests --- .../main/resources/beekeeper-api-application.yml | 13 +++++++++++++ .../api/BeekeeperApiIntegrationTest.java | 11 +++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 beekeeper-api/src/main/resources/beekeeper-api-application.yml diff --git a/beekeeper-api/src/main/resources/beekeeper-api-application.yml b/beekeeper-api/src/main/resources/beekeeper-api-application.yml new file mode 100644 index 00000000..dc42a41a --- /dev/null +++ b/beekeeper-api/src/main/resources/beekeeper-api-application.yml @@ -0,0 +1,13 @@ +management.endpoints.web.exposure.include: health,info,prometheus + +server: + port: 9008 + +spring: + jpa: + database: default + hibernate: + ddl-auto: validate + driver-class-name: org.h2.Driver + properties.hibernate: + dialect: org.hibernate.dialect.MySQL8Dialect diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index d2c4fbf9..2b44ce15 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -41,7 +41,7 @@ public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { protected HousekeepingMetadataService testClient; @BeforeEach - public void beforeEach() throws Exception { + public void beforeEach() { int port = SocketUtils.findAvailableTcpPort(); final String url = format("http://localhost:%d", port); log.info("Starting to run Beekeeper API on: {}", url); @@ -49,8 +49,8 @@ public void beforeEach() throws Exception { } @AfterEach - public final void afterEach() throws SQLException { - log.info("Stopping Cloverleaf API"); + public final void afterEach() { + log.info("Stopping Beekeeper API"); if (context != null) { context.close(); context = null; @@ -58,6 +58,9 @@ public final void afterEach() throws SQLException { } @Test - public void test(){} + public void test() throws SQLException, InterruptedException { + insertExpiredMetadata("s3://can-you-fix-this-please", "partition=vedant_is_stupid"); + Thread.sleep(1000000L); + } } From 3711af0a7be01817d38f8681080933d3c2e8f510 Mon Sep 17 00:00:00 2001 From: shermosa Date: Mon, 26 Apr 2021 09:25:28 +0100 Subject: [PATCH 070/115] renaming --- .../integration/api/BeekeeperApiIntegrationTest.java | 8 +++++--- .../src/test/resources/logback-test.xml | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 2b44ce15..5faca451 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -43,9 +43,11 @@ public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { @BeforeEach public void beforeEach() { int port = SocketUtils.findAvailableTcpPort(); + String[] args = new String[] { + "--server.port=" + port}; final String url = format("http://localhost:%d", port); - log.info("Starting to run Beekeeper API on: {}", url); - context = SpringApplication.run(BeekeeperApiApplication.class); + log.info("Starting to run Beekeeper API on: {} and args: {}", url, args); + context = SpringApplication.run(BeekeeperApiApplication.class, args); } @AfterEach @@ -59,7 +61,7 @@ public final void afterEach() { @Test public void test() throws SQLException, InterruptedException { - insertExpiredMetadata("s3://can-you-fix-this-please", "partition=vedant_is_stupid"); + insertExpiredMetadata("s3://path/to/s3/table", "partition=random/partition"); Thread.sleep(1000000L); } diff --git a/beekeeper-integration-tests/src/test/resources/logback-test.xml b/beekeeper-integration-tests/src/test/resources/logback-test.xml index 42a3e8ea..66e5e88b 100644 --- a/beekeeper-integration-tests/src/test/resources/logback-test.xml +++ b/beekeeper-integration-tests/src/test/resources/logback-test.xml @@ -24,6 +24,7 @@ + @@ -33,5 +34,6 @@ + From b1f095811d988ea62c74a5c431ff0ca85433736a Mon Sep 17 00:00:00 2001 From: shermosa Date: Wed, 28 Apr 2021 15:28:33 +0100 Subject: [PATCH 071/115] reproducing error --- .../api/BeekeeperApiApplication.java | 14 ++++ .../api/controller/BeekeeperController.java | 5 +- .../controller/BeekeeperControllerTest.java | 6 +- .../DummyHousekeepingMetadataGenerator.java | 2 + .../core/model/HousekeepingMetadata.java | 1 + beekeeper-integration-tests/pom.xml | 13 ++++ .../api/BeekeeperApiIntegrationTest.java | 73 ++++++++++++++++++- .../utils/BeekeeperApiTestClient.java | 47 ++++++++++++ .../integration/utils/RestResponsePage.java | 60 +++++++++++++++ .../src/test/resources/logback-test.xml | 2 +- pom.xml | 16 ++++ 11 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java create mode 100644 beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java index 3763dc74..c2a5fbd3 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/BeekeeperApiApplication.java @@ -15,6 +15,9 @@ */ package com.expediagroup.beekeeper.api; +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; @@ -22,6 +25,9 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + @SpringBootApplication @EnableConfigurationProperties @EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) @@ -32,7 +38,15 @@ "com.expediagroup.beekeeper.api.service" }) public class BeekeeperApiApplication { + @Autowired + private ObjectMapper objectMapper; + public static void main(String[] args) { SpringApplication.run(BeekeeperApiApplication.class, args); } + + @PostConstruct + public void setUp() { + objectMapper.registerModule(new JavaTimeModule()); + } } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 24c2dda4..ba632a70 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -51,8 +51,9 @@ public BeekeeperController(HousekeepingMetadataService housekeepingMetadataServi public ResponseEntity> getAll( @And({ @Spec(path = "tableName", params = "table_name", spec = EqualIgnoreCase.class), - @Spec(path = "databaseName", spec = EqualIgnoreCase.class), - @Spec(path = "status", spec = EqualIgnoreCase.class), + @Spec(path = "databaseName", params = "database_name", spec = EqualIgnoreCase.class), + @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), + @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), @Spec(path = "deletedBefore", spec = LessThan.class), @Spec(path = "deletedAfter", spec = GreaterThan.class) }) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java index fc5de62b..8b9d3bcb 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/controller/BeekeeperControllerTest.java @@ -62,7 +62,8 @@ public class BeekeeperControllerTest { @Autowired private ObjectMapper objectMapper; - private Specification spec = Specification.where(null); + @Mock + private Specification spec; @Mock private Pageable pageable; @@ -70,9 +71,6 @@ public class BeekeeperControllerTest { @MockBean private HousekeepingMetadataService housekeepingMetadataService; - @Captor - ArgumentCaptor emailCaptor; - @Test public void testGetAllWhenTablesValid() throws Exception { HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java index e6799d57..2dddcd8d 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingMetadataGenerator.java @@ -29,6 +29,8 @@ public class DummyHousekeepingMetadataGenerator { private static final String DEFAULT_DB_NAME = "randomDatabase"; private static final String DEFAULT_TABLE_NAME = "randomTable"; + @JsonDeserialize(using = LocalDateDeserializer.class) + @JsonSerialize(using = LocalDateSerializer.class) private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); private static final Duration CLEANUP_DELAY = Duration.parse("P3D"); diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java index eec8a180..a3394b52 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java @@ -35,6 +35,7 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.monitoring.MetricTag; +@com.fasterxml.jackson.annotation.JsonIgnoreProperties(ignoreUnknown = true) @Entity @Table(name = "housekeeping_metadata") public class HousekeepingMetadata implements HousekeepingEntity { diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 9e772002..53c35c54 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -75,6 +75,19 @@ + + + com.fasterxml.jackson.module + jackson-module-parameter-names + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + org.awaitility diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 5faca451..314890cd 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -17,37 +17,83 @@ import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.http.HttpStatus.OK; + +import java.io.IOException; +import java.net.http.HttpResponse; import java.sql.SQLException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; import org.springframework.util.SocketUtils; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; + import com.expediagroup.beekeeper.api.BeekeeperApiApplication; -import com.expediagroup.beekeeper.api.service.HousekeepingMetadataService; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.integration.BeekeeperIntegrationTestBase; +import com.expediagroup.beekeeper.integration.utils.BeekeeperApiTestClient; +import com.expediagroup.beekeeper.integration.utils.RestResponsePage; public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { + + @Mock + private Specification spec; + + @Mock + private Pageable pageable; + + @Bean + @Primary + public ObjectMapper geObjMapper(){ + return new ObjectMapper() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } + private static final Logger log = LoggerFactory.getLogger(BeekeeperApiIntegrationTest.class); // APP CONTEXT AND TEST CLIENT protected static ConfigurableApplicationContext context; - protected HousekeepingMetadataService testClient; + protected BeekeeperApiTestClient testClient; + + protected final ObjectMapper mapper = geObjMapper(); + + @BeforeEach public void beforeEach() { + + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + +// mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); +// mapper.setVisibility(VisibilityChecker.Std.defaultInstance().withFieldVisibility(JsonAutoDetect.Visibility.ANY)); int port = SocketUtils.findAvailableTcpPort(); String[] args = new String[] { "--server.port=" + port}; final String url = format("http://localhost:%d", port); log.info("Starting to run Beekeeper API on: {} and args: {}", url, args); context = SpringApplication.run(BeekeeperApiApplication.class, args); + testClient = new BeekeeperApiTestClient(url); } @AfterEach @@ -60,9 +106,28 @@ public final void afterEach() { } @Test - public void test() throws SQLException, InterruptedException { + public void test() throws SQLException, InterruptedException, IOException { + //HousekeepingMetadata table1 = generateDummyHousekeepingMetadata("aRandomTable", "aRandomDatabase"); + // table2 = generateDummyHousekeepingMetadata("aRandomTable2", "aRandomDatabase2"); + insertExpiredMetadata("s3://path/to/s3/table", "partition=random/partition"); - Thread.sleep(1000000L); + insertExpiredMetadata("s3://path/to/s3/table2", "partition=random/partition2"); + //insertExpiredMetadata("saras_table", "a/path", "a_random_partition", "P7D"); + + //Thread.sleep(1000000L); + System.out.println("breakpoint1"); + HttpResponse response = testClient.getTables(); + System.out.println("breakpoint2"); + assertThat(response.statusCode()).isEqualTo(OK.value()); + System.out.println("breakpoint3"); + String body = response.body(); + System.out.println("body:"+body); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + System.out.println("breakpoint5"); +// assertThat(responsePage.getContent()).isEqualTo(List.of()); + + } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java new file mode 100644 index 00000000..093f9580 --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.integration.utils; + +import static java.net.http.HttpRequest.newBuilder; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; + +public class BeekeeperApiTestClient { + + private static final String PREFIX = "/api/v1"; + private static final String TABLES_PATH = PREFIX + "/tables"; + + private final String tablesUrl; + private final HttpClient httpClient; + + public BeekeeperApiTestClient(String baseUrl) { + System.out.println("breakpoint4"); + this.tablesUrl = baseUrl + TABLES_PATH; + this.httpClient = HttpClient.newHttpClient(); + } + + public HttpResponse getTables() throws IOException, InterruptedException { + System.out.println("AAA:"+tablesUrl); + HttpRequest request = newBuilder().uri(URI.create(tablesUrl)).GET().build(); + return httpClient.send(request, BodyHandlers.ofString()); + } + +} diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java new file mode 100644 index 00000000..5fc30012 --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2019-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.integration.utils; +import java.sql.Date; +import java.util.List; + +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + + +public class RestResponsePage extends PageImpl { + +// @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) +// public RestResponsePage(@JsonProperty("content") List content, +// @JsonProperty("number") int page, +// @JsonProperty("size") int size, +// @JsonProperty("totalElements") long total) { +// +// super(content, PageRequest.of(page, size), total); +// } + + + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public RestResponsePage( + @JsonProperty("content") List content, + @JsonProperty("number") int number, + @JsonProperty("size") int size, + @JsonProperty("totalElements") Long totalElements, + @JsonProperty("id") int id, + @JsonProperty("path") String path, + @JsonProperty("databaseName") String databaseName, + @JsonProperty("tableName") String tableName, + @JsonProperty("partitionName") String partitionName, + @JsonProperty("housekeepingStatus") String housekeepingStatus, + @JsonProperty("creationTimestamp") Date creationTimestamp, + @JsonProperty("metricTag") String metricTag +// @JsonProperty("modifiedTimestamp") String modifiedTimestamp, +// @JsonProperty("cleanupTimestamp") String cleanupTimestamp, +// @JsonProperty("cleanupDelay") String cleanupDelay + ){ + super(content, PageRequest.of(number, size), totalElements); + } +} diff --git a/beekeeper-integration-tests/src/test/resources/logback-test.xml b/beekeeper-integration-tests/src/test/resources/logback-test.xml index 66e5e88b..0b0174af 100644 --- a/beekeeper-integration-tests/src/test/resources/logback-test.xml +++ b/beekeeper-integration-tests/src/test/resources/logback-test.xml @@ -1,6 +1,6 @@