diff --git a/pom.xml b/pom.xml index e048b939f2cae..2f0d2ac0c3f81 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ 0.11.0 0.19.0 2.3.1 - 0.12.1 + 0.14.0-SNAPSHOT 1.15.1 3.2.12 2.6.0 @@ -1294,6 +1294,10 @@ org.apache.avro avro + + org.objenesis + objenesis + diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index ca10f43da0ac2..76d60103cd6e6 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -187,6 +187,7 @@ parquet.thrift about.html + mozilla/public-suffix-list.txt shaded.parquet.it.unimi.dsi.fastutil.* diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 39e778dcca084..fec1482326822 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -462,6 +462,7 @@ parquet.thrift about.html + mozilla/public-suffix-list.txt shaded.parquet.it.unimi.dsi.fastutil.* diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java index 3d1baf7aa338a..859e37a6e5792 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java @@ -417,11 +417,7 @@ static boolean isHudiParquetInputFormat(InputFormat inputFormat) private static boolean shouldUseFileSplitsForHudi(InputFormat inputFormat, HoodieTableMetaClient metaClient) { - if (inputFormat instanceof HoodieParquetRealtimeInputFormat) { - return true; - } - - return metaClient.getTableConfig().getBootstrapBasePath().isPresent(); + return inputFormat instanceof HoodieParquetRealtimeInputFormat; } public static long parseHiveDate(String value) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HudiDirectoryLister.java b/presto-hive/src/main/java/com/facebook/presto/hive/HudiDirectoryLister.java index ed6851dcbeeb9..8c503e2804f6d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HudiDirectoryLister.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HudiDirectoryLister.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.model.BaseFile; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.FileSystemViewManager; @@ -132,7 +133,8 @@ public boolean hasNext() public HiveFileInfo next() throws IOException { - FileStatus fileStatus = hoodieBaseFileIterator.next().getFileStatus(); + HoodieBaseFile baseFile = hoodieBaseFileIterator.next(); + FileStatus fileStatus = baseFile.getBootstrapBaseFile().map(BaseFile::getFileStatus).orElse(baseFile.getFileStatus()); String[] name = new String[] {"localhost:" + DFS_DATANODE_DEFAULT_PORT}; String[] host = new String[] {"localhost"}; LocatedFileStatus hoodieFileStatus = new LocatedFileStatus(fileStatus, diff --git a/presto-hudi/src/main/java/com/facebook/presto/hudi/split/HudiPartitionSplitGenerator.java b/presto-hudi/src/main/java/com/facebook/presto/hudi/split/HudiPartitionSplitGenerator.java index 9222764eb309f..3430fecf86cff 100644 --- a/presto-hudi/src/main/java/com/facebook/presto/hudi/split/HudiPartitionSplitGenerator.java +++ b/presto-hudi/src/main/java/com/facebook/presto/hudi/split/HudiPartitionSplitGenerator.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.HoodieTimer; @@ -121,7 +122,10 @@ private Optional createHudiSplit( HudiPartition partition, HudiSplitWeightProvider splitWeightProvider) { - HudiFile baseFile = slice.getBaseFile().map(f -> new HudiFile(f.getPath(), 0, f.getFileLen())).orElse(null); + HudiFile baseFile = slice.getBaseFile().map( + f -> f.getBootstrapBaseFile().map(bootstrapFile -> new HudiFile(bootstrapFile.getPath(), 0, bootstrapFile.getFileLen())) + .orElse(new HudiFile(f.getPath(), 0, f.getFileLen()))) + .orElse(null); if (null == baseFile && table.getTableType() == HudiTableType.COW) { return Optional.empty(); } diff --git a/presto-iceberg/pom.xml b/presto-iceberg/pom.xml index 610cef6b730ad..9b17c05f086c4 100644 --- a/presto-iceberg/pom.xml +++ b/presto-iceberg/pom.xml @@ -522,6 +522,7 @@ module-info org.apache.avro.* org.apache.parquet.* + com.github.benmanes.caffeine.* diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 3c64447bb7102..1fd2c8f51e5f0 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -203,6 +203,7 @@ parquet.thrift about.html + mozilla/public-suffix-list.txt shaded.parquet.it.unimi.dsi.fastutil.* diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml index ab2c9e92fb063..780abd0502a89 100644 --- a/presto-native-execution/pom.xml +++ b/presto-native-execution/pom.xml @@ -146,7 +146,14 @@ parquet.thrift about.html + mozilla/public-suffix-list.txt + + com.esotericsoftware.kryo.* + com.esotericsoftware.minlog.Log + com.esotericsoftware.reflectasm.* + module-info + diff --git a/presto-spark-base/pom.xml b/presto-spark-base/pom.xml index 8a269f1435854..9a37fba8d0ee1 100644 --- a/presto-spark-base/pom.xml +++ b/presto-spark-base/pom.xml @@ -302,10 +302,14 @@ parquet.thrift about.html + mozilla/public-suffix-list.txt shaded.parquet.it.unimi.dsi.fastutil.* module-info + com.esotericsoftware.kryo.* + com.esotericsoftware.minlog.Log + com.esotericsoftware.reflectasm.* diff --git a/presto-spark-testing/pom.xml b/presto-spark-testing/pom.xml index d31658fb0cd31..6bb68f8b3a4db 100644 --- a/presto-spark-testing/pom.xml +++ b/presto-spark-testing/pom.xml @@ -136,6 +136,7 @@ parquet.thrift about.html + mozilla/public-suffix-list.txt shaded.parquet.it.unimi.dsi.fastutil.*