Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/velox_backend_x86.yml
Original file line number Diff line number Diff line change
Expand Up @@ -821,7 +821,7 @@ jobs:
java -version
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l $SPARK_HOME
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
-DargLine="-Dspark.test.home=$SPARK_HOME"
- name: Upload test report
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,7 @@
*/
package org.apache.gluten.execution

import org.apache.gluten.tags.SkipTest

@SkipTest
class VeloxIcebergSuite extends IcebergSuite
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@
*/
package org.apache.gluten.execution

import org.apache.gluten.tags.SkipTest

import org.apache.spark.SparkConf
import org.apache.spark.sql.functions.col

import org.apache.iceberg.spark.SparkWriteOptions

import java.io.File

@SkipTest
class VeloxTPCHIcebergSuite extends VeloxTPCHSuite {
protected val tpchBasePath: String =
getClass.getResource("/").getPath + "../../../src/test/resources"
Expand Down
10 changes: 5 additions & 5 deletions cpp/velox/memory/GlutenBufferedInputBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ class GlutenBufferedInputBuilder : public facebook::velox::connector::hive::Buff
const facebook::velox::FileHandle& fileHandle,
const facebook::velox::dwio::common::ReaderOptions& readerOpts,
const facebook::velox::connector::ConnectorQueryCtx* connectorQueryCtx,
std::shared_ptr<facebook::velox::io::IoStatistics> ioStats,
std::shared_ptr<facebook::velox::filesystems::File::IoStats> fsStats,
std::shared_ptr<facebook::velox::io::IoStatistics> ioStatistics,
std::shared_ptr<facebook::velox::IoStats> ioStats,
folly::Executor* executor,
const folly::F14FastMap<std::string, std::string>& fileReadOps = {}) override {
if (connectorQueryCtx->cache()) {
Expand All @@ -42,8 +42,8 @@ class GlutenBufferedInputBuilder : public facebook::velox::connector::hive::Buff
connectorQueryCtx->cache(),
facebook::velox::connector::Connector::getTracker(connectorQueryCtx->scanId(), readerOpts.loadQuantum()),
fileHandle.groupId,
ioStats,
std::move(fsStats),
std::move(ioStatistics),
std::move(ioStats),
executor,
readerOpts,
fileReadOps);
Expand All @@ -54,8 +54,8 @@ class GlutenBufferedInputBuilder : public facebook::velox::connector::hive::Buff
fileHandle.uuid,
facebook::velox::connector::Connector::getTracker(connectorQueryCtx->scanId(), readerOpts.loadQuantum()),
fileHandle.groupId,
std::move(ioStatistics),
std::move(ioStats),
std::move(fsStats),
executor,
readerOpts,
fileReadOps);
Expand Down
6 changes: 3 additions & 3 deletions cpp/velox/memory/GlutenDirectBufferedInput.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ class GlutenDirectBufferedInput : public facebook::velox::dwio::common::DirectBu
facebook::velox::StringIdLease fileNum,
std::shared_ptr<facebook::velox::cache::ScanTracker> tracker,
facebook::velox::StringIdLease groupId,
std::shared_ptr<facebook::velox::io::IoStatistics> ioStats,
std::shared_ptr<facebook::velox::filesystems::File::IoStats> fsStats,
std::shared_ptr<facebook::velox::io::IoStatistics> ioStatistics,
std::shared_ptr<facebook::velox::IoStats> ioStats,
folly::Executor* executor,
const facebook::velox::io::ReaderOptions& readerOptions,
folly::F14FastMap<std::string, std::string> fileReadOps = {})
Expand All @@ -40,8 +40,8 @@ class GlutenDirectBufferedInput : public facebook::velox::dwio::common::DirectBu
std::move(fileNum),
std::move(tracker),
std::move(groupId),
std::move(ioStatistics),
std::move(ioStats),
std::move(fsStats),
executor,
readerOptions,
std::move(fileReadOps)) {}
Expand Down
4 changes: 2 additions & 2 deletions cpp/velox/utils/VeloxWriterUtils.cc
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ std::unique_ptr<WriterOptions> makeParquetWriteOption(const std::unordered_map<s
auto parquetGzipWindowSizeStr = sparkConfs.find(kParquetGzipWindowSize)->second;
if (parquetGzipWindowSizeStr == kGzipWindowSize4k) {
auto codecOptions = std::make_shared<parquet::arrow::util::GZipCodecOptions>();
codecOptions->window_bits = kGzipWindowBits4k;
codecOptions->windowBits = kGzipWindowBits4k;
writeOption->codecOptions = std::move(codecOptions);
}
}
Expand All @@ -77,7 +77,7 @@ std::unique_ptr<WriterOptions> makeParquetWriteOption(const std::unordered_map<s
auto codecOptions = std::make_shared<parquet::arrow::util::CodecOptions>();
auto it = sparkConfs.find(kParquetZSTDCompressionLevel);
auto compressionLevel = it != sparkConfs.end() ? std::stoi(it->second) : kZSTDDefaultCompressionLevel;
codecOptions->compression_level = compressionLevel;
codecOptions->compressionLevel = compressionLevel;
writeOption->codecOptions = std::move(codecOptions);
} else if (boost::iequals(compressionCodecStr, "uncompressed")) {
compressionCodec = CompressionKind::CompressionKind_NONE;
Expand Down
4 changes: 2 additions & 2 deletions ep/build-velox/src/get-velox.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ set -exu

CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
VELOX_REPO=https://github.com/IBM/velox.git
VELOX_BRANCH=dft-2026_02_06
VELOX_ENHANCED_BRANCH=ibm-2026_02_06
VELOX_BRANCH=dft-2026_02_17
VELOX_ENHANCED_BRANCH=ibm-2026_02_17
VELOX_HOME=""
RUN_SETUP_SCRIPT=ON
ENABLE_ENHANCED_FEATURES=OFF
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,12 +141,12 @@ from
-- !query schema
struct<a:int,rand(0):double,sum(b):bigint>
-- !query output
1 0.5488135024422883 1
1 0.7151893651681639 2
2 0.5448831775801376 2
2 0.6027633705776989 1
3 0.4236547969336536 1
3 0.6458941151817286 2
1 0.5234194256885571 2
1 0.7604953758285915 1
2 0.0953472826424725 1
2 0.3163249920547614 2
3 0.2710259815484829 2
3 0.7141011170991605 1


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -662,5 +662,5 @@ GROUP BY a IS NULL
-- !query schema
struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint>
-- !query output
0.5488135024422883 7
0.7604953758285915 7
1.0 2
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,31 @@ SELECT rand(0)
-- !query schema
struct<rand(0):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(3 / 7 AS int))
-- !query schema
struct<rand(CAST((3 / 7) AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(NULL)
-- !query schema
struct<rand(NULL):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(NULL AS int))
-- !query schema
struct<rand(CAST(NULL AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,12 +141,12 @@ from
-- !query schema
struct<a:int,rand(0):double,sum(b):bigint>
-- !query output
1 0.5488135024422883 1
1 0.7151893651681639 2
2 0.5448831775801376 2
2 0.6027633705776989 1
3 0.4236547969336536 1
3 0.6458941151817286 2
1 0.5234194256885571 2
1 0.7604953758285915 1
2 0.0953472826424725 1
2 0.3163249920547614 2
3 0.2710259815484829 2
3 0.7141011170991605 1


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -691,7 +691,7 @@ GROUP BY a IS NULL
-- !query schema
struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint>
-- !query output
0.5488135024422883 7
0.7604953758285915 7
1.0 2


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,31 @@ SELECT rand(0)
-- !query schema
struct<rand(0):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(3 / 7 AS int))
-- !query schema
struct<rand(CAST((3 / 7) AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(NULL)
-- !query schema
struct<rand(NULL):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(NULL AS int))
-- !query schema
struct<rand(CAST(NULL AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,12 +208,12 @@ from
-- !query schema
struct<a:int,rand(0):double,sum(b):bigint>
-- !query output
1 0.5488135024422883 1
1 0.7151893651681639 2
2 0.5448831775801376 2
2 0.6027633705776989 1
3 0.4236547969336536 1
3 0.6458941151817286 2
1 0.5234194256885571 2
1 0.7604953758285915 1
2 0.0953472826424725 1
2 0.3163249920547614 2
3 0.2710259815484829 2
3 0.7141011170991605 1

-- !query
select * from data group by a, b, 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -907,7 +907,7 @@ GROUP BY a IS NULL
-- !query schema
struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint>
-- !query output
0.5488135024422883 7
0.7604953758285915 7
1.0 2


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,31 @@ SELECT rand(0)
-- !query schema
struct<rand(0):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(3 / 7 AS int))
-- !query schema
struct<rand(CAST((3 / 7) AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(NULL)
-- !query schema
struct<rand(NULL):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(NULL AS int))
-- !query schema
struct<rand(CAST(NULL AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,12 +208,12 @@ from
-- !query schema
struct<a:int,rand(0):double,sum(b):bigint>
-- !query output
1 0.5488135024422883 1
1 0.7151893651681639 2
2 0.5448831775801376 2
2 0.6027633705776989 1
3 0.4236547969336536 1
3 0.6458941151817286 2
1 0.5234194256885571 2
1 0.7604953758285915 1
2 0.0953472826424725 1
2 0.3163249920547614 2
3 0.2710259815484829 2
3 0.7141011170991605 1


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -907,7 +907,7 @@ GROUP BY a IS NULL
-- !query schema
struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint>
-- !query output
0.5488135024422883 7
0.7604953758285915 7
1.0 2


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,31 @@ SELECT rand(0)
-- !query schema
struct<rand(0):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(3 / 7 AS int))
-- !query schema
struct<rand(CAST((3 / 7) AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(NULL)
-- !query schema
struct<rand(NULL):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
SELECT rand(cast(NULL AS int))
-- !query schema
struct<rand(CAST(NULL AS INT)):double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ select * from explode(array(rand(0)))
-- !query schema
struct<col:double>
-- !query output
0.5488135024422883
0.7604953758285915


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class GlutenGeneratorFunctionSuite extends GeneratorFunctionSuite with GlutenSQL
testGluten("SPARK-45171: Handle evaluated nondeterministic expression") {
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
val df = sql("select explode(array(rand(0)))")
checkAnswer(df, Row(0.5488135024422883))
checkAnswer(df, Row(0.7604953758285915))
}
}
}
Loading
Loading