From 62b92def4131d90733736708fed03c8236ead280 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Sat, 12 Apr 2025 13:56:59 +0500 Subject: [PATCH 1/9] WIP: Add pxf-dirvers --- server/pxf-drivers/.gitignore | 2 ++ server/pxf-drivers/build.gradle | 31 +++++++++++++++++++++++++++++++ server/settings.gradle | 1 + 3 files changed, 34 insertions(+) create mode 100644 server/pxf-drivers/.gitignore create mode 100644 server/pxf-drivers/build.gradle diff --git a/server/pxf-drivers/.gitignore b/server/pxf-drivers/.gitignore new file mode 100644 index 000000000..4adf20cd1 --- /dev/null +++ b/server/pxf-drivers/.gitignore @@ -0,0 +1,2 @@ +/build +/bin/ \ No newline at end of file diff --git a/server/pxf-drivers/build.gradle b/server/pxf-drivers/build.gradle new file mode 100644 index 000000000..7e53f23c0 --- /dev/null +++ b/server/pxf-drivers/build.gradle @@ -0,0 +1,31 @@ + +dependencies { + // AthenaJDBC42.jar + implementation 'com.clickhouse:clickhouse-jdbc:0.6.1' + // clickhouse-jdbc-0.6.1-http.jar ?? + // commons-cli-1.5.0.jar ?? + + implementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' + + // DB2 + implementation 'net.sf.jt400:jt400:11.2' + + // MSSQL + implementation 'com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11' + + // MySQL + implementation 'com.mysql:mysql-connector-java:8.0.26' + + // Snowflake + implementation 'net.snowflake:snowflake-jdbc:3.13.9' + + // ORA + implementation 'com.oracle.database.jdbc:ojdbc11:23.3.0.23.09' + implementation 'com.oracle.database.jdbc:ucp11:23.3.0.23.09' + // orai18n - unknown version + + + // trino + testImplementation 'io.trino:trino-jdbc:403' + +} \ No newline at end of file diff --git a/server/settings.gradle b/server/settings.gradle index dd4196798..16328286c 100644 --- a/server/settings.gradle +++ b/server/settings.gradle @@ -28,3 +28,4 @@ include 'pxf-service', 'pxf-json', 'pxf-s3', 'pxf-diagnostic' + 'pxf-drivers' From 535a4b50fbe53313338cfd7b6a847718edb5729d Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Thu, 19 Feb 2026 18:55:47 +0500 Subject: [PATCH 2/9] pxf-jdbc-drivers --- server/Makefile | 12 +++++++++++- server/build.gradle | 8 ++++++++ server/settings.gradle | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/server/Makefile b/server/Makefile index 4e6ee0403..1ad45466a 100644 --- a/server/Makefile +++ b/server/Makefile @@ -36,7 +36,10 @@ help: @echo " - test - runs unit tests for all PXF modules" @echo " - coverage - runs unit tests for all PXF modules and gives a coverage report" @echo " - install - setup PXF in the configured deployPath" - @echo " - stage - build PXF server and stage resulting artifacts for packaging" + @echo " - stage - build PXF server and stage resulting artifacts for packaging (no JDBC drivers in build/stage/lib)" + @echo " - stage-notest - same as stage but skip tests (no JDBC drivers in build/stage/lib)" + @echo " - stage-jdbc-drivers - copy JDBC driver JARs to build/stage/lib (run after stage if full layout needed)" + @echo " - stage-with-jdbc-drivers - full stage without tests plus JDBC drivers in build/stage/lib" @echo " - doc - creates aggregate javadoc under docs" all: @@ -91,6 +94,13 @@ stage-notest: install -m 700 -d "build/stage/run" install -m 700 -d "build/stage/keytabs" +.PHONY: stage-jdbc-drivers +stage-jdbc-drivers: + ./gradlew $(PXF_GRADLE_PROPERTIES) stageJdbcDrivers + +.PHONY: stage-with-jdbc-drivers +stage-with-jdbc-drivers: stage-notest stage-jdbc-drivers + clean: ./gradlew clean rm -rf build diff --git a/server/build.gradle b/server/build.gradle index 012e866d4..79d3fdeb6 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -364,3 +364,11 @@ task stage(type: Copy) { doLast { new File("${buildDir}/stage/version").text = "${version}\n" } } + +task stageJdbcDrivers(type: Copy) { + description "Copy JDBC driver JARs from pxf-jdbc-drivers to build/stage/lib." + dependsOn ":pxf-jdbc-drivers:jar" + from(project(':pxf-jdbc-drivers').configurations.runtimeClasspath) + into("${buildDir}/stage/lib") + duplicatesStrategy = DuplicatesStrategy.INCLUDE +} diff --git a/server/settings.gradle b/server/settings.gradle index 16328286c..ba5b6565f 100644 --- a/server/settings.gradle +++ b/server/settings.gradle @@ -25,6 +25,7 @@ include 'pxf-service', 'pxf-hdfs', 'pxf-hive', 'pxf-jdbc', + 'pxf-jdbc-drivers', 'pxf-json', 'pxf-s3', 'pxf-diagnostic' From 43bad268fe37df5d69d7f3cb21ecc7f81ffa794b Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Thu, 19 Feb 2026 18:58:29 +0500 Subject: [PATCH 3/9] drivers --- server/pxf-jdbc-drivers/build.gradle | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 server/pxf-jdbc-drivers/build.gradle diff --git a/server/pxf-jdbc-drivers/build.gradle b/server/pxf-jdbc-drivers/build.gradle new file mode 100644 index 000000000..41b95e241 --- /dev/null +++ b/server/pxf-jdbc-drivers/build.gradle @@ -0,0 +1,34 @@ +dependencies { + // PostgreSQL driver is used by PXF itself. + // Its version is defined in server/build.gradle + + + // AthenaJDBC42.jar + implementation 'com.clickhouse:clickhouse-jdbc:0.6.1' + // clickhouse-jdbc-0.6.1-http.jar ?? + // commons-cli-1.5.0.jar ?? + + implementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' + + // DB2 + implementation 'net.sf.jt400:jt400:11.2' + + // MSSQL + implementation 'com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11' + + // MySQL + implementation 'com.mysql:mysql-connector-java:8.0.26' + + // Snowflake + implementation 'net.snowflake:snowflake-jdbc:3.13.9' + + // ORA + implementation 'com.oracle.database.jdbc:ojdbc11:23.3.0.23.09' + implementation 'com.oracle.database.jdbc:ucp11:23.3.0.23.09' + // orai18n - unknown version + + + // trino + testImplementation 'io.trino:trino-jdbc:403' + +} \ No newline at end of file From 8a3aed60228c46e3b7fa352191a815fff4cfcf31 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Fri, 20 Feb 2026 22:33:51 +0500 Subject: [PATCH 4/9] better --- server/pxf-jdbc-drivers/build.gradle | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/server/pxf-jdbc-drivers/build.gradle b/server/pxf-jdbc-drivers/build.gradle index 41b95e241..023402f17 100644 --- a/server/pxf-jdbc-drivers/build.gradle +++ b/server/pxf-jdbc-drivers/build.gradle @@ -2,11 +2,12 @@ dependencies { // PostgreSQL driver is used by PXF itself. // Its version is defined in server/build.gradle + // Amazon Athena + // AthenaJDBC42.jar -> Athena:AthenaJDBC42:2.0.31.1001 (without AWS SDK bundled) - // AthenaJDBC42.jar - implementation 'com.clickhouse:clickhouse-jdbc:0.6.1' - // clickhouse-jdbc-0.6.1-http.jar ?? - // commons-cli-1.5.0.jar ?? + // Clickhouse + implementation 'com.clickhouse:clickhouse-jdbc:0.6.5' + implementation 'com.clickhouse:clickhouse-jdbc:0.6.5:http' implementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' @@ -14,10 +15,10 @@ dependencies { implementation 'net.sf.jt400:jt400:11.2' // MSSQL - implementation 'com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11' + implementation('com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11') // MySQL - implementation 'com.mysql:mysql-connector-java:8.0.26' + implementation 'mysql:mysql-connector-java:8.0.26' // Snowflake implementation 'net.snowflake:snowflake-jdbc:3.13.9' @@ -25,10 +26,10 @@ dependencies { // ORA implementation 'com.oracle.database.jdbc:ojdbc11:23.3.0.23.09' implementation 'com.oracle.database.jdbc:ucp11:23.3.0.23.09' - // orai18n - unknown version + implementation 'com.oracle.database.nls:orai18n:23.3.0.23.09' // trino - testImplementation 'io.trino:trino-jdbc:403' + implementation 'io.trino:trino-jdbc:403' } \ No newline at end of file From ead4cbf7dbf21eefffa6f748760a4114321ef8f2 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Mon, 23 Feb 2026 15:56:52 +0500 Subject: [PATCH 5/9] add dependencies --- server/build.gradle | 11 ++++++++--- server/pxf-jdbc-drivers/build.gradle | 27 +++++++++++++-------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index 79d3fdeb6..e8034a9df 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -92,7 +92,7 @@ configure(javaProjects) { dependency("commons-lang:commons-lang:2.6") dependency("commons-lang:commons-lang3:3.9") dependency("commons-logging:commons-logging:1.1.3") - dependency("io.airlift:aircompressor:2.0.2") + dependency("io.airlift:aircompressor:0.27") dependency("javax.jdo:jdo-api:3.0.1") dependency("joda-time:joda-time:2.8.1") dependency("net.sf.opencsv:opencsv:2.3") @@ -114,11 +114,15 @@ configure(javaProjects) { // revert once springboot provided postgresql is upgraded to bundle postgresql:42.7.2+ dependency("org.postgresql:postgresql:42.7.2") + dependency("org.apache.tomcat.embed:tomcat-embed-core:9.0.72") + dependency("org.apache.tomcat.embed:tomcat-embed-el:9.0.72") + dependency("org.apache.tomcat.embed:tomcat-embed-websocket:9.0.72") + dependency("org.simplify4u:slf4j-mock:2.3.0") dependency("org.threeten:threeten-extra:1.5.0") dependency("org.tukaani:xz:1.8") dependency("org.wildfly.openssl:wildfly-openssl:1.0.7.Final") - dependency("org.xerial.snappy:snappy-java:1.1.10.7") + dependency("org.xerial.snappy:snappy-java:1.1.10.4") // Hadoop dependencies dependencySet(group:"org.apache.hadoop", version:"${hadoopVersion}") { @@ -208,8 +212,9 @@ configure(javaProjects) { entry("avro") entry("avro-mapred") } - // Zstd support for Avro/Parquet + // Compression support for Avro/Parquet dependency("com.github.luben:zstd-jni:1.5.7-6") + dependency("at.yawk.lz4:lz4-java:1.10.3") // Jackson 1.x dependencies dependencySet(group:"org.codehaus.jackson", version:"1.9.13") { diff --git a/server/pxf-jdbc-drivers/build.gradle b/server/pxf-jdbc-drivers/build.gradle index 023402f17..b3f9a9a76 100644 --- a/server/pxf-jdbc-drivers/build.gradle +++ b/server/pxf-jdbc-drivers/build.gradle @@ -2,34 +2,33 @@ dependencies { // PostgreSQL driver is used by PXF itself. // Its version is defined in server/build.gradle - // Amazon Athena - // AthenaJDBC42.jar -> Athena:AthenaJDBC42:2.0.31.1001 (without AWS SDK bundled) - // Clickhouse - implementation 'com.clickhouse:clickhouse-jdbc:0.6.5' - implementation 'com.clickhouse:clickhouse-jdbc:0.6.5:http' + implementation('com.clickhouse:clickhouse-jdbc:0.6.5') + implementation('com.clickhouse:clickhouse-jdbc:0.6.5:http') + implementation('at.yawk.lz4:lz4-java') + implementation('org.apache.httpcomponents.client5:httpclient5:5.3.1') { transitive = false } + implementation('org.apache.httpcomponents.client5:httpcore5:5.2.4') { transitive = false } + implementation('org.apache.httpcomponents.client5:httpcore5-h2:5.2.4') { transitive = false } - implementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' // DB2 - implementation 'net.sf.jt400:jt400:11.2' + implementation('net.sf.jt400:jt400:11.2') // MSSQL implementation('com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11') // MySQL - implementation 'mysql:mysql-connector-java:8.0.26' + implementation('mysql:mysql-connector-java:8.0.26') { transitive = false } // Snowflake - implementation 'net.snowflake:snowflake-jdbc:3.13.9' + implementation('net.snowflake:snowflake-jdbc:3.13.9') // ORA - implementation 'com.oracle.database.jdbc:ojdbc11:23.3.0.23.09' - implementation 'com.oracle.database.jdbc:ucp11:23.3.0.23.09' - implementation 'com.oracle.database.nls:orai18n:23.3.0.23.09' - + implementation('com.oracle.database.jdbc:ojdbc11:23.3.0.23.09') + implementation('com.oracle.database.jdbc:ucp11:23.3.0.23.09') + implementation('com.oracle.database.nls:orai18n:23.3.0.23.09') // trino - implementation 'io.trino:trino-jdbc:403' + implementation('io.trino:trino-jdbc:403') } \ No newline at end of file From 9ece8127376fd981ed68e8ff52db46daaf81244f Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Tue, 24 Feb 2026 13:45:03 +0500 Subject: [PATCH 6/9] cleanup --- server/pxf-drivers/.gitignore | 2 -- server/pxf-drivers/build.gradle | 31 ------------------------------- server/settings.gradle | 1 - 3 files changed, 34 deletions(-) delete mode 100644 server/pxf-drivers/.gitignore delete mode 100644 server/pxf-drivers/build.gradle diff --git a/server/pxf-drivers/.gitignore b/server/pxf-drivers/.gitignore deleted file mode 100644 index 4adf20cd1..000000000 --- a/server/pxf-drivers/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/build -/bin/ \ No newline at end of file diff --git a/server/pxf-drivers/build.gradle b/server/pxf-drivers/build.gradle deleted file mode 100644 index 7e53f23c0..000000000 --- a/server/pxf-drivers/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ - -dependencies { - // AthenaJDBC42.jar - implementation 'com.clickhouse:clickhouse-jdbc:0.6.1' - // clickhouse-jdbc-0.6.1-http.jar ?? - // commons-cli-1.5.0.jar ?? - - implementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' - - // DB2 - implementation 'net.sf.jt400:jt400:11.2' - - // MSSQL - implementation 'com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11' - - // MySQL - implementation 'com.mysql:mysql-connector-java:8.0.26' - - // Snowflake - implementation 'net.snowflake:snowflake-jdbc:3.13.9' - - // ORA - implementation 'com.oracle.database.jdbc:ojdbc11:23.3.0.23.09' - implementation 'com.oracle.database.jdbc:ucp11:23.3.0.23.09' - // orai18n - unknown version - - - // trino - testImplementation 'io.trino:trino-jdbc:403' - -} \ No newline at end of file diff --git a/server/settings.gradle b/server/settings.gradle index ba5b6565f..5df3ff21b 100644 --- a/server/settings.gradle +++ b/server/settings.gradle @@ -29,4 +29,3 @@ include 'pxf-service', 'pxf-json', 'pxf-s3', 'pxf-diagnostic' - 'pxf-drivers' From b0914ef8da9682a8d8d5305eec8b73505edf643d Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Thu, 26 Feb 2026 17:12:01 +0500 Subject: [PATCH 7/9] bump devops scripts --- devops | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops b/devops index 0326d58af..b9a924c07 160000 --- a/devops +++ b/devops @@ -1 +1 @@ -Subproject commit 0326d58af66ab94eb63c569ca79e2790ec5bc405 +Subproject commit b9a924c071af99eca5dd70398fcd83ee716407ec From 305aae0249f518d2a842643b39f3cad5e10ea241 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Fri, 27 Feb 2026 19:02:20 +0500 Subject: [PATCH 8/9] Fix jdbc-drivers install script --- devops | 2 +- server/Makefile | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/devops b/devops index b9a924c07..468a9314f 160000 --- a/devops +++ b/devops @@ -1 +1 @@ -Subproject commit b9a924c071af99eca5dd70398fcd83ee716407ec +Subproject commit 468a9314f6669b61683eaeea5aa262a462879ab2 diff --git a/server/Makefile b/server/Makefile index 1ad45466a..cbfa91615 100644 --- a/server/Makefile +++ b/server/Makefile @@ -40,6 +40,7 @@ help: @echo " - stage-notest - same as stage but skip tests (no JDBC drivers in build/stage/lib)" @echo " - stage-jdbc-drivers - copy JDBC driver JARs to build/stage/lib (run after stage if full layout needed)" @echo " - stage-with-jdbc-drivers - full stage without tests plus JDBC drivers in build/stage/lib" + @echo " - install-jdbc-drivers - setup PXF JDBC drivers in the configured PXF_HOME" @echo " - doc - creates aggregate javadoc under docs" all: @@ -98,8 +99,13 @@ stage-notest: stage-jdbc-drivers: ./gradlew $(PXF_GRADLE_PROPERTIES) stageJdbcDrivers -.PHONY: stage-with-jdbc-drivers -stage-with-jdbc-drivers: stage-notest stage-jdbc-drivers +.PHONY: install +install-jdbc-drivers: stage-jdbc-drivers + @if [ -z "$(PXF_HOME)" ]; then \ + echo "ERROR: PXF_HOME is not set"; exit 2; \ + fi + mkdir -p "$(PXF_HOME)"/lib + cp -R build/stage/lib/* "$(PXF_HOME)"/lib clean: ./gradlew clean From 1417cb4b48de12049b9f85b727cd4ea76b547697 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Fri, 27 Feb 2026 19:03:25 +0500 Subject: [PATCH 9/9] cleanup --- server/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/server/Makefile b/server/Makefile index cbfa91615..7caff54a0 100644 --- a/server/Makefile +++ b/server/Makefile @@ -39,7 +39,6 @@ help: @echo " - stage - build PXF server and stage resulting artifacts for packaging (no JDBC drivers in build/stage/lib)" @echo " - stage-notest - same as stage but skip tests (no JDBC drivers in build/stage/lib)" @echo " - stage-jdbc-drivers - copy JDBC driver JARs to build/stage/lib (run after stage if full layout needed)" - @echo " - stage-with-jdbc-drivers - full stage without tests plus JDBC drivers in build/stage/lib" @echo " - install-jdbc-drivers - setup PXF JDBC drivers in the configured PXF_HOME" @echo " - doc - creates aggregate javadoc under docs"