diff --git a/devops b/devops index 0326d58a..468a9314 160000 --- a/devops +++ b/devops @@ -1 +1 @@ -Subproject commit 0326d58af66ab94eb63c569ca79e2790ec5bc405 +Subproject commit 468a9314f6669b61683eaeea5aa262a462879ab2 diff --git a/server/Makefile b/server/Makefile index 4e6ee040..7caff54a 100644 --- a/server/Makefile +++ b/server/Makefile @@ -36,7 +36,10 @@ help: @echo " - test - runs unit tests for all PXF modules" @echo " - coverage - runs unit tests for all PXF modules and gives a coverage report" @echo " - install - setup PXF in the configured deployPath" - @echo " - stage - build PXF server and stage resulting artifacts for packaging" + @echo " - stage - build PXF server and stage resulting artifacts for packaging (no JDBC drivers in build/stage/lib)" + @echo " - stage-notest - same as stage but skip tests (no JDBC drivers in build/stage/lib)" + @echo " - stage-jdbc-drivers - copy JDBC driver JARs to build/stage/lib (run after stage if full layout needed)" + @echo " - install-jdbc-drivers - setup PXF JDBC drivers in the configured PXF_HOME" @echo " - doc - creates aggregate javadoc under docs" all: @@ -91,6 +94,18 @@ stage-notest: install -m 700 -d "build/stage/run" install -m 700 -d "build/stage/keytabs" +.PHONY: stage-jdbc-drivers +stage-jdbc-drivers: + ./gradlew $(PXF_GRADLE_PROPERTIES) stageJdbcDrivers + +.PHONY: install +install-jdbc-drivers: stage-jdbc-drivers + @if [ -z "$(PXF_HOME)" ]; then \ + echo "ERROR: PXF_HOME is not set"; exit 2; \ + fi + mkdir -p "$(PXF_HOME)"/lib + cp -R build/stage/lib/* "$(PXF_HOME)"/lib + clean: ./gradlew clean rm -rf build diff --git a/server/build.gradle b/server/build.gradle index 012e866d..e8034a9d 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -92,7 +92,7 @@ configure(javaProjects) { dependency("commons-lang:commons-lang:2.6") dependency("commons-lang:commons-lang3:3.9") dependency("commons-logging:commons-logging:1.1.3") - dependency("io.airlift:aircompressor:2.0.2") + dependency("io.airlift:aircompressor:0.27") dependency("javax.jdo:jdo-api:3.0.1") dependency("joda-time:joda-time:2.8.1") dependency("net.sf.opencsv:opencsv:2.3") @@ -114,11 +114,15 @@ configure(javaProjects) { // revert once springboot provided postgresql is upgraded to bundle postgresql:42.7.2+ dependency("org.postgresql:postgresql:42.7.2") + dependency("org.apache.tomcat.embed:tomcat-embed-core:9.0.72") + dependency("org.apache.tomcat.embed:tomcat-embed-el:9.0.72") + dependency("org.apache.tomcat.embed:tomcat-embed-websocket:9.0.72") + dependency("org.simplify4u:slf4j-mock:2.3.0") dependency("org.threeten:threeten-extra:1.5.0") dependency("org.tukaani:xz:1.8") dependency("org.wildfly.openssl:wildfly-openssl:1.0.7.Final") - dependency("org.xerial.snappy:snappy-java:1.1.10.7") + dependency("org.xerial.snappy:snappy-java:1.1.10.4") // Hadoop dependencies dependencySet(group:"org.apache.hadoop", version:"${hadoopVersion}") { @@ -208,8 +212,9 @@ configure(javaProjects) { entry("avro") entry("avro-mapred") } - // Zstd support for Avro/Parquet + // Compression support for Avro/Parquet dependency("com.github.luben:zstd-jni:1.5.7-6") + dependency("at.yawk.lz4:lz4-java:1.10.3") // Jackson 1.x dependencies dependencySet(group:"org.codehaus.jackson", version:"1.9.13") { @@ -364,3 +369,11 @@ task stage(type: Copy) { doLast { new File("${buildDir}/stage/version").text = "${version}\n" } } + +task stageJdbcDrivers(type: Copy) { + description "Copy JDBC driver JARs from pxf-jdbc-drivers to build/stage/lib." + dependsOn ":pxf-jdbc-drivers:jar" + from(project(':pxf-jdbc-drivers').configurations.runtimeClasspath) + into("${buildDir}/stage/lib") + duplicatesStrategy = DuplicatesStrategy.INCLUDE +} diff --git a/server/pxf-jdbc-drivers/build.gradle b/server/pxf-jdbc-drivers/build.gradle new file mode 100644 index 00000000..b3f9a9a7 --- /dev/null +++ b/server/pxf-jdbc-drivers/build.gradle @@ -0,0 +1,34 @@ +dependencies { + // PostgreSQL driver is used by PXF itself. + // Its version is defined in server/build.gradle + + // Clickhouse + implementation('com.clickhouse:clickhouse-jdbc:0.6.5') + implementation('com.clickhouse:clickhouse-jdbc:0.6.5:http') + implementation('at.yawk.lz4:lz4-java') + implementation('org.apache.httpcomponents.client5:httpclient5:5.3.1') { transitive = false } + implementation('org.apache.httpcomponents.client5:httpcore5:5.2.4') { transitive = false } + implementation('org.apache.httpcomponents.client5:httpcore5-h2:5.2.4') { transitive = false } + + + // DB2 + implementation('net.sf.jt400:jt400:11.2') + + // MSSQL + implementation('com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11') + + // MySQL + implementation('mysql:mysql-connector-java:8.0.26') { transitive = false } + + // Snowflake + implementation('net.snowflake:snowflake-jdbc:3.13.9') + + // ORA + implementation('com.oracle.database.jdbc:ojdbc11:23.3.0.23.09') + implementation('com.oracle.database.jdbc:ucp11:23.3.0.23.09') + implementation('com.oracle.database.nls:orai18n:23.3.0.23.09') + + // trino + implementation('io.trino:trino-jdbc:403') + +} \ No newline at end of file diff --git a/server/settings.gradle b/server/settings.gradle index dd419679..5df3ff21 100644 --- a/server/settings.gradle +++ b/server/settings.gradle @@ -25,6 +25,7 @@ include 'pxf-service', 'pxf-hdfs', 'pxf-hive', 'pxf-jdbc', + 'pxf-jdbc-drivers', 'pxf-json', 'pxf-s3', 'pxf-diagnostic'