From 138936fa5b77ea1e95cdd8eaf76d64cb31c76bb0 Mon Sep 17 00:00:00 2001 From: Dovi Joel Date: Wed, 29 Jul 2020 08:50:31 +0200 Subject: [PATCH 1/3] added profiles for spark24 and spark30 as profiles. This build on changes from dovijoel. rebaselined with main which is now JDBC 8.4.1 based. --- .github/workflows/scala2.yml | 21 ++++++ .gitignore | 2 +- build.sbt | 32 --------- pom.xml | 65 +++++++++++++------ .../jdbc/spark/bulkwrite/DataSourceTest.scala | 4 +- 5 files changed, 70 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/scala2.yml delete mode 100644 build.sbt diff --git a/.github/workflows/scala2.yml b/.github/workflows/scala2.yml new file mode 100644 index 0000000..f182d0c --- /dev/null +++ b/.github/workflows/scala2.yml @@ -0,0 +1,21 @@ +name: Scala CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Run tests + run: sbt test diff --git a/.gitignore b/.gitignore index c9bb642..358bdee 100644 --- a/.gitignore +++ b/.gitignore @@ -352,4 +352,4 @@ MigrationBackup/ project/project/ project/target/ target/ - +.idea diff --git a/build.sbt b/build.sbt deleted file mode 100644 index 5425a00..0000000 --- a/build.sbt +++ /dev/null @@ -1,32 +0,0 @@ -name := "spark-mssql-connector" - -organization := "com.microsoft.sqlserver.jdbc.spark" - -version := "1.0.0" - -scalaVersion := "2.11.12" - -val sparkVersion = "2.4.6" - -javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint") - -libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", - // Spark Testing Utilities - "org.apache.spark" %% "spark-core" % sparkVersion % "test" classifier - "tests", - "org.apache.spark" %% "spark-sql" % sparkVersion% "test" classifier - "tests", - "org.apache.spark" %% "spark-catalyst" % sparkVersion % "test" classifier - "tests", - "org.scalatest" %% "scalatest" % "3.0.5" % "test", - "com.novocode" % "junit-interface" % "0.11" % "test", - - //SQLServer JDBC jars - "com.microsoft.sqlserver" % "mssql-jdbc" % "8.4.1.jre8" -) - -scalacOptions := Seq("-unchecked", "-deprecation", "evicted") - -// Exclude scala-library from this fat jar. The scala library is already there in spark package. -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) diff --git a/pom.xml b/pom.xml index f899490..6f3f04a 100644 --- a/pom.xml +++ b/pom.xml @@ -13,16 +13,10 @@ - Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt + MIT License + http://www.opensource.org/licenses/mit-license.php - - UTF-8 - 2.11 - 2.11.12 - 2.4.6 - org.scala-lang @@ -56,23 +50,12 @@ test tests - - org.scalatest - scalatest_${scala.binary.version} - 3.0.5 - test - com.novocode junit-interface 0.11 test - - com.microsoft.sqlserver - mssql-jdbc - 8.4.1.jre8 - @@ -213,4 +196,48 @@ + + + spark24 + + 2.11 + 2.11.12 + 2.4.6 + + + + org.scalatest + scalatest_${scala.binary.version} + 3.0.5 + test + + + com.microsoft.sqlserver + mssql-jdbc + 8.4.1.jre8 + + + + + spark30 + + 2.12 + 2.12.11 + 3.0.0 + + + + org.scalatest + scalatest_${scala.binary.version} + 3.0.8 + test + + + com.microsoft.sqlserver + mssql-jdbc + 8.4.1.jre8 + + + + diff --git a/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala b/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala index 2d900bc..d033cfc 100644 --- a/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala +++ b/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala @@ -16,9 +16,9 @@ import java.sql.Connection import org.scalatest.Matchers import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.test.SharedSQLContext +import org.apache.spark.sql.test.SharedSparkSession -class DataSourceTest extends SparkFunSuite with Matchers with SharedSQLContext { +class DataSourceTest extends SparkFunSuite with Matchers with SharedSparkSession { test("Schema validation between Spark DataFrame and SQL Server ResultSet") {} From 7a2409d7e041064d7780c9f8a930f2f1e9bf5046 Mon Sep 17 00:00:00 2001 From: shivsood Date: Wed, 19 May 2021 20:36:45 +0000 Subject: [PATCH 2/3] removed workflow fixes --- .github/workflows/scala2.yml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/scala2.yml diff --git a/.github/workflows/scala2.yml b/.github/workflows/scala2.yml deleted file mode 100644 index f182d0c..0000000 --- a/.github/workflows/scala2.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Scala CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Run tests - run: sbt test From 97857d212847977efaf47d9a173fc0c08b771e92 Mon Sep 17 00:00:00 2001 From: shivsood Date: Wed, 19 May 2021 21:48:51 +0000 Subject: [PATCH 3/3] fixes to pom.xml --- pom.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6f3f04a..d14c735 100644 --- a/pom.xml +++ b/pom.xml @@ -13,10 +13,13 @@ - MIT License - http://www.opensource.org/licenses/mit-license.php + Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + UTF-8 + org.scala-lang