updated the project to work with spark3.1.1 and scala 2.12

build.sbt

@@ -1,37 +1,58 @@
-name := "spark-sftp"
+lazy val scala211Version = "2.11.12"
+lazy val scala212Version = "2.12.10"
+lazy val sparkVersion = "3.1.1"
 
-organization := "com.springml"
 
-scalaVersion := "2.11.8"
 
-sparkVersion := "2.3.0"
+lazy val commonSettings = Seq(
+  name := "spark-sftp",
+  organization := "com.springml",
+  version := "1.3.0",
+  scalaVersion := scala212Version,
+  crossScalaVersions := Seq(scala211Version, scala212Version)
+)
+
+javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint")
+
+initialize := {
+  val _ = initialize.value
+  val javaVersion = sys.props("java.specification.version")
+  if (javaVersion != "1.8")
+    sys.error("Java 1.8 is required for this project. Found " + javaVersion + " instead")
+}
+
 
-spName := "springml/spark-sftp"
+lazy val shaded = (project in file("."))
+  .settings(commonSettings)
 
-version := "1.1.4"
+// Test dependencies
+lazy val commonTestDependencies = Seq(
+  "org.scalatest" %% "scalatest" % "3.0.5" % "test",
+  "org.apache.avro" % "avro-mapred" % "1.7.7" % "test" exclude("org.mortbay.jetty", "servlet-api"),
+  "org.apache.spark" %% "spark-hive" % sparkVersion % "test"
+)
 
 // Dependent libraries
-libraryDependencies ++= Seq(
+libraryDependencies ++= (commonTestDependencies ++ Seq(
+  //spark libs
+  "org.apache.spark" %% "spark-core" % sparkVersion,
+  "org.apache.spark" %% "spark-sql" % sparkVersion,
+  //spark -dependents
+  "org.apache.spark" %% "spark-avro" % sparkVersion,
   "com.springml" % "sftp.client" % "1.0.3",
   "org.mockito" % "mockito-core" % "2.0.31-beta",
-  "com.databricks" % "spark-xml_2.11" % "0.4.1"
-)
+  "com.databricks" %% "spark-xml" % "0.5.0"
+))
 
-// used spark components
-sparkComponents += "sql"
 
 // Repositories
 resolvers += "Spark Package Main Repo" at "https://dl.bintray.com/spark-packages/maven"
 
-// Spark packages
-spDependencies += "com.databricks/spark-avro_2.11:3.2.0"
 
-// Test dependencies
-libraryDependencies += "org.scalatest" %% "scalatest" % "2.2.1" % "test"
-libraryDependencies += "org.apache.avro" % "avro-mapred" % "1.7.7" % "test" exclude("org.mortbay.jetty", "servlet-api")
-libraryDependencies +=  "org.apache.spark" %% "spark-hive" % sparkVersion.value % "test"
 
-spIgnoreProvided := true
+
+
+
 // licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0"))
 
 credentials += Credentials(Path.userHome / ".ivy2" / ".credentials")

project/build.properties

@@ -1,2 +1,2 @@
 // This file should only contain the version of sbt to use.
-sbt.version=0.13.13
+sbt.version=1.4.7

project/plugins.sbt

@@ -1,7 +1,5 @@
 // You may use this file to add plugin dependencies for sbt.
 resolvers += "bintray-spark-packages" at "https://dl.bintray.com/spark-packages/maven/"
+resolvers += Resolver.mavenLocal
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.7")
 
-addSbtPlugin("org.spark-packages" %% "sbt-spark-package" % "0.2.5")
-addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0")
-addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
-addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")

src/main/scala/com/springml/spark/sftp/DatasetRelation.scala

@@ -1,6 +1,6 @@
 package com.springml.spark.sftp
 
-import com.databricks.spark.avro._
+
 import org.apache.log4j.Logger
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Row, SQLContext}
@@ -36,7 +36,7 @@ case class DatasetRelation(
       var df: DataFrame = null
 
       df = fileType match {
-        case "avro" => dataframeReader.avro(fileLocation)
+        case "avro" => dataframeReader.format("avro").load(fileLocation)
         case "txt" => dataframeReader.format("text").load(fileLocation)
         case "xml" => dataframeReader.format(constants.xmlClass)
           .option(constants.xmlRowTag, rowTag)