Spark SBT Build File Example

build.sbt

name := "ImpressionMetric"

version := "1.0"

scalaVersion := "2.10.6"


// Spark Dependencies

libraryDependencies ++= Seq(
//  "org.apache.spark" %% "spark-core" % "1.6.1" % "provided",
//  "org.apache.spark" %% "spark-sql" % "1.6.1" % "provided"
    "org.apache.spark" %% "spark-core" % "1.6.1",
    "org.apache.spark" %% "spark-sql" % "1.6.1"
)
// Third-party Dependencies

libraryDependencies ++= Seq(
  "com.databricks" % "spark-csv_2.10" % "1.4.0",
  "com.memsql" % "memsql-connector_2.10" % "1.3.2",
  "org.alluxio" % "alluxio-core-client" % "1.0.1",
  "com.datastax.spark" %% "spark-cassandra-connector" % "1.6.0-M2",
  "spark.jobserver" %% "job-server-api" % "0.6.2" % "provided",
  //  "spark.jobserver" %% "job-server-api" % "0.6.2",
  //  SQL or Hive job/context is desired
  "spark.jobserver" %% "job-server-extras" % "0.6.2" % "provided",
  "org.postgresql" % "postgresql" % "9.4-1200-jdbc41"
)

// Utils Dependencies

libraryDependencies ++= Seq(
  "com.github.scopt" %% "scopt" % "3.4.0",
  "org.json4s" %% "json4s-native" % "3.2.11",
  "org.scalactic" %% "scalactic" % "2.2.6",
  "net.debasishg" %% "redisclient" % "3.0"
)
// Test Dependencies

libraryDependencies ++= Seq(
  "org.scalatest" %% "scalatest" % "2.2.6" % "test"
)
unmanagedBase := baseDirectory.value / "custom_lib"

resolvers += "Artima Maven Repository" at "http://repo.artima.com/releases"
resolvers += "Job Server Bintray" at "https://dl.bintray.com/spark-jobserver/maven"

// Excluding JARs and files

assemblyJarName in assembly := "impression.jar"
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = false)

// TODO: explain it :v
val meta =
  """META.INF(.)*""".r

assemblyMergeStrategy in assembly := {
  //  case "application.conf" => MergeStrategy.concat
  case meta(_) => MergeStrategy.discard
  case PathList("com", "google", xs@_*) => MergeStrategy.last
  case PathList("com", "esotericsoftware", "minlog", xs@_ *) => MergeStrategy.last
  case PathList("io", "netty", xs@_*) => MergeStrategy.last
  case PathList("javax", "xml", xs@_*) => MergeStrategy.last
  case PathList("org", "apache", "commons", xs@_ *) => MergeStrategy.last
  case PathList("org", "apache", "hadoop", "yarn", xs@_ *) => MergeStrategy.last
  case PathList("org", "apache", "spark", xs@_ *) => MergeStrategy.last
  case PathList("org", "fusesource", xs@_ *) => MergeStrategy.last
  case PathList("org", "slf4j", xs@_ *) => MergeStrategy.last
  case n if n.startsWith("application.conf") => MergeStrategy.concat
  case n if n.endsWith(".conf") => MergeStrategy.concat
  case x =>
    val oldStrategy = (assemblyMergeStrategy in assembly).value
    oldStrategy(x)
}