Dataproc plugin for sbt allowing to spin-up and tear down clusters around integration tests.

build.sbt

import DataprocPlugin._
lazy val root= (project in file("."))
  .enablePlugins(BuildInfoPlugin, DataprocPlugin)
  .settings(
    buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, assemblyOutputPath in assembly),
    buildInfoPackage := "com.tapad.scorer"
  )
  .settings(
    Defaults.itSettings,
    test in IntegrationTest := {
      (test in IntegrationTest).dependsOn(assembly in assembly)
    }.value,
    testOptions in IntegrationTest += Tests.Setup(() => createClusterFunc(dataprocClusterName.value, dataprocZone.value, dataprocServiceAccount.value)),
    testOptions in IntegrationTest += Tests.Cleanup(() => deleteClusterFunc(dataprocClusterName.value))
  )

DataprocPlugin.scala

import sbt.Keys._
import sbt._

object DataprocPlugin extends AutoPlugin {

  object autoImport {
    val dataprocZone = settingKey[String]("Zone to run Dataproc cluster in")
    val dataprocClusterName = settingKey[String]("Dataproc cluster name")
    val dataprocServiceAccount = settingKey[String]("Service account to use for Dataproc cluster")
    val createCluster = taskKey[Unit]("dataproc-start")
    val deleteCluster = taskKey[Unit]("dataproc-stop")
  }

  import autoImport._

  def createClusterFunc(clusterName: String, zone: String, serviceAccount: String): Unit = { println(s"create cluster $clusterName")
    val rc = Process("gcloud" :: "dataproc" :: "clusters" :: "create" ::
      "--zone" :: zone :: "--service-account" :: serviceAccount :: clusterName :: Nil).!
    if (rc > 0) {
      sys.error(s"Error creating dataproc cluster (processed exited with code `$rc)")
    }
  }

  def deleteClusterFunc(clusterName: String): Unit = { println(s"delete cluster $clusterName")
    val rc = Process("gcloud" :: "dataproc" :: "clusters" :: "delete" :: clusterName :: "--async" :: Nil).!
    if (rc > 0) {
      sys.error(s"Error deleting dataproc cluster (processed exited with code $rc)")
    }
  }

  override def projectSettings = {
    Seq(
      dataprocZone := dataprocZone.?.value.getOrElse("us-central1-a"),
      dataprocClusterName := dataprocClusterName.?.value.getOrElse(s"${name.value}-it-test-cluster"),
      dataprocServiceAccount := dataprocServiceAccount.?.value.getOrElse("default-service-account"),
      createCluster := createClusterFunc(dataprocClusterName.value, dataprocZone.value, dataprocServiceAccount.value),
      deleteCluster := deleteClusterFunc(dataprocClusterName.value)
    )
  }
}