Finding unique records between ORC file and MySQL rows using Apache Spark

unique_orc_records_to_orc.scala

import spark.implicits._
import org.apache.spark.sql.SaveMode

val products = spark.sqlContext.read.format("jdbc").option("driver", "com.mysql.jdbc.Driver").option("dbtable", "products").option("user", "gobblin").option("password", "gobblin").option("url", "jdbc:mysql://localhost/mopar_demo").load()

scala> val newProducts = spark.sqlContext.read.format("orc").load("/Users/tilak/gobblin/mopar-demo/output/org/apache/gobblin/copy/user/tilak/pricing.products_1521799535.csv/20180325023900_append/part.task_PullCsvFromS3_1521945534992_0_0.orc")

scala> val reparitionedProducts = products.repartition(10)

val joined = newProducts.as("np").join(reparitionedProducts.as("op"), reparitionedProducts("sha") === newProducts("sha"), "left_outer")

val newNewProducts = joined.select("np.*")

newNewProducts.write.mode(SaveMode.Overwrite).format("orc").save("/tmp/myapp.orc")