Spark Json

sparkjson.sc

import org.apache.spark.sql.types.{IntegerType, MapType, StringType, StructField, StructType}
import org.apache.spark.sql.{Column, DataFrame, Encoders}

val DataSchema: StructType = StructType(
  List(
    StructField("A", StringType, nullable = true), // required
    StructField("B", StringType, nullable = true), // required
    StructField("C", StringType, nullable = true) // required
  )
)

val eventStr = """
{"A": 1, "C": "world"}
{"A": 2, "B": "Hello"}
"""

val events = eventStr.split("\n").filter(_.nonEmpty)
val df: DataFrame = spark.createDataset(events)(Encoders.STRING).toDF("data")

df.withColumn("json", from_json(col("data"), DataSchema)).select("json.*")