This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Map(driver -> oracle.jdbc.OracleDriver, | |
numPartitions -> 2000, | |
timestampFormat -> yyyy-mm-dd hh:mm:ss, | |
upperBound -> 2019-05-01 23:59:59.0, | |
lowerBound -> 2002-03-31 00:00:00.0, | |
partitionColumn -> DATA_DATE, | |
url -> jdbc:oracle:thin:@, | |
user -> user1, password -> pwd1, | |
customSchema -> DATA_DATE TIMESTAMP, DATA_ITEM_VALUE_NUMERIC DECIMAL(38, 15) , MODEL_FAMILY_ID INTEGER , FISCAL_YEAR INTEGER , FISCAL_QUARTER INTEGER, | |
fetchsize -> 10000) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
object { | |
@throws(classOf[Exception]) | |
def process(ymlFilename:String):Option[QueryEntities] = { | |
val ymlFilePath = URLDecoder.decode(ymlFilename, "UTF-8") | |
Try(Source.fromFile(ymlFilePath).mkString.parseYaml) match { | |
case Success(yml) => { |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
object ConfigUtils { | |
def loadEnvProperties(environment : String) : Config ={ | |
import com.typesafe.config._ | |
val appConf = ConfigFactory.load() | |
appConf.getConfig(environment) | |
} | |
def getSparkSession(devProps :Config) : SparkSession = { | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package oracle2cassandra.scala | |
import org.apache.spark.sql.SparkSession | |
import org.apache.spark.sql.DataFrameReader | |
import com.snp.helpers.Model._ | |
import org.slf4j.Logger | |
import org.slf4j.LoggerFactory | |
import scala.collection.Seq | |
import java.io.IOException | |
import com.snp.utils.DbUtils |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
trait Processor { | |
def process()=println("default process") | |
} | |
case class ProcessorOne() extends Processor | |
case class ProcessorTwo() extends Processor { | |
override def process=println("process 2") | |
} | |
case class ProcessorThree() extends Processor { | |
override def process=println("process 3") | |
} |