Last active
July 2, 2020 19:09
-
-
Save soeirosantos/535b249d255fc077aade2fbec837b799 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
plugins { | |
id 'java' | |
id 'org.jetbrains.kotlin.jvm' version '1.3.72' | |
} | |
group 'br.com.soeirosantos' | |
version '1.0.0-SNAPSHOT' | |
repositories { | |
mavenCentral() | |
} | |
dependencies { | |
compile group: 'org.twitter4j', name: 'twitter4j-core', version: '4.0.7' | |
compile group: 'org.twitter4j', name: 'twitter4j-stream', version: '4.0.7' | |
compile group: 'com.github.kittinunf.fuel', name: 'fuel', version: '2.2.3' | |
compile group: 'com.github.kittinunf.fuel', name: 'fuel-gson', version: '2.2.3' | |
compile group: 'com.google.code.gson', name: 'gson', version: '2.8.5' | |
compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.8.0-alpha2' | |
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" | |
testCompile group: 'junit', name: 'junit', version: '4.12' | |
} | |
compileKotlin { | |
kotlinOptions.jvmTarget = "1.8" | |
} | |
compileTestKotlin { | |
kotlinOptions.jvmTarget = "1.8" | |
} | |
repositories { | |
maven { | |
url "https://dl.bintray.com/kittinunf/maven" | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
log4j.rootLogger = INFO, FILE, stdout | |
log4j.appender.FILE=org.apache.log4j.FileAppender | |
log4j.appender.FILE.File=./log.out | |
log4j.appender.FILE.ImmediateFlush=true | |
log4j.appender.FILE.Threshold=info | |
log4j.appender.FILE.Append=true | |
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout | |
log4j.appender.FILE.layout.conversionPattern=%m%n | |
log4j.appender.stdout=org.apache.log4j.ConsoleAppender | |
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout | |
log4j.appender.stdout.layout.ConversionPattern=%d{yy-MM-dd HH:mm:ss:SSS} %5p %t %c{2}:%L - %m%n |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package br.com.soeirosantos.twitter | |
import com.github.kittinunf.fuel.Fuel | |
import com.github.kittinunf.fuel.core.ResponseDeserializable | |
import com.google.gson.Gson | |
import org.slf4j.Logger | |
import org.slf4j.LoggerFactory | |
import twitter4j.* | |
import java.io.File | |
import kotlin.math.roundToInt | |
import kotlin.streams.toList | |
class TwitterEngagement | |
val log: Logger = LoggerFactory.getLogger(TwitterEngagement::class.java) | |
const val FILTER_PROPERTY = "filter" | |
const val LANGUAGE_FILTER = "en" | |
const val LOG_FILE = "./log.out" | |
const val TWITTER_URL = "https://twitter.com" | |
const val SENTIMENT_URL = "http://text-processing.com/api/sentiment/" | |
val FAVORITE_CACHE = HashSet<Long>() | |
val RETWEET_CACHE = HashSet<String>() | |
fun main() { | |
reloadCache() | |
val twitter = TwitterFactory.getSingleton() | |
val twitterStream: TwitterStream = TwitterStreamFactory().instance | |
twitterStream.onException { log.error("Boo!", it) } | |
twitterStream.onStatus { | |
if (!it.isRetweet) return@onStatus | |
if (it.retweetedStatus.id in FAVORITE_CACHE) return@onStatus | |
if (it.retweetedStatus.favoriteCount == 0) return@onStatus | |
if (it.retweetedStatus.retweetCount == 0) return@onStatus | |
if (favorite(it.retweetedStatus)) { | |
val sentiment = sentiment(it.retweetedStatus.text) | |
if (sentiment.label == "neg") return@onStatus | |
twitter.createFavorite(it.retweetedStatus.id) | |
FAVORITE_CACHE.add(it.retweetedStatus.id) | |
if (!(it.retweetedStatus.user.screenName in RETWEET_CACHE) && retweet(it.retweetedStatus)) { | |
twitter.retweetStatus(it.retweetedStatus.id) | |
RETWEET_CACHE.add(it.retweetedStatus.user.screenName) | |
} | |
log.info( | |
"favCount: ${it.retweetedStatus.favoriteCount} | " + | |
"retweetCount: ${it.retweetedStatus.retweetCount} | " + | |
"sentiment: ${sentiment.label} - " + | |
" $TWITTER_URL/${it.retweetedStatus.user.screenName}/status/${it.retweetedStatus.id}" | |
) | |
} | |
} | |
val query = FilterQuery() | |
query | |
.language(LANGUAGE_FILTER) | |
.track( | |
*System.getProperty(FILTER_PROPERTY, "devops") | |
.split(",").map { it.trim() }.toTypedArray() | |
) | |
twitterStream.filter(query) | |
} | |
private fun favorite(status: Status) = status.favoriteCount > favoriteEngagementFactor() || | |
status.retweetCount > favoriteEngagementFactor() | |
private fun retweet(status: Status) = status.favoriteCount > retweetEngagementFactor() || | |
status.retweetCount > retweetEngagementFactor() | |
/** | |
* This function is used to weight the engagement for favorite tweets | |
* | |
* The current implementation says that we reduce the engagement rate as | |
* the favorite engagement grows. | |
*/ | |
private fun favoriteEngagementFactor(): Int { | |
return (5 + .5 * FAVORITE_CACHE.size).roundToInt() //customize the rule to obtain the favorite factor | |
} | |
/** | |
* This function is used to weight the engagement for retweets | |
* | |
* The current implementation says it's constant at 1000 | |
*/ | |
private fun retweetEngagementFactor(): Int { | |
return 1000 //customize the rule to obtain the retweet factor | |
} | |
/** | |
* The cache keeps track of tweets and users we have already engaged | |
* to avoid an excess of similar interactions. | |
* | |
* This function reloads the cache from the log file to make sure | |
* we are in a good shape across executions. | |
*/ | |
private fun reloadCache() { | |
val file = File(LOG_FILE) | |
if (!file.exists()) return | |
file.forEachLine { line -> | |
if (line.contains(TWITTER_URL)) { | |
val split = line.split("/") | |
val tweetId = split[split.size - 1] | |
FAVORITE_CACHE.add(tweetId.toLong()) | |
val userScreenName = split[split.size - 3] | |
RETWEET_CACHE.add(userScreenName) | |
} | |
} | |
} | |
data class Sentiment(var label: String) { | |
class Deserializer : ResponseDeserializable<Sentiment> { | |
override fun deserialize(content: String): Sentiment = Gson().fromJson(content, Sentiment::class.java) | |
} | |
} | |
fun sentiment(text: String): Sentiment { | |
val (_, _, result) = Fuel.post(SENTIMENT_URL) | |
.body("text=$text") | |
.responseObject(Sentiment.Deserializer()) | |
return result.get() | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment