Skip to content

Instantly share code, notes, and snippets.

@cole-gillespie
Last active August 29, 2015 14:17
Show Gist options
  • Save cole-gillespie/6e1e3330cf8895616eef to your computer and use it in GitHub Desktop.
Save cole-gillespie/6e1e3330cf8895616eef to your computer and use it in GitHub Desktop.
simple update of mongo documents
var MongoClient = require('mongodb').MongoClient,
bunyan = require('bunyan'),
log = bunyan.createLogger({
name: 'harvest'
});
request = require('request')
var url = 'mongodb://localhost:27017/openNews';
MongoClient.connect(url, function(err, db) {
log.info("Connected correctly to server");
var collection = db.collection('nyt');
setInterval(function(){
request('http://localhost/nyt', function (error, response, body) {
if (!error && response.statusCode == 200) {
log.info("checking nyt for new content")
JSON.parse(body).forEach(function(element, index, array){
collection.findOne({href:element.href}, function(err, item) {
request('http://localhost:1337/?q=' + element.href, function (error, response, body) {
if(item === null) {
log.info("cannot find ", element.href)
b = JSON.parse(body)
body.time = new Date().getTime()
element.shares = [b];
element.firstSeen = new Date().getTime();
collection.insert(element, {w:1}, function(err, result) {
log.info("adding", result)
});
}
})
});
});
}
})
}, 300000);
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment