Created
March 7, 2013 09:22
-
-
Save petecleary/5106718 to your computer and use it in GitHub Desktop.
CSV, MongoDB example
Loops through an import CSV and writes the rows to mongo
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
//npm install csv | |
//npm install mongodb | |
var csv = require('csv'); | |
var fs = require('fs'); | |
var start = Date.now(); | |
// Retrieve | |
var mongoclient = require('mongodb').MongoClient; | |
// Connect to the db | |
mongoclient.connect("mongodb://localhost:27017/Test", function (err, db) { | |
if (err) { | |
console.log("Unable to attach to database"); | |
} | |
else { | |
var collection = db.collection('test'); | |
csv() | |
.from.stream(fs.createReadStream(__dirname + '/imports/test.csv', { encoding: 'utf8' })) | |
.on('record', function (row, index) { | |
if (index > 0 && index < 5) { | |
var person = { 'email': row[0], 'name': row[1], 'surname': row[2]}; | |
//add in upsert for person with dates and attempts | |
collection.insert(person, { w: 1 }, function (err, result) { | |
if (err) { | |
console.log("*************************"); | |
console.log("Index" + index); | |
console.log("Save to database issue"); | |
console.log(err); | |
console.log("*************************"); | |
} | |
}); | |
} | |
// console.log('#'+index+' '+JSON.stringify(row)); | |
}) | |
.on('end', function (count) { | |
console.log('Number of lines: ' + count); | |
var end = Date.now(); | |
var time = (end - start) / 1000; | |
console.log('Time taken: ' + time + "seconds"); | |
console.log('Time taken per row: ' + time / count); | |
}) | |
.on('error', function (error) { | |
console.log(error.message); | |
}); | |
} | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment