Skip to content

Instantly share code, notes, and snippets.

@achille
Last active October 5, 2016 19:37
Show Gist options
  • Save achille/4cd1afea9d247ab0baf4 to your computer and use it in GitHub Desktop.
Save achille/4cd1afea9d247ab0baf4 to your computer and use it in GitHub Desktop.
//Create 1gb collection & enqueue
for(i=0;i<1000;i++){db.foo.insert({f:''.pad(1024*1024,true,'A')})}
enqueueWork("test.foo")
//Each worker calls dequeue() and works on it's own range
work = dequeue("test.foo")
function enqueueWork(ns,splitSizeBytes=320000000){
split = db.runCommand({splitVector:ns, keyPattern:{_id: 1},
maxChunkSizeBytes: splitSizeBytes})
current = MinKey
split.splitKeys.forEach(function enqueue(key){
db.queue.insert({ns:ns, start:current, end: key._id})
current = key._id })
db.queue.insert({ns:ns, start:current, end: MaxKey})
}
function dequeue(ns){
return db.queue.findAndModify({
query: {status:{$ne:"processed"}, ns:ns},
update: {$set:{status:"processed"}}
})
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment