Batch processing of user documents. Node, mongoose, async (whilst, eachLimit, waterfall), moment.

batch-process.js

var async = require("async");
var moment = require('moment');
var User = require('/path/to/user.js');

module.exports = function(){
    var skip = 0;
    var limit = 700;  //to process the users in batches of 700
    var newUsersFound = true;
    var userCount = 0;
    async.whilst(
        function() { return newUsersFound},
        function(callback) {
        var updatedAt = new Date(moment().subtract(7, 'days').format('YYYY-MM-DD')); //last week
        User.find({email: {$exists: true}, updatedAt: {$gt: updatedAt}}, function(err, users){
            if(err){
                return callback(err)
            }
            else{
                if(!users.length){
                    newUsersFound = false;
                    return callback();
                }
                else{
                    userCount += users.length;
                    skip += limit;
                    processUserBatch(users, function(err){
                        if(err){
                            return callback(err);
                        }
                        else{
                            return callback();
                        }
                    })
                }
            }
        })                    
        .limit(limit)
        .skip(skip)
    },
    function (err) {
        if(err){
            console.log(err)
        }
        else{
            console.log("Users Processed: ", userCount)

        }
    }
    );
}

function processUserBatch(users, done){
    async.eachLimit(users, 20, function(user, callback){
        processUser(user, function(err){
            if(err){
                return callback(err);
            }
            else{
                return callback();
            }
        });
    }, function(err){
        if(err){
            return done(err);
        }
        else{
            return done();
        }
    });
}


function processUser(user, done){
    async.waterfall([
            //dummy functions
            findSettings,
            gatherCyclingData,
            dispatchEmail
            ],function(err){
                if(err){
                    return done(err);
                }
                else{
                    return done();
                }
            });


    function findSettings(callback){
        // return callback(null, settings);
        // OR return callback(err);    
    }

    function gatherCyclingData(settings, callback){
        // return callback(null, cd);
        // OR return callback(err);
    }

    function dispatchEmail(cd, callback){
        // return callback();
        // OR return callback(err);
    }
}