Created
April 24, 2015 19:39
-
-
Save ealeksandrov/eed152005cdbaa5b4e5a to your computer and use it in GitHub Desktop.
Web API crawler on node.js
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var Q = require('q'); | |
var request = Q.denodeify(require('request')); | |
var domain = "http://challenge.shopcurbside.com/"; | |
var localSessionId; | |
function getSession() { | |
var response = request({ | |
uri: domain + 'get-session', | |
method: 'GET' | |
}) | |
return response.then(function (res) { | |
return res[0].body; | |
}) | |
} | |
function getNode(nodeId,sessionId) { | |
var response = request({ | |
uri: domain + nodeId, | |
method: 'GET', | |
headers: {"Session" : sessionId} | |
}) | |
return response.then(function (res) { | |
var jsonobj = JSON.parse(res[0].body); | |
if(jsonobj.error) { | |
throw new Error(jsonobj.error) | |
} else { | |
return jsonobj; | |
} | |
}).then(function (res) { | |
if(res.secret) { | |
process.stdout.write(res.secret); | |
} | |
return res; | |
}, function (error) { | |
if(error.message == "\"Session\" header is missing. \"/get-session\" to get a session id." || error.message == "Invalid session id, a token is valid for 10 requests.") { | |
var session = getSession(); | |
return session.then(function (newSessionId) { | |
localSessionId = newSessionId; | |
return getNode(nodeId,newSessionId); | |
}) | |
} else { | |
//console.log('Unhandled error: ' + error); | |
} | |
}); | |
} | |
function getChildren(nodeArray) { | |
var idx = 0; | |
var childrenArray = []; | |
function getChildNode(nodeId) { | |
return getNode(nodeId,localSessionId).then(function(node) { | |
var promise1; | |
var promise2; | |
var childNodes = node['next']; | |
if(!childNodes) { | |
childNodes = node['nExt']; | |
} | |
if(!childNodes) { | |
childNodes = node['neXT']; | |
} | |
if(childNodes) { | |
childrenArray = childrenArray.concat(childNodes); | |
} | |
idx++; | |
if(idx < nodeArray.length) { | |
getChildNode(nodeArray[idx]); | |
} else { | |
getChildren(childrenArray); | |
} | |
}); | |
} | |
return getChildNode(nodeArray[0]); | |
} | |
var nodeArray = ["start"]; | |
getChildren(nodeArray); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment