Created
February 14, 2020 17:59
-
-
Save Sasquire/5f67f603249fb73c9aabedec39d3a733 to your computer and use it in GitHub Desktop.
A simple script that will download information about sets and determine the user who owns the most sets.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const request_main = require('request'); | |
function request(url) { | |
let options = {url:url, headers: { 'User-Agent': 'Downloading user set information' }}; | |
console.log(options); | |
return new Promise(function (resolve, reject) { | |
request_main(options, function (error, res, body) { | |
console.log(res.statusCode); | |
if (!error && res.statusCode == 200) { | |
resolve(body); | |
} else { | |
reject(error); | |
} | |
}); | |
}); | |
} | |
async function start(){ | |
for(let i = 1; i < 41; i++){ | |
try{ | |
const text = await request('https://e621.net/set/index.xml?page='+i); | |
await writefile('/path/to/file'+i+'.xml', text); | |
console.log('wrote '+i); | |
} catch(e) { | |
console.log(e) | |
} | |
} | |
} | |
start(); | |
async function writefile(path, tex){ | |
return new Promise(function(resolve, reject){ | |
fs.writeFile(path, tex, function(err){ | |
resolve(); | |
}) | |
}); | |
} | |
// grep -hEi "<user_id type=\"integer\">([0-9]*)</user_id>" *.xml | grep -oEi '[0-9]+' | sort | uniq -c | sort |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment