Downloading files test script (plain js)

downloaded.json

[]

index.js

const url = require('url');
const path = require('path');
const https = require('https');
const fs = require('fs');
const dl = fs.existsSync('./downloaded.json') ? require('./downloaded.json') : [];

const dirs = p => fs.readdirSync(p).filter(f => fs.statSync(path.join(p, f)).isDirectory());
const files = p => fs.readdirSync(p).filter(f => !fs.statSync(path.join(p, f)).isDirectory());

const mediaDirs = dirs(".");
var totals = {};
var downloaded = dl;

if (dl.lengh == 0) {
	downloaded = dirs(".")
		.map(dir => files(`./${dir}`).map(file => `./${dir}/${file}`))
		.reduce((acc, val) => acc.concat(val), []);
	fs.writeFileSync("./downloaded.json", JSON.stringify(downloaded));
}

const download = async (url, dest, dir) => {
	return new Promise((resolve, reject) => {
		if (fs.existsSync(dest) && fs.statSync(dest).size > 0) {
			totals[dir]--;
			return reject("File exists");
		}

		let file = fs.createWriteStream(dest);
		let request = https.get(url, (response) => {
			response.pipe(file);
			file.on('finish', () => {
				totals[dir]--;
				file.close(() => {
					console.log(`Downloaded ${dest}`);
					resolve();
				});
			});
		});
		
		request.on('error', (err) => {
			fs.unlinkSync(dest)
			console.log(`Deleted file ${dest}`);
			reject(`${err.message} for ${dest}`);
		});
	});
};

const filter = dir => {
	const json = require(`./${dir}/${dir}.json`)
		.filter(_ => !!_)
		.filter(item => {
			let file = `./${dir}/${path.basename(url.parse(item).pathname)}`;
			
			if (downloaded.indexOf(file) >= 0) {
				return false;
			}
			
			return !(fs.existsSync(file) && fs.statSync(file).size > 0 );
		});
		
		totals[dir] = json.length;
		console.log(`Total ${dir}: ${json.length}`);
	return json;
}

const loop = async () => {
	mediaDirs.forEach(dir => {
		filter(dir)
			.forEach(async item => {
				try {
					await download(
						item, 
						`./${dir}/${path.basename(url.parse(item).pathname)}`,
						dir
					);
				} catch (e) {
					console.log(e);
				}
			});
	});
};

loop();