scan.js

"use strict";

let request     = require("request"),
    cheerio     = require("cheerio"),
    querystring = require("querystring"),
    sendgrid    = require("sendgrid");

let config = require("./config.json");

let mail = sendgrid(config.sendgrid.api_user, config.sendgrid.api_key);

const DOMAIN = "http://www.ksl.com";
const PARAMS = {
    nid: 231,
    sid: 74268,
    cat: 144,
    search: "bmw nine"
};
const FREQUENCY = 1000 * 60 * 5; // 5 minutes


let url = `${DOMAIN}/?${querystring.stringify(PARAMS)}`;
let found = [];
let errCount = 0;
const ERROR_THRESHOLD = 5;
function doError(err) {
    console.error("\n" + err);
    if(++errCount >= ERROR_THRESHOLD) {
        console.error("\nerror threshold reached");
        process.exit(1);
    }
}

search();
setInterval(search, FREQUENCY);

function search() {
    process.stdout.write(".");
    request(url, function (err, response, body) {
        if (err) {
            doError(err);
        } else if (response.statusCode !== 200) {
            doError(`${url} return status code: ${response.statusCode}`);
        } else {
            errCount = 0;
            let $ = cheerio.load(body);
            $('.adBox').each((i, elem) => {
                elem = $(elem);
                let title = elem.find('.adTitle').text().trim();
                if (
                    // does title contains ALL words in search string
                    PARAMS.search.split(" ").reduce((m, i) => {
                        return m && title.toLowerCase().includes(i.toLowerCase())
                    }, true)
                ) {
                    let href = elem.find("a").attr("href"),
                        search = querystring.parse(href.slice(1)),
                        id = parseInt(search.ad, 10),
                        link = `${DOMAIN}/?${querystring.stringify(search)}`,
                        location = elem.find(".adTime span:first-child").text(),
                        desc = elem.find(".adDesc").text().replace(/\s+/g, " ").trim(),
                        price = $(elem.find(".priceBox > a > span").contents()[0]).text();

                    // only send 1 email per item
                    if (found.indexOf(id) === -1) {
                        let subject = `KSL listing ${id} | ${title} | in ${location} for ${price}: `;
                        let text = `${link}\n\n${desc}`;

                        console.log("\nfound", subject);

                        mail.send({
                            to: config.email.to,
                            from: config.email.from,
                            subject,
                            text
                        }, function (err, json) {
                            if (err) {
                                console.error(err);
                            } else {
                                found.push(id);
                            }
                        });
                    }
                }
            });
        }
    });
}