Utility script to get department codes and course UIDs for all classes at Cal through the dev api.

gistfile1.txt

/**
 *
 *  Little utility script to get list of all department codes
 *
 */

var request = require('request');
var async = require('async');
var _ = require('lodash');
var fs = require('fs');

var ALPHABET = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
    "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"];

var API_URL = "https://apis-dev.berkeley.edu/cxf/asws",
    API_APP_ID = "",
    API_APP_KEY = "";

var departments = [],
    courses = [];


async.eachSeries(ALPHABET, function (letter, cb) {
    // Send out requests in series to the Berkeley dev APIs to retrieve a list
    // of all of the department codes

    request({
        url: API_URL + '/department?_type=json&departmentName=' + letter,
        headers: {
            "app_id": API_APP_KEY,
            "app_key": API_APP_ID
        }
    }, function (error, response, body) {
        if (!error) {
            var data = JSON.parse(body).CanonicalDepartment;

            if (data && data.length > 0) {

                for (var i = 0; i < data.length; i++) {
                    var department = data[i];

                    var exists = !!_.find(departments, function (curr) {
                      return curr.departmentCode === department.departmentCode;
                    });

                    if (!exists) {
                        departments.push({
                            departmentCode: department.departmentCode,
                            departmentName: department.departmentName
                        });
                    }
                }
            }
        } else {
            console.log("Error: " + error);
        }
        cb();
    });
}, function done() {
    console.log("Done collecting department codes!");

    console.log(departments);

    fs.writeFile('departments.data', JSON.stringify(departments));

    // Now that we've finished the initial request for department codes,
    // we make another request to the APIs for all of the course for each of
    // the departments

    async.eachSeries(departments, function (department, cb) {

        var url = API_URL + '/course?_type=json&departmentCode=' +
            encodeURIComponent(department.departmentCode);

        console.log("Issuing request to " + url + "...");

        request({
            url: url,
            headers: {
                "app_id": API_APP_KEY,
                "app_key": API_APP_ID
            }
        }, function (error, response, body) {
            if (error || response.statusCode != 200) {
                console.log("Error: " + error);
            } else {
                var data = JSON.parse(body).CanonicalCourse;
                if (data && data.length > 0) {

                    for (var i = 0; i < data.length; i++) {
                        var course = data[i];

                        if (!_.contains(courses, course.courseUID)) {
                            // Add to courses array
                            courses.push(course);
                        }

                    }
                }
            }

            cb();
        });

    }, function done() {
        console.log("Done collecting course information for each department!");

        fs.writeFile('courses.data', JSON.stringify(courses));
    });
});