COVID School Closing w/ IPEDS (closings from https://gist.github.com/jessejanderson/09155afe313914498a32baa477584fae)

fuzzymatch.py

# Changes before you can run it on your own:
# 1. change the data directory from raw_data
# 2. download the IPEDS data as a CSV

import pickle as pkl
from fuzzywuzzy import fuzz
from fuzzywuzzy import process

ipeds = pd.read_csv("../raw_data/ipeds.csv")

# IPEDS join with fuzzy match
hardcode = {
    "California State University (San Jose)": 122755,
    "California State University (Sonoma)": 123572,
    "University System of Georgia": 139959,
    "University of Illinois System": 145637,
    "Emerson University": 165662,
    "Calvin University": 169080,
    # TODO: investigate which codes belong to which for Minnesota
    "Minnesota State Colleges and Universities (Campuses with spring break the week of March 9-13)": 173920, 
    "Minnesota State Colleges and Universities (Campuses with spring break the week of March 16-20)": 174358,
    "Washington University (Danforth)": 179867,
    "Washington University (Medical)": 179867,
    "University of Nebraska System (Lincoln)": 181464,
    "University of Nebraska System (Omaha)": 181394,
    # I just chose an arbitrary CUNY: city college
    "City University of New York": 190567,
    "Vassar University": 197133,
    "University Northern Colorado": 127741,
    "Alamo Colleges District": 222497,
    "University of Texas System (Austin)": 228778,
    "University of Texas System (Dallas)": 228787,
    "University of Texas System (San Antonio)": 229027,
    "University of Texas System (Tyler)": 228802,
    "University of Texas System (Arlington)": 228769,
    "University of Tennessee System (Knoxville)": 221759,
    "University of Tennessee System (Martin)": 221768,
    "Concordia St. Paul": 173328,
}
try:
    cache = pkl.load(open("../raw_data/ipeds_fuzzy_cache.pkl", "rb"))
except:
    cache = dict()
for school in schools:
    if school["school"] in hardcode:
        school["unitid"] = hardcode[school["school"]]
        continue
    if school["school"] in cache:
        school["unitid"] = cache[school["school"]]
        continue
    matches = ipeds["Institution (entity) name"].apply(lambda x: fuzz.partial_ratio(x, school["school"].replace(" & ", " and ")))
    if (np.max(matches) > 85):
        school["unitid"] = ipeds.iloc[np.argmax(matches)]["UnitID"]
    else:
        new_sname = school["school"].replace("College", "University") if "College" in school["school"] else school["school"].replace("University", "College")
        new_matches = ipeds["Institution (entity) name"].apply(lambda x: fuzz.partial_ratio(x, new_sname))
        ind = np.argmax(new_matches) if np.max(new_matches) > np.max(matches) else np.max(matches)
        school["unitid"] = ipeds.iloc[ind]["UnitID"]
        if (np.max([np.max(new_matches), np.max(matches)]) <= 85):
            print("({}) {} <==> {}".format(np.max([np.max(new_matches), np.max(matches)]), ipeds["Institution (entity) name"].iloc[np.argmax(matches)], school["school"]))
for school in schools:
    if school["school"] not in cache:
        cache[school["school"]] = school["unitid"]
pkl.dump(cache, open("../raw_data/ipeds_fuzzy_cache.pkl", "wb"))

simple_closings.csv