Retrieve historical data from the GDAX API and output to CSV

get_historical_data.py

import requests
import datetime
import time
import csv
import progressbar
import argparse
import logging

url = 'https://api.gdax.com'
bar = progressbar.ProgressBar(max_value=1, redirect_stdout=True)

def main():
    progressbar.streams.wrap_stderr()
    logging.basicConfig()

    parser = argparse.ArgumentParser()
    parser.add_argument("days", help="Number of days to retrieve historically", type=int)
    parser.add_argument("granularity", help="Seconds per timeslice: {60, 300, 900, 3600, 21600, 86400} (default: 60)",
    type=int, default=60, choices = [60, 300, 900, 3600, 21600, 86400])
    args = parser.parse_args()

    download_data(args.days, args.granularity)

def download_data(numDays, gran):
    assert gran in [60, 300, 900, 3600, 21600, 86400]

    today = datetime.datetime.today()
    time_ago = today - datetime.timedelta(numDays)

    interv = 350 * gran
    start = today - datetime.timedelta(seconds=interv)
    end = today
    start = time_ago if start < time_ago else start

    historical = []

    while start > time_ago:
        json_resp = gdax_candle_request(start.isoformat(), end.isoformat(), gran)
        for t_slice in json_resp:
            historical.append(t_slice)

        frac_complete = ((today - end).total_seconds()/(today - time_ago).total_seconds())
        bar.update(frac_complete)

        start -= datetime.timedelta(seconds=interv)
        end -= datetime.timedelta(seconds=interv)

        # 3 requests/second rate limit
        time.sleep(0.34)

    if end > time_ago:
        # final chunk
        json_resp = gdax_candle_request(time_ago.isoformat(), end.isoformat(), gran)
        for t_slice in json_resp:
            historical.append(t_slice)
        bar.update(1)

    print("DOWNLOADED: %d time slices from GDAX.", len(historical))
    write_to_csv(historical, numDays, gran)

def gdax_candle_request(start, end, gran):
    # ISO 8601 format: 2017-01-01T00:00:09+00:00
    params = '?start='+start+'&end='+end+'&granularity=%d' % gran
    req = url+'/products/ETH-USD/candles'+params
    #print(req)
    r = requests.get(req)
    if(r.status_code >= 400):
        logging.error('Got status code %d for request %s', r.status_code, req)

    #print(r.status_code)
    return r.json()

def write_to_csv(historical, numDays, gran):
    csvfile = "historical_data_%d_days_gran_%d.csv" % (numDays, gran)

    with open(csvfile, "w") as output:
        writer = csv.writer(output, lineterminator='\n')
        writer.writerow(["time", "low", "high", "open", "close", "volume"])
        writer.writerows(historical)


if __name__ == '__main__':
    main()