download_symbols_polygon_loop.py

download_symbols_polygon_loop.py

import json, os, csv, time
from datetime import datetime

import requests

import _0_load_price_by_date


data_dir = os.path.expanduser('~/stock_data/polygon_prices')
if not os.path.exists(data_dir):
  os.makedirs(data_dir)

with open(os.path.expanduser('~/alpaca_config.json')) as f:
  text = f.read()
api_key = json.loads(text)['live']['api_key']

while True:
  for page in range(1, 10 ** 5):
    print('page:', page)

    for attempt in range(5):
      try:
        tickers_resp = requests.get(
          'https://api.polygon.io/v2/reference/tickers?apiKey={}&page={}&market={}'.format(
            api_key, page, 'stocks'
          ), timeout=5,
        )
      except(
        requests.exceptions.ConnectionError,
        requests.exceptions.ConnectTimeout,
        requests.exceptions.ReadTimeout
      ):
        print('timeout loading tickers')
        continue
      break
    else:
      print('too many timeouts, giving up')
      continue

    # keys: ['page', 'perPage', 'count', 'status', 'tickers']
    try:
      ticker_count = tickers_resp.json()['count']
    except json.decoder.JSONDecodeError:
      print('json error:', tickers_resp.content)
      continue
    per_page = tickers_resp.json()['perPage']

    tickers = tickers_resp.json()['tickers']
    if not tickers:
      break

    for i_ticker, ticker_dict in enumerate(tickers):
      ticker = ticker_dict.get('ticker')
      print('{}/{}: {}'.format((page - 1) * per_page + i_ticker, ticker_count, ticker))
      _0_load_price_by_date.load_price_by_date(ticker, use_filesystem_cache=False)

  print('completed at:', datetime.utcnow())
  time.sleep(60 * 60 * 24)