gani_threaded.py

import requests
import math
from threading import Thread

THREAD_LIMIT = 8
PAGE_SIZE = 10


def main():
    req = requests.Session()
    req.headers.update({
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
        'Cookie': 'ASP.NET_SessionId=qelk5t3y0lcdzjdak3f1ujr5',
        'Host': 'egcap.bluespringsgov.com',
        'Origin': 'http://egcap.bluespringsgov.com',
        'Referer': 'http://egcap.bluespringsgov.com/CitizenAccess/Site/Permit/Search',
        'X-Requested-With': 'XMLHttpRequest'
    })

    payload = {
        'PermitSearchModel.IssueDateStart': '6/1/2016',
        'PermitSearchModel.IssueDateStop': '6/30/2016',
        'CountryTypeID': '0',
        'County': '',
        'City': '',
        'AddressLine1': '',
        'StreetDirectionPrefixID': '',
        'AddressLine2': '',
        'StreetTypeID': '',
        'StreetDirectionPostfixID': '',
        'UnitOrSuite': '',
        'StateID': '',
        'PostalCode': '',
        'PermitSearchModel.PermitNumber': '',
        'PermitSearchModel.ProjectName': '',
        'PermitSearchModel.PermitTypeID': '',
        'PermitSearchModel.PermitTypeWorkClassID': '',
        'PermitSearchModel.ExpirationDateStart': '',
        'PermitSearchModel.ExpirationDateStop': '',
        'PermitSearchModel.PermitStatusID': '',
        'PermitSearchModel.FinalizedDateStart': '',
        'PermitSearchModel.FinalizedDateStop': ''
    }

    res = req.post('http://egcap.bluespringsgov.com/CitizenAccess/Site/Permit/Search/AjaxFormPost', data=payload)

    data = []
    threads = []
    i = 0
    for x in range(0, THREAD_LIMIT):
        i += 1
        threads.append(Thread(target=send_request, args=(i, req, data)))

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

    '''
    * data now contains the data of the pages
    * not arranged because it's threaded
    * rename data if you want because data['data'] sounds wrong

    data[0] = {
        'page':
        'total':
        'data':
    }

    '''


def send_request(thread_id, req, data):
    grid_payload = {
        'page': str(thread_id),
        'size': str(PAGE_SIZE)
    }

    is_finished = False
    while not is_finished:
        grid_data = req.post('http://egcap.bluespringsgov.com/CitizenAccess/Site/Permit/Search/_SearchResultGridPopulate?Grid-page=1&Grid-size=10&Grid-CERT_KEYSIZE=&Grid-CERT_SECRETKEYSIZE=&Grid-HTTPS_KEYSIZE=&Grid-HTTPS_SECRETKEYSIZE=', data=grid_payload)
        json_data = grid_data.json()

        page = int(grid_payload['page'])
        json_data['page'] = page

        data.append(json_data)

        grid_payload['page'] = str(page + THREAD_LIMIT)
        is_finished = math.ceil(int(json_data['total']) / PAGE_SIZE) < int(grid_payload['page'])
        print('Finished page:', page, len(data), grid_data.status_code)

if __name__ == '__main__':
    main()