vic-lgas.py

import requests
import json
import scraperwiki
from datetime import datetime

url = 'https://wabi-australia-southeast-api.analysis.windows.net/public/reports/querydata?synchronous=true'

headers = {
    "Accept": "application/json, text/plain, */*",
    "ActivityId": "a1955107-c5d9-12b5-87a4-561001265ed6",
    "Content-Type": "application/json;charset=UTF-8",
    "Origin": "https://app.powerbi.com",
    "Referer": "https://app.powerbi.com/view?r=eyJrIjoiODBmMmE3NWQtZWNlNC00OWRkLTk1NjYtMjM2YTY1MjI2NzdjIiwidCI6ImMwZTA2MDFmLTBmYWMtNDQ5Yy05Yzg4LWExMDRjNGViOWYyOCJ9",
    "RequestId": "c1ef00f5-e7b9-fe25-927d-a8f673615040",
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
    "X-PowerBI-ResourceKey": "80f2a75d-ece4-49dd-9566-236a6522677c",
    "modelID":"1959902"
}

payload = {"version":"1.0.0","queries":[{"Query":{"Commands":[{"SemanticQueryDataShapeCommand":{"Query":{"Version":2,"From":[{"Name":"d1","Entity":"dimLGA"},{"Name":"l","Entity":"Linelist"}],"Select":[{"Column":{"Expression":{"SourceRef":{"Source":"d1"}},"Property":"LGAName"},"Name":"dimLGA.LGAName"},{"Aggregation":{"Expression":{"Column":{"Expression":{"SourceRef":{"Source":"l"}},"Property":"PHESSID"}},"Function":5},"Name":"Linelist.PHESSID"}],"Where":[{"Condition":{"Not":{"Expression":{"Comparison":{"ComparisonKind":0,"Left":{"Column":{"Expression":{"SourceRef":{"Source":"d1"}},"Property":"LGAName"}},"Right":{"Literal":{"Value":"null"}}}}}}}]},"Binding":{"Primary":{"Groupings":[{"Projections":[0,1]}]},"DataReduction":{"DataVolume":3,"Primary":{"Window":{"Count":500}}},"Version":1}}}]},"CacheKey":"{\"Commands\":[{\"SemanticQueryDataShapeCommand\":{\"Query\":{\"Version\":2,\"From\":[{\"Name\":\"d1\",\"Entity\":\"dimLGA\"},{\"Name\":\"l\",\"Entity\":\"Linelist\"}],\"Select\":[{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"d1\"}},\"Property\":\"LGAName\"},\"Name\":\"dimLGA.LGAName\"},{\"Aggregation\":{\"Expression\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"l\"}},\"Property\":\"PHESSID\"}},\"Function\":5},\"Name\":\"Linelist.PHESSID\"}],\"Where\":[{\"Condition\":{\"Not\":{\"Expression\":{\"Comparison\":{\"ComparisonKind\":0,\"Left\":{\"Column\":{\"Expression\":{\"SourceRef\":{\"Source\":\"d1\"}},\"Property\":\"LGAName\"}},\"Right\":{\"Literal\":{\"Value\":\"null\"}}}}}}}]},\"Binding\":{\"Primary\":{\"Groupings\":[{\"Projections\":[0,1]}]},\"DataReduction\":{\"DataVolume\":3,\"Primary\":{\"Window\":{\"Count\":500}}},\"Version\":1}}}]}","QueryId":"","ApplicationContext":{"DatasetId":"5b547437-24c9-4b22-92de-900b3b3f4785","Sources":[{"ReportId":"964ef513-8ff4-407c-8068-ade1e7f64ca5"}]}}],"cancelQueries":[],"modelId":1959902}

# Adding empty header as parameters are being sent in payload
# headers = {}
# print(json.dumps(payload))

r = requests.post(url, data=json.dumps(payload), headers=headers)
print(r.content)

data = json.loads(r.text)

lgas = data['results'][0]['result']['data']['dsr']['DS'][0]['PH'][0]['DM0']

clean = []
# print(json.dumps(lgas, indent=4))
for lga in lgas:
	print(lga)
	newRow = {}
	newRow['lga'] = lga['C'][0]
	print(newRow['lga'])
	if len(lga['C']) > 1:
		newRow['count'] = lga['C'][1]
	
	clean.append(newRow)

scraperwiki.sqlite.save(unique_keys=["lga"], data=clean, table_name="vic")

# print(clean)

# print(data['results'][0])