reskassa-scraper.py

from datetime import timedelta, date
import subprocess
import json
import pandas


def run(command, current_date, end_date):
    transactions = []
    fifteen_days = timedelta(days=15)

    while current_date < end_date:
        print("current date: " + str(current_date))
        output = subprocess.check_output(command.replace("@@DATE@@", str(current_date)), shell=True)
        data = json.loads(str(output, encoding='utf-8'))

        print('fetched ' + str(len(data['data']['travel_card_transaction_list'])) + ' transactions')

        for transaction in data['data']['travel_card_transaction_list']:
            transactions.append(transaction['travel_card_transaction'])

        current_date = current_date + fifteen_days

    with open("output.csv", "w") as f:
        df = pandas.read_json(json.dumps(transactions))
        f.write(df.to_csv())


if __name__ == "__main__":
    command = r'''curl -k 'https://sl.se/api/MySL/GetTravelCardHistory' -H 'Cookie: s_fid=71A5A7AC3DCAE469-11EC60D5337B30A9; EPi:StateMarker=true; s_cc=true; MySL-Persistent-Object=140138b6-3fcb-4c3d-8e9c-5cb568b66cd3; MySL-Session=e41ef559-b7b4-4554-adf8-1405d6b83e8e; s_sq=%5B%5BB%5D%5D' -H 'Origin: https://sl.se' -H 'CSRF-GUARD-CHALLENGE: 3291239944' -H 'Accept-Language: en-GB,en-US;q=0.9,en;q=0.8,sv;q=0.7,la;q=0.6' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' -H 'CSRF-GUARD-AUTH: FL7Ebk/ZW7cNNJBQNebWafVAarFYVi20QZnWgh4j/FoAWgPX/ZNQPbKBVohBKlVkPZjmwZ9k6U84EQ0w9gm+IK3OIu+rzfZiNyWbkZf2dgPRD4Nt1bH0Fm4jVF+9QwlGDtGau5bbQMyMn0rXDDOaTtHF8qc7vpzLW2F2xKf03IYMfuuKAkxRWBAxjp2U0SVtMzNAIaBLvbEsQM8L5Etq4QEOgqpi8/FJsS9WyKdJpo6gAcA6MGPXOB0lRVtZiUSNL8rdt7PM8VB6c6MWZqIF9dGn6QcGJWprp9t1rgOLHS9BLEVDrqmznAh8CpFtCq3ahJwUM9sX7Au9cf/f7v6N9QBfqH1FNaWGouCDeoskan6IPGK/xUBbBcoF0MEwPVMqydho+YiYPlurmUlEdH8uwx+O/O9SSxPOzEoXm9An6pTwZRuLenIQEPIoyM6TE+KOTk8fd0Zcck8mM7RLI4vzDH/3mpfQaG6t2ghsKAsmWAbMwbhAR4ly2k2aor60EQ23tltWr+YEC46jBaSl5j/z/VCfCxPu1Y3qGqTf+EKTtYb7hkPKuMcPkax8U34octW5XaHo9pfC/noT14flYjyCeMQ2btE04rN0sdaq+tSQiUpnCTA13lsfgJTNgruRDM/vVC+0reeJWo93wXBZuXW9VCfriAagcj8m/vMSb2TC4dzluo3Sy4tvKVAM8yge8uenGqISL6cC8eY4RlxVXVG2k2WFVSV7SHmEeYCOt3UDDt+szHHc4GoBGJWPQVbeSSNevQfehs3M+xKb9BFa0JewUtJMJ9wFwNgTnLMrKXKn34IV747pqDBAt8B6EHC5fZC8ri2P9MOJBZY38ICaMEw7QlfqYo2cICPyFNwgyASeJuEJGXjj0xb9g3LoQd4aNGctqCU3swqP5m30OPJhlHrTcAYUhyFoDbIYWdN8XL11ajL31WC24X/ROipfTZrLc3E1+r1+q8iBoMTDXX/7PjO3YiZ3B6ZBEjtBqYdYoUUx7gumLN/9IxoSyZIFq0YS96lC3Lr6gLJMExbXugdTJ49L6fsNPSWwmBeeOHthuwrYDqqU/V7J2jb94o4Rtwp4CKr0/UjzZEmfkjvcjVpOFnDDjgBlLf3artRaIYl5T9cYf9yO93Guz+A/e8hyo+f5QrH2awoLNhqeXqhCr/hmtXW2Oc9u8Jky0v5WrqnnhbZ2frD1CU89yl5ccvbCxgqoG+5zPKJGYbt1FLHZls4o8RUljOflM+G7/vuXBdvoRe4+Q9mwGWpM9ymbmVp8DJG4YUDxr5S4RcEhM2qD5PAcUxpOihg5n19DN18qPkrUG/bDOzl3jk8S//NugCceEg5gxWkoCsC8nDBMr5A2YpMrvqMLyySFh/0riIf/UP3uyPTQUvPstNpYaiL2XBlSbmhx5z4SRPnqkKaqG8Dw4gPz9L8HWu37bWvCiHEM41mfcqOBSFlQDiIeG8NOrXXU+YFu3QQn2w5mDbEVkHQ5fQ5o5nGyMFQ8JNJEKkb9iXABVH8W4Fv/tKaqSJP2tLZUuw0MkcA5KGLXL6HwM9/ni0ZbsAegtxXec/IgIgPIq3sFFSXYKbAexuKA08rTmTMZF79AU807Fk+dQUSj4WOUH70UzEMjsp/0lV0qkU33OhPfcQk2bGnFWrM2sZ1+uKzvZrWaCXM7O/9sfkl7i/Qmpf6eIAkdR8ApB4VWoRjhNncdrVyGThM1h1b4XS//64W7t0fsjp7ME0JHv25ZYeRSWUTA9UeT+7Piub4qIkL+DGP7t7NgukqJCucR/FXcYPykCVWEsXNnzGcL9ZHayJhcz+ORDjjcfvwpEpqRfV2LPbD2Yknm1XiNYjvo9RrFw4x7x9Vwl5LgndOpMYPf7Scup0JWbtAYjZ+hifZXMHmQYe9tkNHgsSak7P0YewgLj+sdRqy/U+TVtFs5vpe02oWw3dZCF62YIoJ4sTUY78YSc61KVo18f/+R3JLqpWhD8wI/MsJ6bvvfFkBFezcGoHSzeExXSIIFNJS0hVIQLYMSNGOZ5GKfcq3IVS1xjSJMG4dwK1TUq4CkZY5RTAs8ZURxEHk+JdCNPfp9k2B1FgwOfb+34Rqpy6XjBuOGlYi7Bztg0JbFAdZ54tCH7AZM6solfc+HK/VnCKRuA14du/1ChPXteZv5d5Ia837HjvyjpL8DZaCPvR+GeUGVdA==' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' -H 'Referer: https://sl.se/sv/mitt-sl/konto/' -H 'Accept-Encoding: gzip, deflate, br' -H 'Connection: keep-alive' -H 'CSRF-GUARD-URL: https://sl.se/sv/mitt-sl/konto/' --data-binary '{"travel_card":{"serial_number":"198113854","end_date":"@@DATE@@","limit":5000,"skip":0}}' --compressed'''
    run(command, date(year=2018, month=1, day=1), date(year=2018, month=12, day=31))