Python @ Polytech'Lille - TP2

python-gis2a4-tp1.py

#!/usr/bin/python

# Goal: scrap the "Annuaire" web page of Polytech Lille to gather the list of
#       all students and their id (encoded in base 64). Print the number of 
#       entry found
#
# You can use requests for scraping and re for extracting data from the page

import base64
import requests
import re

FILTER = '<a href=\'annuaire.php\?a=([\w=]+)\'>([^&]+)\&nbsp;([^<]+)<'

annuaire = {}

if __name__ == '__main__':

	for c in range(ord('a'),ord('z')+1):
		r = requests.post('http://www.polytech-lille.fr/annuaire.php', 
			data={'nom':chr(c)})
		groups = re.findall(FILTER, r.text)
		for i in groups:
			code = int(str(base64.b64decode(i[0]),'utf-8').partition('*')[0])
			firstname = i[1]
			lastname = i[2]
			if code not in annuaire:
				annuaire[code] = (firstname,lastname)
		print(chr(c) + " loaded")

print("number of entries: %i" % len(annuaire))