example of using selenium to parse HTML

selenium_parse_html.py

# Get a table object
table = driver.find_element_by_class_name('my_table_class')

# Get all rows, including rows in rows
rows = table.find_elements_by_tag_name('tr')

# Get header which is in the first row
# Get text from <th> element
header = rows[0]
header_text_list = [i.text for i in header.find_elements_by_tag_name('th')]

# Get direct rows of top level table
direct_trs = table.find_elements_by_xpath('.//tbody//tr')

# This uses lxml to parse source code to get valid td tag which has raw data
tr_table = []
for tr in direct_trs:
    # Get HTML source code string to accelerate
    root = lxml.html.fromstring(tr.get_attribute('innerHTML'))
    tr_text = []
    # Assume <tr> has a few <td> inside
    for td in root.xpath('td'):
        tr_text.append(td.text)