Last active
October 23, 2020 20:41
-
-
Save oktomus/bd88007e8cc8aee2aaab7ea364fb8300 to your computer and use it in GitHub Desktop.
Tell me the difference
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
import urllib.request | |
import time | |
from bs4 import BeautifulSoup | |
import difflib | |
import os | |
import smtplib | |
import email.message | |
import io | |
from requests_html import HTMLSession | |
import sys | |
things_to_test = [ | |
#{ | |
#'name': 'clock', | |
#'url': 'https://free-onlineclock.com/online-clock', | |
#'div': 'vue_clock' | |
#}, | |
] | |
local_mail = 'localhost@localhost' | |
script_folder = os.path.dirname(os.path.realpath(__file__)) | |
def send_self_local_mail(subject, body): | |
sender_mail = local_mail | |
receivers_mail = [local_mail] | |
m = email.message.Message() | |
m['From'] = local_mail | |
m['To'] = local_mail | |
m['Subject'] = subject | |
m.set_payload(body); | |
m = m.as_string() | |
try: | |
smtpObj = smtplib.SMTP('localhost') | |
smtpObj.sendmail(sender_mail, receivers_mail, m) | |
print("Successfully sent email") | |
except Exception as e: | |
print("Error: unable to send email") | |
print(str(e)) | |
_, _, stack = sys.exc_info() | |
print(str(stack)) | |
def tell_me_the_difference(url, div_id, name): | |
print('Tell me the difference\nname={}\nurl={}\nid={}'.format(name, url, div_id)) | |
session = HTMLSession() | |
try: | |
r = session.get(url) | |
try: | |
r.raise_for_status() | |
except ValueError as e: | |
raise('Dead link') | |
r.html.render(sleep = 2, timeout = 20) # this call executes the js in the page | |
#soup = BeautifulSoup(response.text, 'html.parser') | |
details = r.html.find('#{}'.format(div_id))[0].text.replace('\t', ' ').splitlines() | |
#details = str(soup.findAll(id=div_id)[0]).replace('\t', ' ').splitlines() | |
file_path = os.path.join(script_folder, name) | |
# Write it in a file the first time. | |
if os.path.exists(file_path): | |
previous_details = None | |
with io.open(file_path, 'r', encoding='utf-8') as file: | |
previous_details = file.read().replace('\t', ' ').splitlines() | |
diff = difflib.ndiff(details, previous_details) | |
changes = [l for l in diff if l.startswith('+ ') or l.startswith('- ')] | |
has_changes = len(changes) > 0 | |
pretty_changes = '\n'.join(changes) | |
print('DIFF?: ', has_changes) | |
if has_changes: | |
send_self_local_mail('{} diff'.format(name), '{}\n\n{}'.format(url, pretty_changes)) | |
with io.open(file_path, 'w', encoding='utf-8') as file: | |
file.write('\n'.join(details)) | |
except Exception as e: | |
print('FAILURE') | |
print(str(e)) | |
_, _, stack = sys.exc_info() | |
send_self_local_mail('{} failed'.format(name), '{}\n\n{}\n\n{}'.format(url, str(e), stack)) | |
if __name__ == '__main__': | |
for thing in things_to_test: | |
url = thing['url'] | |
div_id = thing['div'] | |
name = thing['name'] | |
tell_me_the_difference(url, div_id, name) | |
time.sleep(10) | |
sys.exit(0) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment