sitemap_test.py

from requests import get
from bs4 import BeautifulSoup


def get_urls_sitemap(sitemap_url = ''):
	print("Scan sitemap "+sitemap_url)
	res = get(sitemap_url).text
	soup = BeautifulSoup(res, 'xml')
	urls = soup.select('url')
	urls_test = []
	if len(urls) > 0:
		for url in urls:
			loc = url.select('loc')[0].getText()
			# print(loc)
			urls_test.append(loc)
		return urls_test
	else:
		return None



URL = input("URL sitemap.xml: ")

RES_XML = get(URL).text
soup = BeautifulSoup(RES_XML, 'xml')
urls_res = []
sitemaps = soup.select('sitemap')
if len(sitemaps) > 0:
	print("Parsing sitenaos ")
	url_sitemaps = []
	for sitemap in sitemaps:
		url = sitemap.select('loc')[0].getText()
		urls_res += get_urls_sitemap(url)
else:
	urls_res += get_urls_sitemap(URL)

print("Search urls: " + str(len(urls_res)))
print("Start test links...")
for u in urls_res:
	res = get(u);
	if (res.status_code != 200):
		print("Problem in: " + u + " code: "+ str(res.status_code))
print("Testing finish")