109 lines
3.0 KiB
Python
109 lines
3.0 KiB
Python
import urllib.request,urllib.parse,urllib.error
|
|
from urllib.error import HTTPError
|
|
from urllib.error import URLError
|
|
from urllib.parse import urljoin
|
|
import requests
|
|
import re
|
|
|
|
|
|
|
|
class url:
|
|
|
|
url = "" # the url of the website to be checked
|
|
sites = dict() # dic. with all sites and urls on those sites
|
|
does_work = [] # array with all prev. positiv tested urls
|
|
does_not_work = dict() # dic. with all not working urls and the site that linked there
|
|
header_values = {
|
|
'Connection:' : 'Keep-alive',
|
|
'name' : 'Michael Foord',
|
|
'location' : 'Northampton',
|
|
'language' : 'English',
|
|
'User-Agent': 'Mozilla 4/0'}
|
|
|
|
def __init__(self, url):
|
|
self.url = urllib.request.urlopen(url).geturl()
|
|
|
|
|
|
def make_url(self, link, start):
|
|
ret_link = urljoin(start, link)
|
|
|
|
return ret_link
|
|
|
|
def test_url(self, link, root):
|
|
|
|
if link in self.sites or link in self.does_work:
|
|
return True
|
|
elif link in self.does_not_work:
|
|
return False
|
|
else:
|
|
try:
|
|
|
|
header = urllib.parse.urlencode(self.header_values)
|
|
header=header.encode('ascii')
|
|
request = urllib.request.Request(link, header)
|
|
response = urllib.request.urlopen(request)
|
|
self.does_work.append(link)
|
|
#print(" works " + link)
|
|
return True
|
|
|
|
except (urllib.error.HTTPError, urllib.error.URLError, ValueError):
|
|
self.does_not_work[link]=root
|
|
#print(" doesn't work " + link)
|
|
return False
|
|
|
|
def get_actual_urls(self, links, root):
|
|
temp_links = []
|
|
for each_link in links:
|
|
|
|
if each_link.startswith("http") | each_link.startswith("//"):
|
|
temp_links.append(each_link)
|
|
else:
|
|
temp_links.append(urljoin(root, each_link))
|
|
|
|
for each_temp_link in temp_links:
|
|
self.test_url(each_temp_link, root)
|
|
|
|
return temp_links
|
|
|
|
def run_check(self, root=None): # root is the url of the current Site
|
|
|
|
if root == None:
|
|
root = self.url
|
|
else:
|
|
pass
|
|
|
|
if root in self.sites or self.url.rsplit('/', 1)[0] not in root or not self.test_url(root, root):
|
|
return
|
|
|
|
header = urllib.parse.urlencode(self.header_values)
|
|
header=header.encode('ascii')
|
|
|
|
request = urllib.request.Request(root, header)
|
|
http_response = urllib.request.urlopen(request)
|
|
root = http_response.geturl()
|
|
response_data= http_response.read()
|
|
|
|
|
|
links = re.findall(r'href="(.*?)"' , str(response_data))
|
|
|
|
links = self.get_actual_urls(links, root)
|
|
print(root, links)
|
|
|
|
self.sites[root]=links
|
|
for each_link in links:
|
|
self.run_check(each_link)
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|