104 lines
2.8 KiB
Python
104 lines
2.8 KiB
Python
import bs4
|
|
from urllib.request import Request, urlopen
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import urlparse, urlunparse, urljoin
|
|
|
|
|
|
def spider(target, exclude):
|
|
parsed_target = urlparse(target)
|
|
return spider_rec(dict(), target, parsed_target, exclude)
|
|
|
|
|
|
def spider_rec(page_links, current_href, base_parse, exclude):
|
|
target_url = urlunparse(base_parse)
|
|
parse_result = urlparse(urljoin(target_url, current_href))
|
|
req = Request(urlunparse(parse_result))
|
|
|
|
postfix = parse_result.path
|
|
if parse_result.query:
|
|
postfix += "?" + parse_result.query
|
|
|
|
if len(postfix) == 0:
|
|
postfix = "/"
|
|
|
|
try:
|
|
html_page = urlopen(req)
|
|
|
|
if parse_result.hostname == base_parse.hostname:
|
|
page_links[postfix] = []
|
|
|
|
soup = bs4.BeautifulSoup(html_page, "lxml")
|
|
|
|
for link in soup.findAll('a'):
|
|
href = link.get('href')
|
|
href = href.replace(" ", "%20")
|
|
|
|
if "mailto:" not in href:
|
|
if not urlparse(href).hostname:
|
|
href_parse = urlparse(urljoin(target_url, href))
|
|
href = href_parse.path
|
|
|
|
if href_parse.query:
|
|
href += "?" + href_parse.query
|
|
|
|
page_links[postfix].append(href)
|
|
|
|
if href not in page_links.keys():
|
|
found = False
|
|
for d in exclude:
|
|
if d in href:
|
|
found = True
|
|
break
|
|
|
|
if found:
|
|
continue
|
|
|
|
spider_rec(page_links, href, base_parse, exclude)
|
|
except HTTPError as e:
|
|
page_links[postfix] = e
|
|
|
|
return page_links
|
|
|
|
|
|
def main():
|
|
print("Reading conf...")
|
|
|
|
conf = []
|
|
with open('crawl.conf', 'r') as file:
|
|
for line in file.readlines():
|
|
line = line.replace("\n", "")
|
|
line = line.replace("\r", "")
|
|
conf.append(line)
|
|
|
|
target = conf[1]
|
|
ignores = conf[3:]
|
|
|
|
print("Crawling site...")
|
|
pages = spider(target, ignores)
|
|
|
|
count = 0
|
|
for link in pages.keys():
|
|
if type(pages[link]) == HTTPError:
|
|
count += 1
|
|
|
|
found = []
|
|
for search_link in pages.keys():
|
|
if type(pages[search_link]) != HTTPError:
|
|
for href in pages[search_link]:
|
|
if href == link:
|
|
found.append(search_link)
|
|
|
|
print(''.join(['='] * 100))
|
|
print(link, pages[link].status, pages[link].reason)
|
|
print(''.join(['-'] * 100))
|
|
print("Found in:")
|
|
|
|
for href in found:
|
|
print(href)
|
|
|
|
print(''.join(['='] * 100), "\n")
|
|
|
|
print("Done.")
|
|
|
|
|
|
main()
|