bMorgan/link-crawler
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

limit "broken codes"

Browse source
This commit is contained in:
bMorgan01 2022-09-25 12:09:04 -06:00
parent 72a195a0e7
commit 75d88ae1b1
1 changed files with 5 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
main.py
View file

@ -64,10 +64,11 @@ def spider_rec(page_links, current_href, base_parse, exclude):
spider_rec(page_links, href, base_parse, exclude)
except HTTPError as e:
if parse_result.hostname == base_parse.hostname:
page_links[postfix] = e
else:
page_links[current_href] = e
if e.code == 400 or e.code in range(404, 500):
if parse_result.hostname == base_parse.hostname:
page_links[postfix] = e
else:
page_links[current_href] = e
return page_links