made it possible to get http code even when a HTTPError kicks up.
This commit is contained in:
parent
16d1af9057
commit
78f1f68a98
1 changed files with 6 additions and 3 deletions
9
Utils.py
9
Utils.py
|
@ -1,4 +1,4 @@
|
||||||
import json, re, traceback, urllib.request, urllib.parse
|
import json, re, traceback, urllib.request, urllib.parse, urllib.error
|
||||||
import bs4
|
import bs4
|
||||||
|
|
||||||
USER_AGENT = ("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
|
USER_AGENT = ("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
|
||||||
|
@ -47,9 +47,12 @@ def get_url(url, **kwargs):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = urllib.request.urlopen(request)
|
response = urllib.request.urlopen(request)
|
||||||
except:
|
except urllib.error.HTTPError as e:
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
return False
|
if kwargs.get("code"):
|
||||||
|
return e.code, False
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
response_content = response.read()
|
response_content = response.read()
|
||||||
encoding = response.info().get_content_charset()
|
encoding = response.info().get_content_charset()
|
||||||
|
|
Loading…
Reference in a new issue