Firepup650/bitbot-3.11-fork
2
0
Fork 0
Code Issues Pull requests Activity Actions

Don't try to .decode non-html things, default iso-lat-1 for non-html too

Browse source
This commit is contained in:
jesopo 2019-09-09 16:17:26 +01:00
parent c8ece388cc
commit a9b106c6be
1 changed files with 5 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/utils/http.py
View file

@ -121,10 +121,10 @@ def request(url: str, method: str="GET", get_params: dict={},
response_headers = utils.CaseInsensitiveDict(dict(response.headers)) response_headers = utils.CaseInsensitiveDict(dict(response.headers))
content_type = response.headers.get("Content-Type", "").split(";", 1)[0] content_type = response.headers.get("Content-Type", "").split(";", 1)[0]
encoding = response.encoding encoding = response.encoding or "iso-8859-1"
if detect_encoding and content_type and content_type in SOUP_CONTENT_TYPES: if detect_encoding and content_type and content_type in SOUP_CONTENT_TYPES:
souped = bs4.BeautifulSoup(response_content, parser) souped = bs4.BeautifulSoup(response_content, parser)
encoding = _find_encoding(souped) or encoding or "iso-8859-1" encoding = _find_encoding(souped) or encoding
def _decode_data(): def _decode_data():
return response_content.decode(encoding) return response_content.decode(encoding)
@ -137,15 +137,15 @@ def request(url: str, method: str="GET", get_params: dict={},
raise HTTPWrongContentTypeException( raise HTTPWrongContentTypeException(
"Tried to soup non-html/non-xml data (%s)" % content_type) "Tried to soup non-html/non-xml data (%s)" % content_type)
if json and response_content:
data = _decode_data() data = _decode_data()
if json and data:
try: try:
return Response(response.status_code, _json.loads(data), return Response(response.status_code, _json.loads(data),
response_headers) response_headers)
except _json.decoder.JSONDecodeError as e: except _json.decoder.JSONDecodeError as e:
raise HTTPParsingException(str(e), data) raise HTTPParsingException(str(e), data)
return Response(response.status_code, data, response_headers) return Response(response.status_code, response_content, response_headers)
def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]: def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
responses = {} responses = {}