Firepup650/bitbot-3.11-fork
2
0
Fork 0
Code Issues Pull requests Activity Actions

add flag to disable encoding detection

Browse source
This commit is contained in:
jesopo 2019-09-09 14:59:08 +01:00
parent 5ef2b7af27
commit b83f5d9e30
1 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/utils/http.py
View file

@ -86,9 +86,9 @@ def _find_encoding(soup: bs4.BeautifulSoup) -> typing.Optional[str]:
def request(url: str, method: str="GET", get_params: dict={}, def request(url: str, method: str="GET", get_params: dict={},
post_data: typing.Any=None, headers: dict={}, post_data: typing.Any=None, headers: dict={},
json_data: typing.Any=None, code: bool=False, json: bool=False, json_data: typing.Any=None, code: bool=False, json: bool=False,
soup: bool=False, parser: str="lxml", fallback_encoding: str="utf8", soup: bool=False, parser: str="lxml", detect_encoding: bool=True,
allow_redirects: bool=True, check_content_type: bool=True fallback_encoding: str="utf8", allow_redirects: bool=True,
) -> Response: check_content_type: bool=True) -> Response:
if not urllib.parse.urlparse(url).scheme: if not urllib.parse.urlparse(url).scheme:
url = "http://%s" % url url = "http://%s" % url
@ -121,9 +121,8 @@ def request(url: str, method: str="GET", get_params: dict={},
response_headers = utils.CaseInsensitiveDict(dict(response.headers)) response_headers = utils.CaseInsensitiveDict(dict(response.headers))
content_type = response.headers.get("Content-Type", "").split(";", 1)[0] content_type = response.headers.get("Content-Type", "").split(";", 1)[0]
souped = None
encoding = response.encoding encoding = response.encoding
if content_type and content_type in SOUP_CONTENT_TYPES: if detect_encoding and content_type and content_type in SOUP_CONTENT_TYPES:
souped = bs4.BeautifulSoup(response_content, parser) souped = bs4.BeautifulSoup(response_content, parser)
encoding = _find_encoding(souped) or encoding or "iso-8859-1" encoding = _find_encoding(souped) or encoding or "iso-8859-1"