From 1df82c1cb20eab4611c1f260818ac9c0ac9bec64 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 9 Sep 2019 14:48:26 +0100 Subject: [PATCH] still default to iso-latin-1 if no on-page or in-header content-type is present --- src/utils/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/http.py b/src/utils/http.py index 232b1c6b..b20b2264 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -125,7 +125,7 @@ def request(url: str, method: str="GET", get_params: dict={}, encoding = response.encoding if content_type and content_type in SOUP_CONTENT_TYPES: souped = bs4.BeautifulSoup(response_content, parser) - encoding = _find_encoding(souped) or encoding + encoding = _find_encoding(souped) or encoding or "iso-8859-1" def _decode_data(): return response_content.decode(encoding)