From 2d21dfa22931bdb887ccccba4a33e6fa4755e94b Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 26 Nov 2019 13:41:40 +0000 Subject: [PATCH] utils.http.Response.data should always be `bytes` - add .decode and .soup --- modules/bitcoin.py | 9 ++++--- modules/duckduckgo.py | 6 ++--- modules/eval_python.py | 2 +- modules/git_webhooks/github.py | 6 ++--- modules/github.py | 43 ++++++++++++++++++---------------- modules/google.py | 17 +++++++------- modules/imdb.py | 12 ++++------ modules/ip_addresses.py | 22 ++++++++--------- modules/location.py | 9 ++++--- modules/markov.py | 2 +- modules/onionoo.py | 6 ++--- modules/rss.py | 5 ++-- modules/shorturl.py | 6 ++--- modules/soundcloud.py | 6 ++--- modules/spotify.py | 14 +++++------ modules/thesaurus.py | 15 ++++++------ modules/trakt.py | 24 +++++++++---------- modules/translate.py | 6 ++--- modules/urbandictionary.py | 8 +++---- modules/weather.py | 20 ++++++++-------- modules/wolframalpha.py | 4 ++-- modules/youtube.py | 2 +- src/utils/http.py | 32 ++++++------------------- 23 files changed, 127 insertions(+), 149 deletions(-) diff --git a/modules/bitcoin.py b/modules/bitcoin.py index 91fd84fb..c130d20d 100644 --- a/modules/bitcoin.py +++ b/modules/bitcoin.py @@ -12,16 +12,15 @@ class Module(ModuleManager.BaseModule): :usage: [currency] """ currency = (event["args"] or "USD").upper() - page = utils.http.request("https://blockchain.info/ticker", - json=True) + page = utils.http.request("https://blockchain.info/ticker").json() if page: - if currency in page.data: - conversion = page.data[currency] + if currency in page: + conversion = page[currency] buy, sell = conversion["buy"], conversion["sell"] event["stdout"].write("1 BTC = %.2f %s (buy) %.2f %s " "(sell)" % (buy, currency, sell, currency)) else: event["stderr"].write("Unknown currency, available " - "currencies: %s" % ", ".join(page.data.keys())) + "currencies: %s" % ", ".join(page.keys())) else: raise utils.EventResultsError() diff --git a/modules/duckduckgo.py b/modules/duckduckgo.py index 6660d158..f43d2376 100644 --- a/modules/duckduckgo.py +++ b/modules/duckduckgo.py @@ -18,9 +18,9 @@ class Module(ModuleManager.BaseModule): if phrase: page = utils.http.request(URL_DDG, get_params={ "q": phrase, "format": "json", "no_html": "1", - "no_redirect": "1"}, json=True) + "no_redirect": "1"}).json() - if page and page.data["AbstractURL"]: - event["stdout"].write(page.data["AbstractURL"]) + if page and page["AbstractURL"]: + event["stdout"].write(page["AbstractURL"]) else: event["stderr"].write("No results found") diff --git a/modules/eval_python.py b/modules/eval_python.py index b16671bc..5454b7d3 100644 --- a/modules/eval_python.py +++ b/modules/eval_python.py @@ -21,6 +21,6 @@ class Module(ModuleManager.BaseModule): if page and page.data: event["stdout"].write("%s: %s" % (event["user"].nickname, - page.data.rstrip("\n"))) + page.decode().rstrip("\n"))) else: event["stderr"].write("%s: failed to eval" % event["user"].nickname) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index 5a252c65..b7f13805 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -171,7 +171,7 @@ class GitHub(object): self.log.debug("git.io shortening: %s" % url) try: page = utils.http.request("https://git.io", method="POST", - post_data={"url": url}, detect_encoding=False) + post_data={"url": url}) return page.headers["Location"] except utils.http.HTTPTimeoutException: self.log.warn( @@ -294,11 +294,11 @@ class GitHub(object): action_desc = "committed to %s" % number commits_url = data["pull_request"]["commits_url"] - commits = utils.http.request(commits_url, json=True) + commits = utils.http.request(commits_url).json() if commits: seen_before = False new_commits = [] - for commit in commits.data: + for commit in commits: if seen_before: new_commits.append({"id": commit["sha"], "message": commit["commit"]["message"]}) diff --git a/modules/github.py b/modules/github.py index 6212b9dd..b528ad2b 100644 --- a/modules/github.py +++ b/modules/github.py @@ -70,17 +70,18 @@ class Module(ModuleManager.BaseModule): headers = {} if not oauth2_token == None: headers["Authorization"] = "token %s" % oauth2_token - request = utils.http.Request(url, headers=headers, json=True) + request = utils.http.Request(url, headers=headers) return utils.http.request(request) def _commit(self, username, repository, commit): page = self._get(API_COMMIT_URL % (username, repository, commit)) if page and page.code == 200: + page = page.json() repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO) - sha = utils.irc.color(page.data["sha"][:8], COLOR_ID) + sha = utils.irc.color(page["sha"][:8], COLOR_ID) return "(%s@%s) %s - %s %s" % (repo, sha, - page.data["author"]["login"], page.data["commit"]["message"], - self._short_url(page.data["html_url"])) + page["author"]["login"], page["commit"]["message"], + self._short_url(page["html_url"])) def _parse_commit(self, target, ref): username, repository, commit = self._parse_ref(target, ref, "@") return self._commit(username, repository, commit) @@ -116,21 +117,21 @@ class Module(ModuleManager.BaseModule): def _parse_issue(self, page, username, repository, number): repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO) number = utils.irc.color("#%s" % number, COLOR_ID) - labels = [label["name"] for label in page.data["labels"]] + labels = [label["name"] for label in page["labels"]] labels_str = "" if labels: labels_str = "[%s] " % ", ".join(labels) - url = self._short_url(page.data["html_url"]) + url = self._short_url(page["html_url"]) - state = page.data["state"] + state = page["state"] if state == "open": state = utils.irc.color("open", COLOR_NEUTRAL) elif state == "closed": state = utils.irc.color("closed", COLOR_NEGATIVE) return "(%s issue%s, %s) %s %s%s" % ( - repo, number, state, page.data["title"], labels_str, url) + repo, number, state, page["title"], labels_str, url) def _get_issue(self, username, repository, number): return self._get(API_ISSUE_URL % (username, repository, number)) @@ -147,21 +148,21 @@ class Module(ModuleManager.BaseModule): page = self._get_issue(username, repository, number) if page and page.code == 200: - self._parse_issue(page, username, repository, number) + self._parse_issue(page.json(), username, repository, number) else: event["stderr"].write("Could not find issue") def _parse_pull(self, page, username, repository, number): repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO) number = utils.irc.color("#%s" % number, COLOR_ID) - branch_from = page.data["head"]["label"] - branch_to = page.data["base"]["label"] - added = self._added(page.data["additions"]) - removed = self._removed(page.data["deletions"]) - url = self._short_url(page.data["html_url"]) + branch_from = page["head"]["label"] + branch_to = page["base"]["label"] + added = self._added(page["additions"]) + removed = self._removed(page["deletions"]) + url = self._short_url(page["html_url"]) - state = page.data["state"] - if page.data["merged"]: + state = page["state"] + if page["merged"]: state = utils.irc.color("merged", COLOR_POSITIVE) elif state == "open": state = utils.irc.color("open", COLOR_NEUTRAL) @@ -170,7 +171,7 @@ class Module(ModuleManager.BaseModule): return "(%s PR%s, %s) %s → %s [%s/%s] %s %s" % ( repo, number, state, branch_from, branch_to, added, removed, - page.data["title"], url) + page["title"], url) def _get_pull(self, username, repository, number): return self._get(API_PULL_URL % (username, repository, number)) @utils.hook("received.command.ghpull", min_args=1) @@ -187,7 +188,7 @@ class Module(ModuleManager.BaseModule): page = self._get_pull(username, repository, number) if page and page.code == 200: - self._parse_pull(page, username, repository, number) + self._parse_pull(page.json(), username, repository, number) else: event["stderr"].write("Could not find pull request") @@ -198,9 +199,11 @@ class Module(ModuleManager.BaseModule): page = self._get_issue(username, repository, number) if page and page.code == 200: - if "pull_request" in page.data: + page = page.json() + if "pull_request" in page: pull = self._get_pull(username, repository, number) - return self._parse_pull(pull, username, repository, number) + return self._parse_pull(pull.json(), username, repository, + number) else: return self._parse_issue(page, username, repository, number) else: diff --git a/modules/google.py b/modules/google.py index 47700eb9..6cddb8a3 100644 --- a/modules/google.py +++ b/modules/google.py @@ -27,13 +27,13 @@ class Module(ModuleManager.BaseModule): safe = "active" if safe_setting else "off" page = utils.http.request(URL_GOOGLESEARCH, get_params={ - "q": phrase, "key": self.bot.config[ - "google-api-key"], "cx": self.bot.config[ - "google-search-id"], "prettyPrint": "true", - "num": 1, "gl": "gb", "safe": safe}, json=True) + "q": phrase, "prettyPrint": "true", "num": 1, "gl": "gb", + "key": self.bot.config["google-api-key"], + "cx": self.bot.config["google-search-id"], + "safe": safe}).json() if page: - if "items" in page.data and len(page.data["items"]): - item = page.data["items"][0] + if "items" in page and len(page["items"]): + item = page["items"][0] link = item["link"] title = utils.parse.line_normalise(item["title"]) event["stdout"].write( @@ -54,10 +54,11 @@ class Module(ModuleManager.BaseModule): phrase = event["args"] or event["target"].buffer.get() if phrase: page = utils.http.request(URL_GOOGLESUGGEST, get_params={ - "output": "json", "client": "hp", "gl": "gb", "q": phrase}) + "output": "json", "client": "hp", "gl": "gb", "q": phrase} + ).json() if page: # google gives us jsonp, so we need to unwrap it. - page = page.data.split("(", 1)[1][:-1] + page = page.split("(", 1)[1][:-1] page = json.loads(page) suggestions = page[1] suggestions = [utils.http.strip_html(s[0]) for s in suggestions] diff --git a/modules/imdb.py b/modules/imdb.py index d6b0a487..9436768e 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -17,15 +17,13 @@ class Module(ModuleManager.BaseModule): :usage: """ page = utils.http.request(URL_OMDB, get_params={ - "t": event["args"], - "apikey": self.bot.config["omdbapi-api-key"]}, - json=True) + "apikey": self.bot.config["omdbapi-api-key"], + "t": event["args"]}).json() if page: - if "Title" in page.data: + if "Title" in page: event["stdout"].write("%s, %s (%s) %s (%s/10.0) %s" % ( - page.data["Title"], page.data["Year"], page.data["Runtime"], - page.data["Plot"], page.data["imdbRating"], - URL_IMDBTITLE % page.data["imdbID"])) + page["Title"], page["Year"], page["Runtime"], page["Plot"], + page["imdbRating"], URL_IMDBTITLE % page["imdbID"])) else: event["stderr"].write("Title not found") else: diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index 744900f6..73ec061f 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -85,19 +85,17 @@ class Module(ModuleManager.BaseModule): :usage: :prefix: GeoIP """ - page = utils.http.request(URL_GEOIP % event["args_split"][0], - json=True) + page = utils.http.request(URL_GEOIP % event["args_split"][0]).json() if page: - if page.data["status"] == "success": - data = page.data["query"] - data += " | Organisation: %s" % page.data["org"] - data += " | City: %s" % page.data["city"] - data += " | Region: %s (%s)" % (page.data["regionName"], - page.data["countryCode"]) - data += " | ISP: %s" % page.data["isp"] - data += " | Lon/Lat: %s/%s" % (page.data["lon"], - page.data["lat"]) - data += " | Timezone: %s" % page.data["timezone"] + if page["status"] == "success": + data = page["query"] + data += " | Organisation: %s" % page["org"] + data += " | City: %s" % page["city"] + data += " | Region: %s (%s)" % ( + page["regionName"], page["countryCode"]) + data += " | ISP: %s" % page["isp"] + data += " | Lon/Lat: %s/%s" % (page["lon"], page["lat"]) + data += " | Timezone: %s" % page["timezone"] event["stdout"].write(data) else: event["stderr"].write("No geoip data found") diff --git a/modules/location.py b/modules/location.py index 1858a915..dfef9244 100644 --- a/modules/location.py +++ b/modules/location.py @@ -14,11 +14,10 @@ class Module(ModuleManager.BaseModule): self.exports.add("get-location", self._get_location) def _get_location(self, s): - page = utils.http.request(URL_OPENCAGE, get_params={ - "q": s, "key": self.bot.config["opencagedata-api-key"], "limit": "1" - }, json=True) - if page and page.data["results"]: - result = page.data["results"][0] + page = utils.http.request(URL_OPENCAGE, get_params={"limit": "1", + "q": s, "key": self.bot.config["opencagedata-api-key"]}).json() + if page and page["results"]: + result = page["results"][0] timezone = result["annotations"]["timezone"]["name"] lat = result["geometry"]["lat"] lon = result["geometry"]["lng"] diff --git a/modules/markov.py b/modules/markov.py index c2d4a792..9ba9d31f 100644 --- a/modules/markov.py +++ b/modules/markov.py @@ -39,7 +39,7 @@ class Module(ModuleManager.BaseModule): if page.code == 200: event["stdout"].write("Importing...") self._load_thread = threading.Thread(target=self._load_loop, - args=[event["target"].id, page.data]) + args=[event["target"].id, page.decode()]) self._load_thread.daemon = True self._load_thread.start() else: diff --git a/modules/onionoo.py b/modules/onionoo.py index 427cc487..bfd882cd 100644 --- a/modules/onionoo.py +++ b/modules/onionoo.py @@ -11,9 +11,9 @@ URL_RELAY_SEARCH_DETAILS = "https://metrics.torproject.org/rs.html#details/" def _get_relays_details(search): page = utils.http.request( - URL_ONIONOO_DETAILS, get_params={"search": search}, json=True) - if page and "relays" in page.data: - return page.data["relays"] + URL_ONIONOO_DETAILS, get_params={"search": search}).json() + if page and "relays" in page: + return page["relays"] raise utils.EventResultsError() def _format_relay_summary_message(relays, search): diff --git a/modules/rss.py b/modules/rss.py index e8cba9e8..cb68a304 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -67,7 +67,7 @@ class Module(ModuleManager.BaseModule): # async url get failed continue - feed = feedparser.parse(pages[url].data) + feed = feedparser.parse(pages[url].decode()) feed_title = feed["feed"].get("title", None) max_ids = len(feed["entries"])*10 @@ -105,8 +105,7 @@ class Module(ModuleManager.BaseModule): def _get_entries(self, url, max: int=None): try: - data = utils.http.request(url) - feed = feedparser.parse(data.data) + feed = feedparser.parse(utils.http.request(url).data) except Exception as e: self.log.warn("failed to parse RSS %s", [url], exc_info=True) feed = None diff --git a/modules/shorturl.py b/modules/shorturl.py index e18cdcfd..10bd9dd3 100644 --- a/modules/shorturl.py +++ b/modules/shorturl.py @@ -57,10 +57,10 @@ class Module(ModuleManager.BaseModule): access_token = self.bot.config.get("bitly-api-key", None) if access_token: page = utils.http.request(URL_BITLYSHORTEN, get_params={ - "access_token": access_token, "longUrl": url}, json=True) + "access_token": access_token, "longUrl": url}).json() - if page and page.data["data"]: - return page.data["data"]["url"] + if page["data"]: + return page["data"]["url"] return None def _find_url(self, target, args): diff --git a/modules/soundcloud.py b/modules/soundcloud.py index 74b57efc..9355caef 100644 --- a/modules/soundcloud.py +++ b/modules/soundcloud.py @@ -45,11 +45,11 @@ class Module(ModuleManager.BaseModule): page = utils.http.request( URL_SOUNDCLOUD_TRACK if has_query else URL_SOUNDCLOUD_RESOLVE, - get_params=get_params, json=True) + get_params=get_params).json() if page: - if len(page.data): - page = page.data[0] if has_query else page + if len(page): + page = page[0] if has_query else page title = page["title"] user = page["user"]["username"] duration = time.strftime("%H:%M:%S", time.gmtime(page[ diff --git a/modules/spotify.py b/modules/spotify.py index 962b0a62..9438b149 100644 --- a/modules/spotify.py +++ b/modules/spotify.py @@ -24,12 +24,11 @@ class Module(ModuleManager.BaseModule): page = utils.http.request(URL_TOKEN, method="POST", headers={"Authorization": "Basic %s" % bearer}, - post_data={"grant_type": "client_credentials"}, - json=True) + post_data={"grant_type": "client_credentials"}).json() - token = page.data["access_token"] + token = page["access_token"] self._token = token - self._token_expires = time.time()+page.data["expires_in"] + self._token_expires = time.time()+page["expires_in"] return token @utils.hook("received.command.sp", alias_of="spotify") @@ -42,11 +41,10 @@ class Module(ModuleManager.BaseModule): token = self._get_token() page = utils.http.request(URL_SEARCH, get_params={"type": "track", "limit": 1, "q": event["args"]}, - headers={"Authorization": "Bearer %s" % token}, - json=True) + headers={"Authorization": "Bearer %s" % token}).json() if page: - if len(page.data["tracks"]["items"]): - item = page.data["tracks"]["items"][0] + if len(page["tracks"]["items"]): + item = page["tracks"]["items"][0] title = item["name"] artist_name = item["artists"][0]["name"] url = item["external_urls"]["spotify"] diff --git a/modules/thesaurus.py b/modules/thesaurus.py index df988c9e..bd30e67e 100644 --- a/modules/thesaurus.py +++ b/modules/thesaurus.py @@ -14,17 +14,18 @@ class Module(ModuleManager.BaseModule): :usage: [type] """ phrase = event["args_split"][0] - page = utils.http.request(URL_THESAURUS % (self.bot.config[ - "bighugethesaurus-api-key"], phrase), json=True) + page = utils.http.request(URL_THESAURUS % ( + self.bot.config["bighugethesaurus-api-key"], phrase)) syn_ant = event["command"][:3] if page: if page.code == 404: raise utils.EventError("Word not found") + page = page.json() if not len(event["args_split"]) > 1: word_types = [] - for word_type in page.data.keys(): - if syn_ant in page.data[word_type]: + for word_type in page.keys(): + if syn_ant in page[word_type]: word_types.append(word_type) if word_types: word_types = sorted(word_types) @@ -35,11 +36,11 @@ class Module(ModuleManager.BaseModule): event["stderr"].write("No categories available") else: category = event["args_split"][1].lower() - if category in page.data: - if syn_ant in page.data[category]: + if category in page: + if syn_ant in page[category]: event["stdout"].write("%ss for %s: %s" % ( event["command"].title(), phrase, ", ".join( - page.data[category][syn_ant]))) + page[category][syn_ant]))) else: event["stderr"].write("No %ss for %s" % ( event["command"], phrase)) diff --git a/modules/trakt.py b/modules/trakt.py index ad7a6644..dc8596c0 100644 --- a/modules/trakt.py +++ b/modules/trakt.py @@ -25,26 +25,26 @@ class Module(ModuleManager.BaseModule): page = utils.http.request(URL_TRAKT % username, headers={ "Content-Type": "application/json", "trakt-api-version": "2", "trakt-api-key": - self.bot.config["trakt-api-key"]}, json=True, - code=True) + self.bot.config["trakt-api-key"]}) if page: if page.code == 200: - type = page.data["type"] + page = page.json() + type = page["type"] if type == "movie": - title = page.data["movie"]["title"] - year = page.data["movie"]["year"] - slug = page.data["movie"]["ids"]["slug"] + title = page["movie"]["title"] + year = page["movie"]["year"] + slug = page["movie"]["ids"]["slug"] event["stdout"].write( "%s is now watching %s (%s) %s" % ( username, title, year, URL_TRAKTSLUG % ("movie", slug))) elif type == "episode": - season = page.data["episode"]["season"] - episode_number = page.data["episode"]["number"] - episode_title = page.data["episode"]["title"] - show_title = page.data["show"]["title"] - show_year = page.data["show"]["year"] - slug = page.data["show"]["ids"]["slug"] + season = page["episode"]["season"] + episode_number = page["episode"]["number"] + episode_title = page["episode"]["title"] + show_title = page["show"]["title"] + show_year = page["show"]["year"] + slug = page["show"]["ids"]["slug"] event["stdout"].write( "%s is now watching %s s%se%s - %s %s" % ( username, show_title, str(season).zfill(2), diff --git a/modules/translate.py b/modules/translate.py index b12f0d0a..e11de291 100644 --- a/modules/translate.py +++ b/modules/translate.py @@ -35,11 +35,11 @@ class Module(ModuleManager.BaseModule): phrase = phrase.split(" ", 1)[1] page = utils.http.request(URL_TRANSLATE, get_params={ - "client": "gtx", "sl": source_language, - "tl": target_language, "dt": "t", "q": phrase}) + "client": "gtx", "dt": "t", "q": phrase, + "sl": source_language, "tl": target_language}) if page and not page.data.startswith(b"[null,null,"): - data = page.data.decode("utf8") + data = page.decode("utf8") while ",," in data: data = data.replace(",,", ",null,") data = data.replace("[,", "[null,") diff --git a/modules/urbandictionary.py b/modules/urbandictionary.py index b71dc45b..a5098c3f 100644 --- a/modules/urbandictionary.py +++ b/modules/urbandictionary.py @@ -24,11 +24,11 @@ class Module(ModuleManager.BaseModule): term = " ".join(term) page = utils.http.request(URL_URBANDICTIONARY, - get_params={"term": term}, json=True) + get_params={"term": term}).json() if page: - if len(page.data["list"]): - if number > 0 and len(page.data["list"]) > number-1: - definition = page.data["list"][number-1] + if len(page["list"]): + if number > 0 and len(page["list"]) > number-1: + definition = page["list"][number-1] event["stdout"].write("%s: %s" % (definition["word"], definition["definition"].replace("\n", " ").replace( "\r", "").replace(" ", " "))) diff --git a/modules/weather.py b/modules/weather.py index 2b950dfe..76ddf3d4 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -56,24 +56,24 @@ class Module(ModuleManager.BaseModule): args["lat"] = lat args["lon"] = lon - page = utils.http.request(URL_WEATHER, get_params=args, json=True) + page = utils.http.request(URL_WEATHER, get_params=args).json() if page: - if "weather" in page.data: + if "weather" in page: if location_name: location_str = location_name else: - location_parts = [page.data["name"]] - if "country" in page.data["sys"]: - location_parts.append(page.data["sys"]["country"]) + location_parts = [page["name"]] + if "country" in page["sys"]: + location_parts.append(page["sys"]["country"]) location_str = ", ".join(location_parts) - celsius = "%dC" % page.data["main"]["temp"] - fahrenheit = "%dF" % ((page.data["main"]["temp"]*(9/5))+32) - description = page.data["weather"][0]["description"].title() - humidity = "%s%%" % page.data["main"]["humidity"] + celsius = "%dC" % page["main"]["temp"] + fahrenheit = "%dF" % ((page["main"]["temp"]*(9/5))+32) + description = page["weather"][0]["description"].title() + humidity = "%s%%" % page["main"]["humidity"] # wind speed is in metres per second - 3.6* for KMh - wind_speed = 3.6*page.data["wind"]["speed"] + wind_speed = 3.6*page["wind"]["speed"] wind_speed_k = "%sKMh" % round(wind_speed, 1) wind_speed_m = "%sMPh" % round(0.6214*wind_speed, 1) diff --git a/modules/wolframalpha.py b/modules/wolframalpha.py index 968b05ae..2b5c36fe 100644 --- a/modules/wolframalpha.py +++ b/modules/wolframalpha.py @@ -20,13 +20,13 @@ class Module(ModuleManager.BaseModule): page = utils.http.request(URL_WA, get_params={"i": event["args"], "appid": self.bot.config["wolframalpha-api-key"], - "reinterpret": "true", "units": "metric"}, code=True) + "reinterpret": "true", "units": "metric"}) except utils.http.HTTPTimeoutException: page = None if page: if page.code == 200: - event["stdout"].write("%s: %s" % (event["args"], page.data)) + event["stdout"].write("%s: %s" % (event["args"], page.decode())) else: event["stdout"].write("No results") else: diff --git a/modules/youtube.py b/modules/youtube.py index 24a662bf..35fe65dd 100644 --- a/modules/youtube.py +++ b/modules/youtube.py @@ -87,7 +87,7 @@ class Module(ModuleManager.BaseModule): def get_playlist_page(self, playlist_id): return utils.http.request(URL_YOUTUBEPLAYLIST, get_params={ "part": "contentDetails,snippet", "id": playlist_id, - "key": self.bot.config["google-api-key"]}, json=True) + "key": self.bot.config["google-api-key"]}).json() def playlist_details(self, playlist_id): page = self.get_playlist_page(playlist_id) if page["items"]: diff --git a/src/utils/http.py b/src/utils/http.py index 80bf5eae..99cce10a 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -67,12 +67,10 @@ class Request(object): cookies: typing.Dict[str, str] = dataclasses.field( default_factory=dict) - json: bool = False json_body: bool = False allow_redirects: bool = True check_content_type: bool = True - detect_encoding: bool = True fallback_encoding: typing.Optional[str] = None content_type: typing.Optional[str] = None proxy: typing.Optional[str] = None @@ -116,7 +114,7 @@ class Request(object): return None class Response(object): - def __init__(self, code: int, data: typing.Any, encoding: str, + def __init__(self, code: int, data: bytes, encoding: str, headers: typing.Dict[str, str], cookies: typing.Dict[str, str]): self.code = code self.data = data @@ -124,8 +122,8 @@ class Response(object): self.encoding = encoding self.headers = headers self.cookies = cookies - def decode(self) -> str: - return self.data + def decode(self, encoding="utf8") -> str: + return self.data.decode(encoding) def json(self) -> typing.Any: return _json.loads(self.data) def soup(self, parser: str="lxml") -> bs4.BeautifulSoup: @@ -199,28 +197,12 @@ def _request(request_obj: Request) -> Response: else: encoding = "iso-8859-1" - if (request_obj.detect_encoding and - response.content_type and + if (response.content_type and response.content_type in SOUP_CONTENT_TYPES): - souped = bs4.BeautifulSoup(response.data, "lxml") - encoding = _find_encoding(souped) or encoding + encoding = _find_encoding(response.data) or encoding + response.encoding = encoding - def _decode_data(): - return response.data.decode(encoding) - - if request_obj.json and response.data: - data = _decode_data() - try: - response.data = _json.loads(data) - return response - except _json.decoder.JSONDecodeError as e: - raise HTTPParsingException(str(e), data) - - if response.content_type in DECODE_CONTENT_TYPES: - response.data = _decode_data() - return response - else: - return response + return response class RequestManyException(Exception): pass