diff --git a/modules/define.py b/modules/define.py index 144ec227..4e83c65c 100644 --- a/modules/define.py +++ b/modules/define.py @@ -15,12 +15,12 @@ class Module(ModuleManager.BaseModule): def _get_definition(self, word): page = utils.http.request(URL_WORDNIK % word, get_params={ "useCanonical": "true", "limit": 1, - "sourceDictionaries": "wiktionary", "api_key": self.bot.config[ - "wordnik-api-key"]}, json=True) + "sourceDictionaries": "wiktionary", + "api_key": self.bot.config["wordnik-api-key"]}) if page: if page.code == 200: - return True, page.data[0] + return True, page.json()[0] else: return True, None else: @@ -58,16 +58,16 @@ class Module(ModuleManager.BaseModule): self._last_called = time.time() page = utils.http.request(URL_WORDNIK_RANDOM, get_params={ - "api_key":self.bot.config["wordnik-api-key"], - "min_dictionary_count":1},json=True) - if page and len(page.data): - success, definition = self._get_definition(page.data["word"]) + "api_key": self.bot.config["wordnik-api-key"], + "min_dictionary_count": 1}).json() + if page: + success, definition = self._get_definition(page["word"]) if not success: raise utils.EventError("Try again in a couple of seconds") text = utils.http.strip_html(definition["text"]) event["stdout"].write("Random Word: %s - Definition: %s" % ( - page.data["word"], text)) + page["word"], text)) else: raise utils.EventResultsError() else: diff --git a/modules/fediverse/ap_actor.py b/modules/fediverse/ap_actor.py index a2557e9b..f87069a5 100644 --- a/modules/fediverse/ap_actor.py +++ b/modules/fediverse/ap_actor.py @@ -14,10 +14,11 @@ class Actor(object): def load(self): response = ap_utils.activity_request(self.url) if response.code == 200: - self.username = response.data["preferredUsername"] - self.inbox = Inbox(response.data["inbox"]) - self.outbox = Outbox(response.data["outbox"]) - self.followers = response.data["followers"] + response = response.json() + self.username = response["preferredUsername"] + self.inbox = Inbox(response["inbox"]) + self.outbox = Outbox(response["outbox"]) + self.followers = response["followers"] return True return False @@ -26,19 +27,19 @@ class Outbox(object): self._url = url def load(self): - outbox = ap_utils.activity_request(self._url) + outbox = ap_utils.activity_request(self._url).json() items = None - if "first" in outbox.data: - if type(outbox.data["first"]) == dict: + if "first" in outbox: + if type(outbox["first"]) == dict: # pleroma - items = outbox.data["first"]["orderedItems"] + items = outbox["first"]["orderedItems"] else: # mastodon - first = ap_utils.activity_request(outbox.data["first"]) - items = first.data["orderedItems"] + first = ap_utils.activity_request(outbox["first"]).json() + items = first["orderedItems"] else: - items = outbox.data["orderedItems"] + items = outbox["orderedItems"] return items class Inbox(object): @@ -58,5 +59,5 @@ class Inbox(object): headers.append(["signature", signature]) return ap_utils.activity_request(self._url, activity.format(sender), - method="POST", headers=dict(headers)).data + method="POST", headers=dict(headers)).json() diff --git a/modules/fediverse/ap_utils.py b/modules/fediverse/ap_utils.py index bedc2f1b..fc6ea5fd 100644 --- a/modules/fediverse/ap_utils.py +++ b/modules/fediverse/ap_utils.py @@ -26,7 +26,7 @@ def activity_request(url, data=None, method="GET", type=ACTIVITY_TYPE, headers = {"Accept": type} request = utils.http.Request(url, headers=headers, - content_type=content_type, post_data=data, method=method, json=True, + content_type=content_type, post_data=data, method=method, json_body=True, fallback_encoding="utf8") return utils.http.request(request) @@ -64,7 +64,7 @@ def find_actor(username, instance): actor_url = None if webfinger.code == 200: - for link in webfinger.data["links"]: + for link in webfinger.json()["links"]: if link["type"] == ACTIVITY_TYPE: return link["href"] else: @@ -128,15 +128,15 @@ def format_note(actor, note, type="Create"): if type == "Announce": retoot_url = note retoot_instance = urllib.parse.urlparse(retoot_url).hostname - retoot = activity_request(retoot_url) - retoot_url = retoot.data.get("url", retoot.data["id"]) + retoot = activity_request(retoot_url).json() + retoot_url = retoot.get("url", retoot["id"]) - original_tooter = ap_actor.Actor(retoot.data["attributedTo"]) + original_tooter = ap_actor.Actor(retoot["attributedTo"]) original_tooter.load() retooted_user = "@%s@%s" % (original_tooter.username, retoot_instance) - retoot_content = _content(retoot.data) + retoot_content = _content(retoot) - return (retoot.data.get("summary", None), "%s (boost %s): %s" % ( + return (retoot.get("summary", None), "%s (boost %s): %s" % ( actor.username, retooted_user, retoot_content), retoot_url) elif type == "Create": diff --git a/modules/imgur.py b/modules/imgur.py index 30208c36..35bb6329 100644 --- a/modules/imgur.py +++ b/modules/imgur.py @@ -48,11 +48,10 @@ class Module(ModuleManager.BaseModule): def _parse_gallery(self, hash): api_key = self.bot.config["imgur-api-key"] result = utils.http.request(URL_GALLERY % hash, - headers={"Authorization": "Client-ID %s" % api_key}, - json=True) + headers={"Authorization": "Client-ID %s" % api_key}).json() - if result and result.data["success"]: - data = result.data["data"] + if result and result["success"]: + data = result["data"] text = "" nsfw = utils.irc.bold(NSFW_TEXT) + " " if data["nsfw"] else "" @@ -75,11 +74,10 @@ class Module(ModuleManager.BaseModule): def _parse_image(self, hash): api_key = self.bot.config["imgur-api-key"] result = utils.http.request(URL_IMAGE % hash, - headers={"Authorization": "Client-ID %s" % api_key}, - json=True) + headers={"Authorization": "Client-ID %s" % api_key}).json() - if result and result.data["success"]: - data = result.data["data"] + if result and result["success"]: + data = result["data"] text = "" nsfw = utils.irc.bold(NSFW_TEXT) + " " if data["nsfw"] else "" @@ -102,11 +100,10 @@ class Module(ModuleManager.BaseModule): def _image_info(self, hash): api_key = self.bot.config["imgur-api-key"] result = utils.http.request(URL_IMAGE % hash, - headers={"Authorization": "Client-ID %s" % api_key}, - json=True) + headers={"Authorization": "Client-ID %s" % api_key}).json() - if result and result.data["success"]: - data = result.data["data"] + if result and result["success"]: + data = result["data"] text = self._prefix(data) text += "(%s %dx%d, %d views)" % (data["type"], data["width"], @@ -120,11 +117,10 @@ class Module(ModuleManager.BaseModule): def _gallery_info(self, hash): api_key = self.bot.config["imgur-api-key"] result = utils.http.request(URL_GALLERY % hash, - headers={"Authorization": "Client-ID %s" % api_key}, - json=True) + headers={"Authorization": "Client-ID %s" % api_key}).json() - if result and result.data["success"]: - data = result.data["data"] + if result and result["success"]: + data = result["data"] text = self._prefix(data) text += "(%d views, %d▲▼%d)" % (data["views"], data["ups"], data["downs"]) diff --git a/modules/lastfm.py b/modules/lastfm.py index 8bf7ebef..ce69abcf 100644 --- a/modules/lastfm.py +++ b/modules/lastfm.py @@ -40,11 +40,10 @@ class Module(ModuleManager.BaseModule): page = utils.http.request(URL_SCROBBLER, get_params={ "method": "user.getrecenttracks", "user": lastfm_username, "api_key": self.bot.config["lastfm-api-key"], - "format": "json", "limit": "1"}, json=True) + "format": "json", "limit": "1"}).json() if page: - if ("recenttracks" in page.data and - len(page.data["recenttracks"]["track"])): - now_playing = page.data["recenttracks"]["track"] + if "recenttracks" in page and len(page["recenttracks"]["track"]): + now_playing = page["recenttracks"]["track"] if type(now_playing) == list: now_playing = now_playing[0] @@ -70,9 +69,9 @@ class Module(ModuleManager.BaseModule): "method": "track.getInfo", "artist": artist, "track": track_name, "autocorrect": "1", "api_key": self.bot.config["lastfm-api-key"], - "user": lastfm_username, "format": "json"}, json=True) + "user": lastfm_username, "format": "json"}).json() - track = info_page.data.get("track", {}) + track = info_page.get("track", {}) tags_str = "" if "toptags" in track and track["toptags"]["tag"]: diff --git a/modules/rust.py b/modules/rust.py index 2e74d945..2b4a2061 100644 --- a/modules/rust.py +++ b/modules/rust.py @@ -35,14 +35,14 @@ class Module(ModuleManager.BaseModule): args["code"] = FN_TEMPLATE % event["args"] try: page = utils.http.request(EVAL_URL, post_data=args, - method="POST", json=True, content_type="application/json") + method="POST", content_type="application/json").json() except socket.timeout: raise utils.EventError("%s: eval timed out" % event["user"].nickname) - err_or_out = "stdout" if page.data["success"] else "stderr" + err_or_out = "stdout" if page["success"] else "stderr" event[err_or_out].write("%s: %s" % (event["user"].nickname, - page.data[err_or_out].strip("\n"))) + page[err_or_out].strip("\n"))) @utils.hook("received.command.crate") @utils.kwarg("min_args", 1) @@ -50,10 +50,10 @@ class Module(ModuleManager.BaseModule): @utils.kwarg("usage", "") def crate(self, event): query = event["args_split"][0] - request = utils.http.Request(API_CRATE % query, json=True) + request = utils.http.Request(API_CRATE % query) response = utils.http.request(request) if response.code == 200: - crate = response.data["crate"] + crate = response.json()["crate"] name = crate["id"] url = URL_CRATE % name event["stdout"].write("%s %s: %s - %s" % ( diff --git a/modules/youtube.py b/modules/youtube.py index 56866681..24a662bf 100644 --- a/modules/youtube.py +++ b/modules/youtube.py @@ -30,8 +30,7 @@ class Module(ModuleManager.BaseModule): def get_video_page(self, video_id): return utils.http.request(URL_YOUTUBEVIDEO, get_params={ "part": "contentDetails,snippet,statistics", - "id": video_id, "key": self.bot.config["google-api-key"]}, - json=True) + "id": video_id, "key": self.bot.config["google-api-key"]}).json() def _number(self, n): if n: @@ -39,8 +38,8 @@ class Module(ModuleManager.BaseModule): def video_details(self, video_id): page = self.get_video_page(video_id) - if page.data["items"]: - item = page.data["items"][0] + if page["items"]: + item = page["items"][0] snippet = item["snippet"] statistics = item["statistics"] content = item["contentDetails"] @@ -91,8 +90,8 @@ class Module(ModuleManager.BaseModule): "key": self.bot.config["google-api-key"]}, json=True) def playlist_details(self, playlist_id): page = self.get_playlist_page(playlist_id) - if page.data["items"]: - item = page.data["items"][0] + if page["items"]: + item = page["items"][0] snippet = item["snippet"] content = item["contentDetails"] @@ -121,12 +120,11 @@ class Module(ModuleManager.BaseModule): search_page = utils.http.request(URL_YOUTUBESEARCH, get_params={"q": query, "part": "snippet", "maxResults": "1", "type": "video", - "key": self.bot.config["google-api-key"]}, - json=True) + "key": self.bot.config["google-api-key"]}).json() if search_page: - if search_page.data["pageInfo"]["totalResults"] > 0: - video_id = search_page.data["items"][0]["id"]["videoId"] + if search_page["pageInfo"]["totalResults"] > 0: + video_id = search_page["items"][0]["id"]["videoId"] return "https://youtu.be/%s" % video_id @utils.hook("received.command.yt", alias_of="youtube") @@ -157,11 +155,10 @@ class Module(ModuleManager.BaseModule): search_page = utils.http.request(URL_YOUTUBESEARCH, get_params={"q": search, "part": "snippet", "maxResults": "1", "type": "video", "key": self.bot.config["google-api-key"], - "safeSearch": safe}, json=True) + "safeSearch": safe}).json() if search_page: - if search_page.data["pageInfo"]["totalResults"] > 0: - url = URL_VIDEO % search_page.data[ - "items"][0]["id"]["videoId"] + if search_page["pageInfo"]["totalResults"] > 0: + url = URL_VIDEO % search_page["items"][0]["id"]["videoId"] else: raise utils.EventError("No videos found") else: diff --git a/src/utils/http.py b/src/utils/http.py index 7da44c87..9e9bf3e6 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -68,7 +68,7 @@ class Request(object): default_factory=dict) json: bool = False - json_body: typing.Any = None + json_body: bool = False allow_redirects: bool = True check_content_type: bool = True @@ -118,13 +118,16 @@ class Request(object): return None class Response(object): - def __init__(self, code: int, data: typing.Any, - headers: typing.Dict[str, str], encoding: str): + def __init__(self, code: int, data: typing.Any, encoding: str, + headers: typing.Dict[str, str], cookies: typing.Dict[str, str]): self.code = code self.data = data - self.headers = headers self.content_type = headers.get("Content-Type", "").split(";", 1)[0] self.encoding = encoding + self.headers = headers + self.cookies = cookies + def json(self): + return _json.loads(self.data) def _meta_content(s: str) -> typing.Dict[str, str]: out = {} @@ -167,7 +170,8 @@ def _request(request_obj: Request) -> Response: params=request_obj.get_params, data=request_obj.get_body(), allow_redirects=request_obj.allow_redirects, - stream=True + stream=True, + cookies=request_obj.cookies ) response_content = response.raw.read(RESPONSE_MAX, decode_content=True) @@ -176,7 +180,8 @@ def _request(request_obj: Request) -> Response: headers = utils.CaseInsensitiveDict(dict(response.headers)) our_response = Response(response.status_code, response_content, - headers=headers, encoding=response.encoding) + encoding=response.encoding, headers=headers, + cookies=response.cookies.get_dict()) return our_response try: @@ -255,7 +260,8 @@ def request_many(requests: typing.List[Request]) -> typing.Dict[str, Response]: headers = utils.CaseInsensitiveDict(dict(response.headers)) data = response.body.decode("utf8") - responses[request.id] = Response(response.code, data, headers, "utf8") + responses[request.id] = Response(response.code, data, "utf8", headers, + {}) loop = asyncio.new_event_loop() awaits = []