add cookies and .json() to utils.http.Response objects

This commit is contained in:
jesopo 2019-11-25 18:17:30 +00:00
parent 8626a29a69
commit 6a6e789ec9
8 changed files with 74 additions and 75 deletions

View file

@ -15,12 +15,12 @@ class Module(ModuleManager.BaseModule):
def _get_definition(self, word):
page = utils.http.request(URL_WORDNIK % word, get_params={
"useCanonical": "true", "limit": 1,
"sourceDictionaries": "wiktionary", "api_key": self.bot.config[
"wordnik-api-key"]}, json=True)
"sourceDictionaries": "wiktionary",
"api_key": self.bot.config["wordnik-api-key"]})
if page:
if page.code == 200:
return True, page.data[0]
return True, page.json()[0]
else:
return True, None
else:
@ -59,15 +59,15 @@ class Module(ModuleManager.BaseModule):
page = utils.http.request(URL_WORDNIK_RANDOM, get_params={
"api_key": self.bot.config["wordnik-api-key"],
"min_dictionary_count":1},json=True)
if page and len(page.data):
success, definition = self._get_definition(page.data["word"])
"min_dictionary_count": 1}).json()
if page:
success, definition = self._get_definition(page["word"])
if not success:
raise utils.EventError("Try again in a couple of seconds")
text = utils.http.strip_html(definition["text"])
event["stdout"].write("Random Word: %s - Definition: %s" % (
page.data["word"], text))
page["word"], text))
else:
raise utils.EventResultsError()
else:

View file

@ -14,10 +14,11 @@ class Actor(object):
def load(self):
response = ap_utils.activity_request(self.url)
if response.code == 200:
self.username = response.data["preferredUsername"]
self.inbox = Inbox(response.data["inbox"])
self.outbox = Outbox(response.data["outbox"])
self.followers = response.data["followers"]
response = response.json()
self.username = response["preferredUsername"]
self.inbox = Inbox(response["inbox"])
self.outbox = Outbox(response["outbox"])
self.followers = response["followers"]
return True
return False
@ -26,19 +27,19 @@ class Outbox(object):
self._url = url
def load(self):
outbox = ap_utils.activity_request(self._url)
outbox = ap_utils.activity_request(self._url).json()
items = None
if "first" in outbox.data:
if type(outbox.data["first"]) == dict:
if "first" in outbox:
if type(outbox["first"]) == dict:
# pleroma
items = outbox.data["first"]["orderedItems"]
items = outbox["first"]["orderedItems"]
else:
# mastodon
first = ap_utils.activity_request(outbox.data["first"])
items = first.data["orderedItems"]
first = ap_utils.activity_request(outbox["first"]).json()
items = first["orderedItems"]
else:
items = outbox.data["orderedItems"]
items = outbox["orderedItems"]
return items
class Inbox(object):
@ -58,5 +59,5 @@ class Inbox(object):
headers.append(["signature", signature])
return ap_utils.activity_request(self._url, activity.format(sender),
method="POST", headers=dict(headers)).data
method="POST", headers=dict(headers)).json()

View file

@ -26,7 +26,7 @@ def activity_request(url, data=None, method="GET", type=ACTIVITY_TYPE,
headers = {"Accept": type}
request = utils.http.Request(url, headers=headers,
content_type=content_type, post_data=data, method=method, json=True,
content_type=content_type, post_data=data, method=method,
json_body=True, fallback_encoding="utf8")
return utils.http.request(request)
@ -64,7 +64,7 @@ def find_actor(username, instance):
actor_url = None
if webfinger.code == 200:
for link in webfinger.data["links"]:
for link in webfinger.json()["links"]:
if link["type"] == ACTIVITY_TYPE:
return link["href"]
else:
@ -128,15 +128,15 @@ def format_note(actor, note, type="Create"):
if type == "Announce":
retoot_url = note
retoot_instance = urllib.parse.urlparse(retoot_url).hostname
retoot = activity_request(retoot_url)
retoot_url = retoot.data.get("url", retoot.data["id"])
retoot = activity_request(retoot_url).json()
retoot_url = retoot.get("url", retoot["id"])
original_tooter = ap_actor.Actor(retoot.data["attributedTo"])
original_tooter = ap_actor.Actor(retoot["attributedTo"])
original_tooter.load()
retooted_user = "@%s@%s" % (original_tooter.username, retoot_instance)
retoot_content = _content(retoot.data)
retoot_content = _content(retoot)
return (retoot.data.get("summary", None), "%s (boost %s): %s" % (
return (retoot.get("summary", None), "%s (boost %s): %s" % (
actor.username, retooted_user, retoot_content), retoot_url)
elif type == "Create":

View file

@ -48,11 +48,10 @@ class Module(ModuleManager.BaseModule):
def _parse_gallery(self, hash):
api_key = self.bot.config["imgur-api-key"]
result = utils.http.request(URL_GALLERY % hash,
headers={"Authorization": "Client-ID %s" % api_key},
json=True)
headers={"Authorization": "Client-ID %s" % api_key}).json()
if result and result.data["success"]:
data = result.data["data"]
if result and result["success"]:
data = result["data"]
text = ""
nsfw = utils.irc.bold(NSFW_TEXT) + " " if data["nsfw"] else ""
@ -75,11 +74,10 @@ class Module(ModuleManager.BaseModule):
def _parse_image(self, hash):
api_key = self.bot.config["imgur-api-key"]
result = utils.http.request(URL_IMAGE % hash,
headers={"Authorization": "Client-ID %s" % api_key},
json=True)
headers={"Authorization": "Client-ID %s" % api_key}).json()
if result and result.data["success"]:
data = result.data["data"]
if result and result["success"]:
data = result["data"]
text = ""
nsfw = utils.irc.bold(NSFW_TEXT) + " " if data["nsfw"] else ""
@ -102,11 +100,10 @@ class Module(ModuleManager.BaseModule):
def _image_info(self, hash):
api_key = self.bot.config["imgur-api-key"]
result = utils.http.request(URL_IMAGE % hash,
headers={"Authorization": "Client-ID %s" % api_key},
json=True)
headers={"Authorization": "Client-ID %s" % api_key}).json()
if result and result.data["success"]:
data = result.data["data"]
if result and result["success"]:
data = result["data"]
text = self._prefix(data)
text += "(%s %dx%d, %d views)" % (data["type"], data["width"],
@ -120,11 +117,10 @@ class Module(ModuleManager.BaseModule):
def _gallery_info(self, hash):
api_key = self.bot.config["imgur-api-key"]
result = utils.http.request(URL_GALLERY % hash,
headers={"Authorization": "Client-ID %s" % api_key},
json=True)
headers={"Authorization": "Client-ID %s" % api_key}).json()
if result and result.data["success"]:
data = result.data["data"]
if result and result["success"]:
data = result["data"]
text = self._prefix(data)
text += "(%d views, %d▲▼%d)" % (data["views"],
data["ups"], data["downs"])

View file

@ -40,11 +40,10 @@ class Module(ModuleManager.BaseModule):
page = utils.http.request(URL_SCROBBLER, get_params={
"method": "user.getrecenttracks", "user": lastfm_username,
"api_key": self.bot.config["lastfm-api-key"],
"format": "json", "limit": "1"}, json=True)
"format": "json", "limit": "1"}).json()
if page:
if ("recenttracks" in page.data and
len(page.data["recenttracks"]["track"])):
now_playing = page.data["recenttracks"]["track"]
if "recenttracks" in page and len(page["recenttracks"]["track"]):
now_playing = page["recenttracks"]["track"]
if type(now_playing) == list:
now_playing = now_playing[0]
@ -70,9 +69,9 @@ class Module(ModuleManager.BaseModule):
"method": "track.getInfo", "artist": artist,
"track": track_name, "autocorrect": "1",
"api_key": self.bot.config["lastfm-api-key"],
"user": lastfm_username, "format": "json"}, json=True)
"user": lastfm_username, "format": "json"}).json()
track = info_page.data.get("track", {})
track = info_page.get("track", {})
tags_str = ""
if "toptags" in track and track["toptags"]["tag"]:

View file

@ -35,14 +35,14 @@ class Module(ModuleManager.BaseModule):
args["code"] = FN_TEMPLATE % event["args"]
try:
page = utils.http.request(EVAL_URL, post_data=args,
method="POST", json=True, content_type="application/json")
method="POST", content_type="application/json").json()
except socket.timeout:
raise utils.EventError("%s: eval timed out" %
event["user"].nickname)
err_or_out = "stdout" if page.data["success"] else "stderr"
err_or_out = "stdout" if page["success"] else "stderr"
event[err_or_out].write("%s: %s" % (event["user"].nickname,
page.data[err_or_out].strip("\n")))
page[err_or_out].strip("\n")))
@utils.hook("received.command.crate")
@utils.kwarg("min_args", 1)
@ -50,10 +50,10 @@ class Module(ModuleManager.BaseModule):
@utils.kwarg("usage", "<crate-name>")
def crate(self, event):
query = event["args_split"][0]
request = utils.http.Request(API_CRATE % query, json=True)
request = utils.http.Request(API_CRATE % query)
response = utils.http.request(request)
if response.code == 200:
crate = response.data["crate"]
crate = response.json()["crate"]
name = crate["id"]
url = URL_CRATE % name
event["stdout"].write("%s %s: %s - %s" % (

View file

@ -30,8 +30,7 @@ class Module(ModuleManager.BaseModule):
def get_video_page(self, video_id):
return utils.http.request(URL_YOUTUBEVIDEO, get_params={
"part": "contentDetails,snippet,statistics",
"id": video_id, "key": self.bot.config["google-api-key"]},
json=True)
"id": video_id, "key": self.bot.config["google-api-key"]}).json()
def _number(self, n):
if n:
@ -39,8 +38,8 @@ class Module(ModuleManager.BaseModule):
def video_details(self, video_id):
page = self.get_video_page(video_id)
if page.data["items"]:
item = page.data["items"][0]
if page["items"]:
item = page["items"][0]
snippet = item["snippet"]
statistics = item["statistics"]
content = item["contentDetails"]
@ -91,8 +90,8 @@ class Module(ModuleManager.BaseModule):
"key": self.bot.config["google-api-key"]}, json=True)
def playlist_details(self, playlist_id):
page = self.get_playlist_page(playlist_id)
if page.data["items"]:
item = page.data["items"][0]
if page["items"]:
item = page["items"][0]
snippet = item["snippet"]
content = item["contentDetails"]
@ -121,12 +120,11 @@ class Module(ModuleManager.BaseModule):
search_page = utils.http.request(URL_YOUTUBESEARCH,
get_params={"q": query, "part": "snippet",
"maxResults": "1", "type": "video",
"key": self.bot.config["google-api-key"]},
json=True)
"key": self.bot.config["google-api-key"]}).json()
if search_page:
if search_page.data["pageInfo"]["totalResults"] > 0:
video_id = search_page.data["items"][0]["id"]["videoId"]
if search_page["pageInfo"]["totalResults"] > 0:
video_id = search_page["items"][0]["id"]["videoId"]
return "https://youtu.be/%s" % video_id
@utils.hook("received.command.yt", alias_of="youtube")
@ -157,11 +155,10 @@ class Module(ModuleManager.BaseModule):
search_page = utils.http.request(URL_YOUTUBESEARCH,
get_params={"q": search, "part": "snippet", "maxResults": "1",
"type": "video", "key": self.bot.config["google-api-key"],
"safeSearch": safe}, json=True)
"safeSearch": safe}).json()
if search_page:
if search_page.data["pageInfo"]["totalResults"] > 0:
url = URL_VIDEO % search_page.data[
"items"][0]["id"]["videoId"]
if search_page["pageInfo"]["totalResults"] > 0:
url = URL_VIDEO % search_page["items"][0]["id"]["videoId"]
else:
raise utils.EventError("No videos found")
else:

View file

@ -68,7 +68,7 @@ class Request(object):
default_factory=dict)
json: bool = False
json_body: typing.Any = None
json_body: bool = False
allow_redirects: bool = True
check_content_type: bool = True
@ -118,13 +118,16 @@ class Request(object):
return None
class Response(object):
def __init__(self, code: int, data: typing.Any,
headers: typing.Dict[str, str], encoding: str):
def __init__(self, code: int, data: typing.Any, encoding: str,
headers: typing.Dict[str, str], cookies: typing.Dict[str, str]):
self.code = code
self.data = data
self.headers = headers
self.content_type = headers.get("Content-Type", "").split(";", 1)[0]
self.encoding = encoding
self.headers = headers
self.cookies = cookies
def json(self):
return _json.loads(self.data)
def _meta_content(s: str) -> typing.Dict[str, str]:
out = {}
@ -167,7 +170,8 @@ def _request(request_obj: Request) -> Response:
params=request_obj.get_params,
data=request_obj.get_body(),
allow_redirects=request_obj.allow_redirects,
stream=True
stream=True,
cookies=request_obj.cookies
)
response_content = response.raw.read(RESPONSE_MAX,
decode_content=True)
@ -176,7 +180,8 @@ def _request(request_obj: Request) -> Response:
headers = utils.CaseInsensitiveDict(dict(response.headers))
our_response = Response(response.status_code, response_content,
headers=headers, encoding=response.encoding)
encoding=response.encoding, headers=headers,
cookies=response.cookies.get_dict())
return our_response
try:
@ -255,7 +260,8 @@ def request_many(requests: typing.List[Request]) -> typing.Dict[str, Response]:
headers = utils.CaseInsensitiveDict(dict(response.headers))
data = response.body.decode("utf8")
responses[request.id] = Response(response.code, data, headers, "utf8")
responses[request.id] = Response(response.code, data, "utf8", headers,
{})
loop = asyncio.new_event_loop()
awaits = []