Firepup650/bitbot-3.11-fork
2
0
Fork 0
Code Issues Pull requests Activity Actions

utils.http.Response.data should always be bytes - add .decode and .soup

Browse source
This commit is contained in:
jesopo 2019-11-26 13:41:40 +00:00
parent 9eb51740b3
commit 2d21dfa229
23 changed files with 127 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
modules/bitcoin.py
View file

@ -12,16 +12,15 @@ class Module(ModuleManager.BaseModule):
:usage: [currency] :usage: [currency]
""" """
currency = (event["args"] or "USD").upper() currency = (event["args"] or "USD").upper()
page = utils.http.request("https://blockchain.info/ticker", page = utils.http.request("https://blockchain.info/ticker").json()
json=True)
if page: if page:
if currency in page.data: if currency in page:
conversion = page.data[currency] conversion = page[currency]
buy, sell = conversion["buy"], conversion["sell"] buy, sell = conversion["buy"], conversion["sell"]
event["stdout"].write("1 BTC = %.2f %s (buy) %.2f %s " event["stdout"].write("1 BTC = %.2f %s (buy) %.2f %s "
"(sell)" % (buy, currency, sell, currency)) "(sell)" % (buy, currency, sell, currency))
else: else:
event["stderr"].write("Unknown currency, available " event["stderr"].write("Unknown currency, available "
"currencies: %s" % ", ".join(page.data.keys())) "currencies: %s" % ", ".join(page.keys()))
else: else:
raise utils.EventResultsError() raise utils.EventResultsError()

6
modules/duckduckgo.py
View file

@ -18,9 +18,9 @@ class Module(ModuleManager.BaseModule):
if phrase: if phrase:
page = utils.http.request(URL_DDG, get_params={ page = utils.http.request(URL_DDG, get_params={
"q": phrase, "format": "json", "no_html": "1", "q": phrase, "format": "json", "no_html": "1",
"no_redirect": "1"}, json=True) "no_redirect": "1"}).json()
if page and page.data["AbstractURL"]: if page and page["AbstractURL"]:
event["stdout"].write(page.data["AbstractURL"]) event["stdout"].write(page["AbstractURL"])
else: else:
event["stderr"].write("No results found") event["stderr"].write("No results found")

2
modules/eval_python.py
View file

@ -21,6 +21,6 @@ class Module(ModuleManager.BaseModule):
if page and page.data: if page and page.data:
event["stdout"].write("%s: %s" % (event["user"].nickname, event["stdout"].write("%s: %s" % (event["user"].nickname,
page.data.rstrip("\n"))) page.decode().rstrip("\n")))
else: else:
event["stderr"].write("%s: failed to eval" % event["user"].nickname) event["stderr"].write("%s: failed to eval" % event["user"].nickname)

6
modules/git_webhooks/github.py
View file

@ -171,7 +171,7 @@ class GitHub(object):
self.log.debug("git.io shortening: %s" % url) self.log.debug("git.io shortening: %s" % url)
try: try:
page = utils.http.request("https://git.io", method="POST", page = utils.http.request("https://git.io", method="POST",
post_data={"url": url}, detect_encoding=False) post_data={"url": url})
return page.headers["Location"] return page.headers["Location"]
except utils.http.HTTPTimeoutException: except utils.http.HTTPTimeoutException:
self.log.warn( self.log.warn(
@ -294,11 +294,11 @@ class GitHub(object):
action_desc = "committed to %s" % number action_desc = "committed to %s" % number
commits_url = data["pull_request"]["commits_url"] commits_url = data["pull_request"]["commits_url"]
commits = utils.http.request(commits_url, json=True) commits = utils.http.request(commits_url).json()
if commits: if commits:
seen_before = False seen_before = False
new_commits = [] new_commits = []
for commit in commits.data: for commit in commits:
if seen_before: if seen_before:
new_commits.append({"id": commit["sha"], new_commits.append({"id": commit["sha"],
"message": commit["commit"]["message"]}) "message": commit["commit"]["message"]})

43
modules/github.py
View file

@ -70,17 +70,18 @@ class Module(ModuleManager.BaseModule):
headers = {} headers = {}
if not oauth2_token == None: if not oauth2_token == None:
headers["Authorization"] = "token %s" % oauth2_token headers["Authorization"] = "token %s" % oauth2_token
request = utils.http.Request(url, headers=headers, json=True) request = utils.http.Request(url, headers=headers)
return utils.http.request(request) return utils.http.request(request)
def _commit(self, username, repository, commit): def _commit(self, username, repository, commit):
page = self._get(API_COMMIT_URL % (username, repository, commit)) page = self._get(API_COMMIT_URL % (username, repository, commit))
if page and page.code == 200: if page and page.code == 200:
page = page.json()
repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO) repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO)
sha = utils.irc.color(page.data["sha"][:8], COLOR_ID) sha = utils.irc.color(page["sha"][:8], COLOR_ID)
return "(%s@%s) %s - %s %s" % (repo, sha, return "(%s@%s) %s - %s %s" % (repo, sha,
page.data["author"]["login"], page.data["commit"]["message"], page["author"]["login"], page["commit"]["message"],
self._short_url(page.data["html_url"])) self._short_url(page["html_url"]))
def _parse_commit(self, target, ref): def _parse_commit(self, target, ref):
username, repository, commit = self._parse_ref(target, ref, "@") username, repository, commit = self._parse_ref(target, ref, "@")
return self._commit(username, repository, commit) return self._commit(username, repository, commit)
@ -116,21 +117,21 @@ class Module(ModuleManager.BaseModule):
def _parse_issue(self, page, username, repository, number): def _parse_issue(self, page, username, repository, number):
repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO) repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO)
number = utils.irc.color("#%s" % number, COLOR_ID) number = utils.irc.color("#%s" % number, COLOR_ID)
labels = [label["name"] for label in page.data["labels"]] labels = [label["name"] for label in page["labels"]]
labels_str = "" labels_str = ""
if labels: if labels:
labels_str = "[%s] " % ", ".join(labels) labels_str = "[%s] " % ", ".join(labels)
url = self._short_url(page.data["html_url"]) url = self._short_url(page["html_url"])
state = page.data["state"] state = page["state"]
if state == "open": if state == "open":
state = utils.irc.color("open", COLOR_NEUTRAL) state = utils.irc.color("open", COLOR_NEUTRAL)
elif state == "closed": elif state == "closed":
state = utils.irc.color("closed", COLOR_NEGATIVE) state = utils.irc.color("closed", COLOR_NEGATIVE)
return "(%s issue%s, %s) %s %s%s" % ( return "(%s issue%s, %s) %s %s%s" % (
repo, number, state, page.data["title"], labels_str, url) repo, number, state, page["title"], labels_str, url)
def _get_issue(self, username, repository, number): def _get_issue(self, username, repository, number):
return self._get(API_ISSUE_URL % (username, repository, number)) return self._get(API_ISSUE_URL % (username, repository, number))
@ -147,21 +148,21 @@ class Module(ModuleManager.BaseModule):
page = self._get_issue(username, repository, number) page = self._get_issue(username, repository, number)
if page and page.code == 200: if page and page.code == 200:
self._parse_issue(page, username, repository, number) self._parse_issue(page.json(), username, repository, number)
else: else:
event["stderr"].write("Could not find issue") event["stderr"].write("Could not find issue")
def _parse_pull(self, page, username, repository, number): def _parse_pull(self, page, username, repository, number):
repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO) repo = utils.irc.color("%s/%s" % (username, repository), COLOR_REPO)
number = utils.irc.color("#%s" % number, COLOR_ID) number = utils.irc.color("#%s" % number, COLOR_ID)
branch_from = page.data["head"]["label"] branch_from = page["head"]["label"]
branch_to = page.data["base"]["label"] branch_to = page["base"]["label"]
added = self._added(page.data["additions"]) added = self._added(page["additions"])
removed = self._removed(page.data["deletions"]) removed = self._removed(page["deletions"])
url = self._short_url(page.data["html_url"]) url = self._short_url(page["html_url"])
state = page.data["state"] state = page["state"]
if page.data["merged"]: if page["merged"]:
state = utils.irc.color("merged", COLOR_POSITIVE) state = utils.irc.color("merged", COLOR_POSITIVE)
elif state == "open": elif state == "open":
state = utils.irc.color("open", COLOR_NEUTRAL) state = utils.irc.color("open", COLOR_NEUTRAL)
@ -170,7 +171,7 @@ class Module(ModuleManager.BaseModule):
return "(%s PR%s, %s) %s → %s [%s/%s] %s %s" % ( return "(%s PR%s, %s) %s → %s [%s/%s] %s %s" % (
repo, number, state, branch_from, branch_to, added, removed, repo, number, state, branch_from, branch_to, added, removed,
page.data["title"], url) page["title"], url)
def _get_pull(self, username, repository, number): def _get_pull(self, username, repository, number):
return self._get(API_PULL_URL % (username, repository, number)) return self._get(API_PULL_URL % (username, repository, number))
@utils.hook("received.command.ghpull", min_args=1) @utils.hook("received.command.ghpull", min_args=1)
@ -187,7 +188,7 @@ class Module(ModuleManager.BaseModule):
page = self._get_pull(username, repository, number) page = self._get_pull(username, repository, number)
if page and page.code == 200: if page and page.code == 200:
self._parse_pull(page, username, repository, number) self._parse_pull(page.json(), username, repository, number)
else: else:
event["stderr"].write("Could not find pull request") event["stderr"].write("Could not find pull request")
@ -198,9 +199,11 @@ class Module(ModuleManager.BaseModule):
page = self._get_issue(username, repository, number) page = self._get_issue(username, repository, number)
if page and page.code == 200: if page and page.code == 200:
if "pull_request" in page.data: page = page.json()
if "pull_request" in page:
pull = self._get_pull(username, repository, number) pull = self._get_pull(username, repository, number)
return self._parse_pull(pull, username, repository, number) return self._parse_pull(pull.json(), username, repository,
number)
else: else:
return self._parse_issue(page, username, repository, number) return self._parse_issue(page, username, repository, number)
else: else:

17
modules/google.py
View file

@ -27,13 +27,13 @@ class Module(ModuleManager.BaseModule):
safe = "active" if safe_setting else "off" safe = "active" if safe_setting else "off"
page = utils.http.request(URL_GOOGLESEARCH, get_params={ page = utils.http.request(URL_GOOGLESEARCH, get_params={
"q": phrase, "key": self.bot.config[ "q": phrase, "prettyPrint": "true", "num": 1, "gl": "gb",
"google-api-key"], "cx": self.bot.config[ "key": self.bot.config["google-api-key"],
"google-search-id"], "prettyPrint": "true", "cx": self.bot.config["google-search-id"],
"num": 1, "gl": "gb", "safe": safe}, json=True) "safe": safe}).json()
if page: if page:
if "items" in page.data and len(page.data["items"]): if "items" in page and len(page["items"]):
item = page.data["items"][0] item = page["items"][0]
link = item["link"] link = item["link"]
title = utils.parse.line_normalise(item["title"]) title = utils.parse.line_normalise(item["title"])
event["stdout"].write( event["stdout"].write(
@ -54,10 +54,11 @@ class Module(ModuleManager.BaseModule):
phrase = event["args"] or event["target"].buffer.get() phrase = event["args"] or event["target"].buffer.get()
if phrase: if phrase:
page = utils.http.request(URL_GOOGLESUGGEST, get_params={ page = utils.http.request(URL_GOOGLESUGGEST, get_params={
"output": "json", "client": "hp", "gl": "gb", "q": phrase}) "output": "json", "client": "hp", "gl": "gb", "q": phrase}
).json()
if page: if page:
# google gives us jsonp, so we need to unwrap it. # google gives us jsonp, so we need to unwrap it.
page = page.data.split("(", 1)[1][:-1] page = page.split("(", 1)[1][:-1]
page = json.loads(page) page = json.loads(page)
suggestions = page[1] suggestions = page[1]
suggestions = [utils.http.strip_html(s[0]) for s in suggestions] suggestions = [utils.http.strip_html(s[0]) for s in suggestions]

12
modules/imdb.py
View file

@ -17,15 +17,13 @@ class Module(ModuleManager.BaseModule):
:usage: <movie/tv title> :usage: <movie/tv title>
""" """
page = utils.http.request(URL_OMDB, get_params={ page = utils.http.request(URL_OMDB, get_params={
"t": event["args"], "apikey": self.bot.config["omdbapi-api-key"],
"apikey": self.bot.config["omdbapi-api-key"]}, "t": event["args"]}).json()
json=True)
if page: if page:
if "Title" in page.data: if "Title" in page:
event["stdout"].write("%s, %s (%s) %s (%s/10.0) %s" % ( event["stdout"].write("%s, %s (%s) %s (%s/10.0) %s" % (
page.data["Title"], page.data["Year"], page.data["Runtime"], page["Title"], page["Year"], page["Runtime"], page["Plot"],
page.data["Plot"], page.data["imdbRating"], page["imdbRating"], URL_IMDBTITLE % page["imdbID"]))
URL_IMDBTITLE % page.data["imdbID"]))
else: else:
event["stderr"].write("Title not found") event["stderr"].write("Title not found")
else: else:

22
modules/ip_addresses.py
View file

@ -85,19 +85,17 @@ class Module(ModuleManager.BaseModule):
:usage: <IP> :usage: <IP>
:prefix: GeoIP :prefix: GeoIP
""" """
page = utils.http.request(URL_GEOIP % event["args_split"][0], page = utils.http.request(URL_GEOIP % event["args_split"][0]).json()
json=True)
if page: if page:
if page.data["status"] == "success": if page["status"] == "success":
data = page.data["query"] data = page["query"]
data += " | Organisation: %s" % page.data["org"] data += " | Organisation: %s" % page["org"]
data += " | City: %s" % page.data["city"] data += " | City: %s" % page["city"]
data += " | Region: %s (%s)" % (page.data["regionName"], data += " | Region: %s (%s)" % (
page.data["countryCode"]) page["regionName"], page["countryCode"])
data += " | ISP: %s" % page.data["isp"] data += " | ISP: %s" % page["isp"]
data += " | Lon/Lat: %s/%s" % (page.data["lon"], data += " | Lon/Lat: %s/%s" % (page["lon"], page["lat"])
page.data["lat"]) data += " | Timezone: %s" % page["timezone"]
data += " | Timezone: %s" % page.data["timezone"]
event["stdout"].write(data) event["stdout"].write(data)
else: else:
event["stderr"].write("No geoip data found") event["stderr"].write("No geoip data found")

9
modules/location.py
View file

@ -14,11 +14,10 @@ class Module(ModuleManager.BaseModule):
self.exports.add("get-location", self._get_location) self.exports.add("get-location", self._get_location)
def _get_location(self, s): def _get_location(self, s):
page = utils.http.request(URL_OPENCAGE, get_params={ page = utils.http.request(URL_OPENCAGE, get_params={"limit": "1",
"q": s, "key": self.bot.config["opencagedata-api-key"], "limit": "1" "q": s, "key": self.bot.config["opencagedata-api-key"]}).json()
}, json=True) if page and page["results"]:
if page and page.data["results"]: result = page["results"][0]
result = page.data["results"][0]
timezone = result["annotations"]["timezone"]["name"] timezone = result["annotations"]["timezone"]["name"]
lat = result["geometry"]["lat"] lat = result["geometry"]["lat"]
lon = result["geometry"]["lng"] lon = result["geometry"]["lng"]

2
modules/markov.py
View file

@ -39,7 +39,7 @@ class Module(ModuleManager.BaseModule):
if page.code == 200: if page.code == 200:
event["stdout"].write("Importing...") event["stdout"].write("Importing...")
self._load_thread = threading.Thread(target=self._load_loop, self._load_thread = threading.Thread(target=self._load_loop,
args=[event["target"].id, page.data]) args=[event["target"].id, page.decode()])
self._load_thread.daemon = True self._load_thread.daemon = True
self._load_thread.start() self._load_thread.start()
else: else:

6
modules/onionoo.py
View file

@ -11,9 +11,9 @@ URL_RELAY_SEARCH_DETAILS = "https://metrics.torproject.org/rs.html#details/"
def _get_relays_details(search): def _get_relays_details(search):
page = utils.http.request( page = utils.http.request(
URL_ONIONOO_DETAILS, get_params={"search": search}, json=True) URL_ONIONOO_DETAILS, get_params={"search": search}).json()
if page and "relays" in page.data: if page and "relays" in page:
return page.data["relays"] return page["relays"]
raise utils.EventResultsError() raise utils.EventResultsError()
def _format_relay_summary_message(relays, search): def _format_relay_summary_message(relays, search):

5
modules/rss.py
View file

@ -67,7 +67,7 @@ class Module(ModuleManager.BaseModule):
# async url get failed # async url get failed
continue continue
feed = feedparser.parse(pages[url].data) feed = feedparser.parse(pages[url].decode())
feed_title = feed["feed"].get("title", None) feed_title = feed["feed"].get("title", None)
max_ids = len(feed["entries"])*10 max_ids = len(feed["entries"])*10
@ -105,8 +105,7 @@ class Module(ModuleManager.BaseModule):
def _get_entries(self, url, max: int=None): def _get_entries(self, url, max: int=None):
try: try:
data = utils.http.request(url) feed = feedparser.parse(utils.http.request(url).data)
feed = feedparser.parse(data.data)
except Exception as e: except Exception as e:
self.log.warn("failed to parse RSS %s", [url], exc_info=True) self.log.warn("failed to parse RSS %s", [url], exc_info=True)
feed = None feed = None

6
modules/shorturl.py
View file

@ -57,10 +57,10 @@ class Module(ModuleManager.BaseModule):
access_token = self.bot.config.get("bitly-api-key", None) access_token = self.bot.config.get("bitly-api-key", None)
if access_token: if access_token:
page = utils.http.request(URL_BITLYSHORTEN, get_params={ page = utils.http.request(URL_BITLYSHORTEN, get_params={
"access_token": access_token, "longUrl": url}, json=True) "access_token": access_token, "longUrl": url}).json()
if page and page.data["data"]: if page["data"]:
return page.data["data"]["url"] return page["data"]["url"]
return None return None
def _find_url(self, target, args): def _find_url(self, target, args):

6
modules/soundcloud.py
View file

@ -45,11 +45,11 @@ class Module(ModuleManager.BaseModule):
page = utils.http.request( page = utils.http.request(
URL_SOUNDCLOUD_TRACK if has_query else URL_SOUNDCLOUD_RESOLVE, URL_SOUNDCLOUD_TRACK if has_query else URL_SOUNDCLOUD_RESOLVE,
get_params=get_params, json=True) get_params=get_params).json()
if page: if page:
if len(page.data): if len(page):
page = page.data[0] if has_query else page page = page[0] if has_query else page
title = page["title"] title = page["title"]
user = page["user"]["username"] user = page["user"]["username"]
duration = time.strftime("%H:%M:%S", time.gmtime(page[ duration = time.strftime("%H:%M:%S", time.gmtime(page[

14
modules/spotify.py
View file

@ -24,12 +24,11 @@ class Module(ModuleManager.BaseModule):
page = utils.http.request(URL_TOKEN, method="POST", page = utils.http.request(URL_TOKEN, method="POST",
headers={"Authorization": "Basic %s" % bearer}, headers={"Authorization": "Basic %s" % bearer},
post_data={"grant_type": "client_credentials"}, post_data={"grant_type": "client_credentials"}).json()
json=True)
token = page.data["access_token"] token = page["access_token"]
self._token = token self._token = token
self._token_expires = time.time()+page.data["expires_in"] self._token_expires = time.time()+page["expires_in"]
return token return token
@utils.hook("received.command.sp", alias_of="spotify") @utils.hook("received.command.sp", alias_of="spotify")
@ -42,11 +41,10 @@ class Module(ModuleManager.BaseModule):
token = self._get_token() token = self._get_token()
page = utils.http.request(URL_SEARCH, page = utils.http.request(URL_SEARCH,
get_params={"type": "track", "limit": 1, "q": event["args"]}, get_params={"type": "track", "limit": 1, "q": event["args"]},
headers={"Authorization": "Bearer %s" % token}, headers={"Authorization": "Bearer %s" % token}).json()
json=True)
if page: if page:
if len(page.data["tracks"]["items"]): if len(page["tracks"]["items"]):
item = page.data["tracks"]["items"][0] item = page["tracks"]["items"][0]
title = item["name"] title = item["name"]
artist_name = item["artists"][0]["name"] artist_name = item["artists"][0]["name"]
url = item["external_urls"]["spotify"] url = item["external_urls"]["spotify"]

15
modules/thesaurus.py
View file

@ -14,17 +14,18 @@ class Module(ModuleManager.BaseModule):
:usage: <word> [type] :usage: <word> [type]
""" """
phrase = event["args_split"][0] phrase = event["args_split"][0]
page = utils.http.request(URL_THESAURUS % (self.bot.config[ page = utils.http.request(URL_THESAURUS % (
"bighugethesaurus-api-key"], phrase), json=True) self.bot.config["bighugethesaurus-api-key"], phrase))
syn_ant = event["command"][:3] syn_ant = event["command"][:3]
if page: if page:
if page.code == 404: if page.code == 404:
raise utils.EventError("Word not found") raise utils.EventError("Word not found")
page = page.json()
if not len(event["args_split"]) > 1: if not len(event["args_split"]) > 1:
word_types = [] word_types = []
for word_type in page.data.keys(): for word_type in page.keys():
if syn_ant in page.data[word_type]: if syn_ant in page[word_type]:
word_types.append(word_type) word_types.append(word_type)
if word_types: if word_types:
word_types = sorted(word_types) word_types = sorted(word_types)
@ -35,11 +36,11 @@ class Module(ModuleManager.BaseModule):
event["stderr"].write("No categories available") event["stderr"].write("No categories available")
else: else:
category = event["args_split"][1].lower() category = event["args_split"][1].lower()
if category in page.data: if category in page:
if syn_ant in page.data[category]: if syn_ant in page[category]:
event["stdout"].write("%ss for %s: %s" % ( event["stdout"].write("%ss for %s: %s" % (
event["command"].title(), phrase, ", ".join( event["command"].title(), phrase, ", ".join(
page.data[category][syn_ant]))) page[category][syn_ant])))
else: else:
event["stderr"].write("No %ss for %s" % ( event["stderr"].write("No %ss for %s" % (
event["command"], phrase)) event["command"], phrase))

24
modules/trakt.py
View file

@ -25,26 +25,26 @@ class Module(ModuleManager.BaseModule):
page = utils.http.request(URL_TRAKT % username, headers={ page = utils.http.request(URL_TRAKT % username, headers={
"Content-Type": "application/json", "Content-Type": "application/json",
"trakt-api-version": "2", "trakt-api-key": "trakt-api-version": "2", "trakt-api-key":
self.bot.config["trakt-api-key"]}, json=True, self.bot.config["trakt-api-key"]})
code=True)
if page: if page:
if page.code == 200: if page.code == 200:
type = page.data["type"] page = page.json()
type = page["type"]
if type == "movie": if type == "movie":
title = page.data["movie"]["title"] title = page["movie"]["title"]
year = page.data["movie"]["year"] year = page["movie"]["year"]
slug = page.data["movie"]["ids"]["slug"] slug = page["movie"]["ids"]["slug"]
event["stdout"].write( event["stdout"].write(
"%s is now watching %s (%s) %s" % ( "%s is now watching %s (%s) %s" % (
username, title, year, username, title, year,
URL_TRAKTSLUG % ("movie", slug))) URL_TRAKTSLUG % ("movie", slug)))
elif type == "episode": elif type == "episode":
season = page.data["episode"]["season"] season = page["episode"]["season"]
episode_number = page.data["episode"]["number"] episode_number = page["episode"]["number"]
episode_title = page.data["episode"]["title"] episode_title = page["episode"]["title"]
show_title = page.data["show"]["title"] show_title = page["show"]["title"]
show_year = page.data["show"]["year"] show_year = page["show"]["year"]
slug = page.data["show"]["ids"]["slug"] slug = page["show"]["ids"]["slug"]
event["stdout"].write( event["stdout"].write(
"%s is now watching %s s%se%s - %s %s" % ( "%s is now watching %s s%se%s - %s %s" % (
username, show_title, str(season).zfill(2), username, show_title, str(season).zfill(2),

6
modules/translate.py
View file

@ -35,11 +35,11 @@ class Module(ModuleManager.BaseModule):
phrase = phrase.split(" ", 1)[1] phrase = phrase.split(" ", 1)[1]
page = utils.http.request(URL_TRANSLATE, get_params={ page = utils.http.request(URL_TRANSLATE, get_params={
"client": "gtx", "sl": source_language, "client": "gtx", "dt": "t", "q": phrase,
"tl": target_language, "dt": "t", "q": phrase}) "sl": source_language, "tl": target_language})
if page and not page.data.startswith(b"[null,null,"): if page and not page.data.startswith(b"[null,null,"):
data = page.data.decode("utf8") data = page.decode("utf8")
while ",," in data: while ",," in data:
data = data.replace(",,", ",null,") data = data.replace(",,", ",null,")
data = data.replace("[,", "[null,") data = data.replace("[,", "[null,")

8
modules/urbandictionary.py
View file

@ -24,11 +24,11 @@ class Module(ModuleManager.BaseModule):
term = " ".join(term) term = " ".join(term)
page = utils.http.request(URL_URBANDICTIONARY, page = utils.http.request(URL_URBANDICTIONARY,
get_params={"term": term}, json=True) get_params={"term": term}).json()
if page: if page:
if len(page.data["list"]): if len(page["list"]):
if number > 0 and len(page.data["list"]) > number-1: if number > 0 and len(page["list"]) > number-1:
definition = page.data["list"][number-1] definition = page["list"][number-1]
event["stdout"].write("%s: %s" % (definition["word"], event["stdout"].write("%s: %s" % (definition["word"],
definition["definition"].replace("\n", " ").replace( definition["definition"].replace("\n", " ").replace(
"\r", "").replace(" ", " "))) "\r", "").replace(" ", " ")))

20
modules/weather.py
View file

@ -56,24 +56,24 @@ class Module(ModuleManager.BaseModule):
args["lat"] = lat args["lat"] = lat
args["lon"] = lon args["lon"] = lon
page = utils.http.request(URL_WEATHER, get_params=args, json=True) page = utils.http.request(URL_WEATHER, get_params=args).json()
if page: if page:
if "weather" in page.data: if "weather" in page:
if location_name: if location_name:
location_str = location_name location_str = location_name
else: else:
location_parts = [page.data["name"]] location_parts = [page["name"]]
if "country" in page.data["sys"]: if "country" in page["sys"]:
location_parts.append(page.data["sys"]["country"]) location_parts.append(page["sys"]["country"])
location_str = ", ".join(location_parts) location_str = ", ".join(location_parts)
celsius = "%dC" % page.data["main"]["temp"] celsius = "%dC" % page["main"]["temp"]
fahrenheit = "%dF" % ((page.data["main"]["temp"]*(9/5))+32) fahrenheit = "%dF" % ((page["main"]["temp"]*(9/5))+32)
description = page.data["weather"][0]["description"].title() description = page["weather"][0]["description"].title()
humidity = "%s%%" % page.data["main"]["humidity"] humidity = "%s%%" % page["main"]["humidity"]
# wind speed is in metres per second - 3.6* for KMh # wind speed is in metres per second - 3.6* for KMh
wind_speed = 3.6*page.data["wind"]["speed"] wind_speed = 3.6*page["wind"]["speed"]
wind_speed_k = "%sKMh" % round(wind_speed, 1) wind_speed_k = "%sKMh" % round(wind_speed, 1)
wind_speed_m = "%sMPh" % round(0.6214*wind_speed, 1) wind_speed_m = "%sMPh" % round(0.6214*wind_speed, 1)

4
modules/wolframalpha.py
View file

@ -20,13 +20,13 @@ class Module(ModuleManager.BaseModule):
page = utils.http.request(URL_WA, page = utils.http.request(URL_WA,
get_params={"i": event["args"], get_params={"i": event["args"],
"appid": self.bot.config["wolframalpha-api-key"], "appid": self.bot.config["wolframalpha-api-key"],
"reinterpret": "true", "units": "metric"}, code=True) "reinterpret": "true", "units": "metric"})
except utils.http.HTTPTimeoutException: except utils.http.HTTPTimeoutException:
page = None page = None
if page: if page:
if page.code == 200: if page.code == 200:
event["stdout"].write("%s: %s" % (event["args"], page.data)) event["stdout"].write("%s: %s" % (event["args"], page.decode()))
else: else:
event["stdout"].write("No results") event["stdout"].write("No results")
else: else:

2
modules/youtube.py
View file

@ -87,7 +87,7 @@ class Module(ModuleManager.BaseModule):
def get_playlist_page(self, playlist_id): def get_playlist_page(self, playlist_id):
return utils.http.request(URL_YOUTUBEPLAYLIST, get_params={ return utils.http.request(URL_YOUTUBEPLAYLIST, get_params={
"part": "contentDetails,snippet", "id": playlist_id, "part": "contentDetails,snippet", "id": playlist_id,
"key": self.bot.config["google-api-key"]}, json=True) "key": self.bot.config["google-api-key"]}).json()
def playlist_details(self, playlist_id): def playlist_details(self, playlist_id):
page = self.get_playlist_page(playlist_id) page = self.get_playlist_page(playlist_id)
if page["items"]: if page["items"]:

30
src/utils/http.py
View file

@ -67,12 +67,10 @@ class Request(object):
cookies: typing.Dict[str, str] = dataclasses.field( cookies: typing.Dict[str, str] = dataclasses.field(
default_factory=dict) default_factory=dict)
json: bool = False
json_body: bool = False json_body: bool = False
allow_redirects: bool = True allow_redirects: bool = True
check_content_type: bool = True check_content_type: bool = True
detect_encoding: bool = True
fallback_encoding: typing.Optional[str] = None fallback_encoding: typing.Optional[str] = None
content_type: typing.Optional[str] = None content_type: typing.Optional[str] = None
proxy: typing.Optional[str] = None proxy: typing.Optional[str] = None
@ -116,7 +114,7 @@ class Request(object):
return None return None
class Response(object): class Response(object):
def __init__(self, code: int, data: typing.Any, encoding: str, def __init__(self, code: int, data: bytes, encoding: str,
headers: typing.Dict[str, str], cookies: typing.Dict[str, str]): headers: typing.Dict[str, str], cookies: typing.Dict[str, str]):
self.code = code self.code = code
self.data = data self.data = data
@ -124,8 +122,8 @@ class Response(object):
self.encoding = encoding self.encoding = encoding
self.headers = headers self.headers = headers
self.cookies = cookies self.cookies = cookies
def decode(self) -> str: def decode(self, encoding="utf8") -> str:
return self.data return self.data.decode(encoding)
def json(self) -> typing.Any: def json(self) -> typing.Any:
return _json.loads(self.data) return _json.loads(self.data)
def soup(self, parser: str="lxml") -> bs4.BeautifulSoup: def soup(self, parser: str="lxml") -> bs4.BeautifulSoup:
@ -199,27 +197,11 @@ def _request(request_obj: Request) -> Response:
else: else:
encoding = "iso-8859-1" encoding = "iso-8859-1"
if (request_obj.detect_encoding and if (response.content_type and
response.content_type and
response.content_type in SOUP_CONTENT_TYPES): response.content_type in SOUP_CONTENT_TYPES):
souped = bs4.BeautifulSoup(response.data, "lxml") encoding = _find_encoding(response.data) or encoding
encoding = _find_encoding(souped) or encoding response.encoding = encoding
def _decode_data():
return response.data.decode(encoding)
if request_obj.json and response.data:
data = _decode_data()
try:
response.data = _json.loads(data)
return response
except _json.decoder.JSONDecodeError as e:
raise HTTPParsingException(str(e), data)
if response.content_type in DECODE_CONTENT_TYPES:
response.data = _decode_data()
return response
else:
return response return response
class RequestManyException(Exception): class RequestManyException(Exception):