Firepup650/bitbot-3.11-fork
2
0
Fork 0
Code Issues Pull requests Activity Actions

switch to using asyncio's event loop

Browse source
This commit is contained in:
jesopo 2019-07-08 12:45:10 +01:00
parent 15e143fcff
commit ecb8364d0d
1 changed files with 7 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/utils/http.py
View file

@ -1,7 +1,8 @@
import ipaddress, re, signal, socket, traceback, typing import asyncio, ipaddress, re, signal, socket, traceback, typing
import urllib.error, urllib.parse import urllib.error, urllib.parse
import json as _json import json as _json
import bs4, netifaces, requests, tornado.gen, tornado.httpclient, tornado.ioloop import bs4, netifaces, requests
import tornado.httpclient
from src import utils from src import utils
REGEX_URL = re.compile("https?://[A-Z0-9{}]+".format(re.escape("-._~:/%?#[]@!$&'()*+,;=")), re.I) REGEX_URL = re.compile("https?://[A-Z0-9{}]+".format(re.escape("-._~:/%?#[]@!$&'()*+,;=")), re.I)
@ -112,19 +113,19 @@ def request(url: str, method: str="GET", get_params: dict={},
def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]: def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
responses = {} responses = {}
@tornado.gen.coroutine async def _request():
def _request():
for url in urls: for url in urls:
client = tornado.httpclient.AsyncHTTPClient() client = tornado.httpclient.AsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(url, method="GET", request = tornado.httpclient.HTTPRequest(url, method="GET",
connect_timeout=2, request_timeout=2) connect_timeout=2, request_timeout=2)
response = yield client.fetch(request) response = await client.fetch(request)
headers = utils.CaseInsensitiveDict(dict(response.headers)) headers = utils.CaseInsensitiveDict(dict(response.headers))
data = response.body.decode("utf8") data = response.body.decode("utf8")
responses[url] = Response(response.code, data, headers) responses[url] = Response(response.code, data, headers)
tornado.ioloop.IOLoop.current().run_sync(_request) loop = asyncio.get_event_loop()
loop.run_until_complete(_request())
return responses return responses
def strip_html(s: str) -> str: def strip_html(s: str) -> str: