Firepup650/bitbot-3.11-fork
2
0
Fork 0
Code Issues Pull requests Activity Actions

switch request_many() to use asyncio.gather

Browse source
This commit is contained in:
jesopo 2019-07-08 13:46:27 +01:00
parent c6c60a5192
commit ee0ec0eca1
1 changed files with 18 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/utils/http.py
View file

@ -116,24 +116,28 @@ def request(url: str, method: str="GET", get_params: dict={},
def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]: def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
responses = {} responses = {}
async def _request(): async def _request(url):
for url in urls: client = tornado.httpclient.AsyncHTTPClient()
client = tornado.httpclient.AsyncHTTPClient() request = tornado.httpclient.HTTPRequest(url, method="GET",
request = tornado.httpclient.HTTPRequest(url, method="GET", connect_timeout=2, request_timeout=2)
connect_timeout=2, request_timeout=2)
response = None response = None
try: try:
response = await client.fetch(request) response = await client.fetch(request)
except: except:
return return
headers = utils.CaseInsensitiveDict(dict(response.headers)) headers = utils.CaseInsensitiveDict(dict(response.headers))
data = response.body.decode("utf8") data = response.body.decode("utf8")
responses[url] = Response(response.code, data, headers) responses[url] = Response(response.code, data, headers)
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.run_until_complete(_request()) awaits = []
for url in urls:
awaits.append(_request(url))
task = asyncio.gather(*awaits, return_exceptions=True)
loop.run_until_complete(task)
return responses return responses
def strip_html(s: str) -> str: def strip_html(s: str) -> str: