switch to using asyncio's event loop
This commit is contained in:
parent
15e143fcff
commit
ecb8364d0d
1 changed files with 7 additions and 6 deletions
|
@ -1,7 +1,8 @@
|
|||
import ipaddress, re, signal, socket, traceback, typing
|
||||
import asyncio, ipaddress, re, signal, socket, traceback, typing
|
||||
import urllib.error, urllib.parse
|
||||
import json as _json
|
||||
import bs4, netifaces, requests, tornado.gen, tornado.httpclient, tornado.ioloop
|
||||
import bs4, netifaces, requests
|
||||
import tornado.httpclient
|
||||
from src import utils
|
||||
|
||||
REGEX_URL = re.compile("https?://[A-Z0-9{}]+".format(re.escape("-._~:/%?#[]@!$&'()*+,;=")), re.I)
|
||||
|
@ -112,19 +113,19 @@ def request(url: str, method: str="GET", get_params: dict={},
|
|||
def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
|
||||
responses = {}
|
||||
|
||||
@tornado.gen.coroutine
|
||||
def _request():
|
||||
async def _request():
|
||||
for url in urls:
|
||||
client = tornado.httpclient.AsyncHTTPClient()
|
||||
request = tornado.httpclient.HTTPRequest(url, method="GET",
|
||||
connect_timeout=2, request_timeout=2)
|
||||
response = yield client.fetch(request)
|
||||
response = await client.fetch(request)
|
||||
|
||||
headers = utils.CaseInsensitiveDict(dict(response.headers))
|
||||
data = response.body.decode("utf8")
|
||||
responses[url] = Response(response.code, data, headers)
|
||||
|
||||
tornado.ioloop.IOLoop.current().run_sync(_request)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(_request())
|
||||
return responses
|
||||
|
||||
def strip_html(s: str) -> str:
|
||||
|
|
Loading…
Reference in a new issue