implement utils.http.request_many as a tonado ioloop yield

README.md

@@ -2,7 +2,7 @@
 Python3 event-driven modular IRC bot!
 
 ## Requirements
-Python3.6+ and [BeautifulSoup4](https://pypi.python.org/pypi/beautifulsoup4), [dnspython](https://pypi.org/project/dnspython/), [feedparser](https://pypi.org/project/feedparser/), [lxml](https://pypi.org/project/lxml/), [netifaces](https://pypi.org/project/netifaces/), [requests](https://pypi.org/project/requests/), [scrypt](https://pypi.python.org/pypi/scrypt), [suds](https://pypi.python.org/pypi/suds-jurko) and [tweepy](https://pypi.org/project/tweepy/). Use `pip3 install -r requirements.txt` to install them all at once.
+Python3.6+ and [BeautifulSoup4](https://pypi.python.org/pypi/beautifulsoup4), [dnspython](https://pypi.org/project/dnspython/), [feedparser](https://pypi.org/project/feedparser/), [lxml](https://pypi.org/project/lxml/), [netifaces](https://pypi.org/project/netifaces/), [requests](https://pypi.org/project/requests/), [scrypt](https://pypi.python.org/pypi/scrypt), [suds](https://pypi.python.org/pypi/suds-jurko), [tornado](https://pypi.org/project/tornado/) and [tweepy](https://pypi.org/project/tweepy/). Use `pip3 install -r requirements.txt` to install them all at once.
 
 ## Setup
 See [docs/help/setup.md](docs/help/setup.md).

requirements.txt

@@ -6,4 +6,5 @@ netifaces
 requests
 scrypt
 suds-jurko
+tornado
 tweepy

src/utils/http.py

@@ -1,7 +1,7 @@
 import ipaddress, re, signal, socket, traceback, typing
 import urllib.error, urllib.parse
 import json as _json
-import bs4, netifaces, requests
+import bs4, netifaces, requests, tornado.gen, tornado.httpclient, tornado.ioloop
 from src import utils
 
 REGEX_URL = re.compile("https?://[A-Z0-9{}]+".format(re.escape("-._~:/%?#[]@!$&'()*+,;=")), re.I)
@@ -109,6 +109,24 @@ def request(url: str, method: str="GET", get_params: dict={},
 
     return Response(response.status_code, data, response_headers)
 
+def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
+    responses = {}
+
+    @tornado.gen.coroutine
+    def _request():
+        for url in urls:
+            client = tornado.httpclient.AsyncHTTPClient()
+            request = tornado.httpclient.HTTPRequest(url, method="GET",
+                connect_timeout=2, request_timeout=2)
+            response = yield client.fetch(request)
+
+            headers = utils.CaseInsensitiveDict(dict(response.headers))
+            data = response.body.decode("utf8")
+            responses[url] = Response(response.code, data, headers)
+
+    tornado.ioloop.IOLoop.current().run_sync(_request)
+    return responses
+
 def strip_html(s: str) -> str:
     return bs4.BeautifulSoup(s, "lxml").get_text()