From 15e143fcff39d64c5acc2d93f61c2f00f65c2ee0 Mon Sep 17 00:00:00 2001
From: jesopo <github@lolnerd.net>
Date: Mon, 8 Jul 2019 11:43:09 +0100
Subject: [PATCH] implement utils.http.request_many as a tonado ioloop yield

---
 README.md         |  2 +-
 requirements.txt  |  1 +
 src/utils/http.py | 20 +++++++++++++++++++-
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index cfb2b07d..21829218 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 Python3 event-driven modular IRC bot!
 
 ## Requirements
-Python3.6+ and [BeautifulSoup4](https://pypi.python.org/pypi/beautifulsoup4), [dnspython](https://pypi.org/project/dnspython/), [feedparser](https://pypi.org/project/feedparser/), [lxml](https://pypi.org/project/lxml/), [netifaces](https://pypi.org/project/netifaces/), [requests](https://pypi.org/project/requests/), [scrypt](https://pypi.python.org/pypi/scrypt), [suds](https://pypi.python.org/pypi/suds-jurko) and [tweepy](https://pypi.org/project/tweepy/). Use `pip3 install -r requirements.txt` to install them all at once.
+Python3.6+ and [BeautifulSoup4](https://pypi.python.org/pypi/beautifulsoup4), [dnspython](https://pypi.org/project/dnspython/), [feedparser](https://pypi.org/project/feedparser/), [lxml](https://pypi.org/project/lxml/), [netifaces](https://pypi.org/project/netifaces/), [requests](https://pypi.org/project/requests/), [scrypt](https://pypi.python.org/pypi/scrypt), [suds](https://pypi.python.org/pypi/suds-jurko), [tornado](https://pypi.org/project/tornado/) and [tweepy](https://pypi.org/project/tweepy/). Use `pip3 install -r requirements.txt` to install them all at once.
 
 ## Setup
 See [docs/help/setup.md](docs/help/setup.md).
diff --git a/requirements.txt b/requirements.txt
index 75897f2c..a229508e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,5 @@ netifaces
 requests
 scrypt
 suds-jurko
+tornado
 tweepy
diff --git a/src/utils/http.py b/src/utils/http.py
index 2885ebee..d12a978b 100644
--- a/src/utils/http.py
+++ b/src/utils/http.py
@@ -1,7 +1,7 @@
 import ipaddress, re, signal, socket, traceback, typing
 import urllib.error, urllib.parse
 import json as _json
-import bs4, netifaces, requests
+import bs4, netifaces, requests, tornado.gen, tornado.httpclient, tornado.ioloop
 from src import utils
 
 REGEX_URL = re.compile("https?://[A-Z0-9{}]+".format(re.escape("-._~:/%?#[]@!$&'()*+,;=")), re.I)
@@ -109,6 +109,24 @@ def request(url: str, method: str="GET", get_params: dict={},
 
     return Response(response.status_code, data, response_headers)
 
+def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
+    responses = {}
+
+    @tornado.gen.coroutine
+    def _request():
+        for url in urls:
+            client = tornado.httpclient.AsyncHTTPClient()
+            request = tornado.httpclient.HTTPRequest(url, method="GET",
+                connect_timeout=2, request_timeout=2)
+            response = yield client.fetch(request)
+
+            headers = utils.CaseInsensitiveDict(dict(response.headers))
+            data = response.body.decode("utf8")
+            responses[url] = Response(response.code, data, headers)
+
+    tornado.ioloop.IOLoop.current().run_sync(_request)
+    return responses
+
 def strip_html(s: str) -> str:
     return bs4.BeautifulSoup(s, "lxml").get_text()