Make all http requests async in a thread pool

This fixes the main thread locking up when there is latency
on http requests like mojang api lookups, sending webhooks,
or even logging elasticsearch lookups.

Fixes #29

Pipfile

@@ -19,3 +19,4 @@ quarry = "*"
 discord = {py = "*"}
 bidict = "*"
 minecraft = {git = "https://github.com/ammaraskar/pyCraft.git"}
+requests-futures = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "9dc84aff633d8c6a344ee0e25946c2f0f57c2588d5daee523d15a95c24a5e537"
+            "sha256": "6db8b05b7b2f00ffb63631f49356c434e46360c4ba848e2e83380f5bc93662ea"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -43,10 +43,10 @@
         },
         "asn1crypto": {
             "hashes": [
-                "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
-                "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"
+                "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292",
+                "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f"
             ],
-            "version": "==0.24.0"
+            "version": "==1.0.1"
         },
         "async-timeout": {
             "hashes": [
@@ -57,10 +57,10 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
-                "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
+                "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
+                "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
             ],
-            "version": "==19.1.0"
+            "version": "==19.2.0"
         },
         "automat": {
             "hashes": [
@@ -71,11 +71,11 @@
         },
         "bidict": {
             "hashes": [
-                "sha256:70af520df680b00a8ff554e60ce305644a59e8d693af1ab28bd89ad2b6a4232f",
-                "sha256:ba01c15664af7bd2b032a60fa21ed3575a66fd8b20b35436bedce45837ae556d"
+                "sha256:1742a25a9ef1b1ac4000683406879a3e1a6577faa02f31e482e6c84e2e3bf628",
+                "sha256:621db384daef94b7031897ca479f4feed025989a64e91c6fae88fb88a842488c"
             ],
             "index": "pypi",
-            "version": "==0.18.2"
+            "version": "==0.18.3"
         },
         "cached-property": {
             "hashes": [
@@ -318,6 +318,13 @@
             "index": "pypi",
             "version": "==2.22.0"
         },
+        "requests-futures": {
+            "hashes": [
+                "sha256:35547502bf1958044716a03a2f47092a89efe8f9789ab0c4c528d9c9c30bc148"
+            ],
+            "index": "pypi",
+            "version": "==1.0.0"
+        },
         "service-identity": {
             "hashes": [
                 "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
@@ -334,10 +341,10 @@
         },
         "sqlalchemy": {
             "hashes": [
-                "sha256:2f8ff566a4d3a92246d367f2e9cd6ed3edeef670dcd6dda6dfdc9efed88bcd80"
+                "sha256:272a835758908412e75e87f75dd0179a51422715c125ce42109632910526b1fd"
             ],
             "index": "pypi",
-            "version": "==1.3.8"
+            "version": "==1.3.9"
         },
         "twisted": {
             "hashes": [
@@ -365,10 +372,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb",
-                "sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47"
+                "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
+                "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
             ],
-            "version": "==1.25.5"
+            "version": "==1.25.6"
         },
         "websockets": {
             "hashes": [
@@ -450,10 +457,10 @@
     "develop": {
         "astroid": {
             "hashes": [
-                "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4",
-                "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4"
+                "sha256:98c665ad84d10b18318c5ab7c3d203fe11714cbad2a4aef4f44651f415392754",
+                "sha256:b7546ffdedbf7abcfbff93cd1de9e9980b1ef744852689decc5aeada324238c6"
             ],
-            "version": "==2.2.5"
+            "version": "==2.3.1"
         },
         "entrypoints": {
             "hashes": [
@@ -566,11 +573,11 @@
         },
         "pylint": {
             "hashes": [
-                "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09",
-                "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1"
+                "sha256:7edbae11476c2182708063ac387a8f97c760d9cfe36a5ede0ca996f90cf346c8",
+                "sha256:844ce067788028c1a35086a5c66bc5e599ddd851841c41d6ee1623b36774d9f2"
             ],
             "index": "pypi",
-            "version": "==2.3.1"
+            "version": "==2.4.2"
         },
         "pyparsing": {
             "hashes": [
@@ -619,7 +626,7 @@
                 "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
                 "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
             ],
-            "markers": "implementation_name == 'cpython'",
+            "markers": "implementation_name == 'cpython' and python_version < '3.8'",
             "version": "==1.4.0"
         },
         "virtualenv": {

minecraft_discord_bridge/elasticsearch_logger.py

@@ -21,11 +21,12 @@ import time
 from enum import Enum
 import logging
 
-import requests
+import requests_futures
 
 
 class ElasticsearchLogger():
-    def __init__(self, url: str, username: str = "", password: str = ""):
+    def __init__(self, futures_session: requests_futures.sessions, url: str, username: str = "", password: str = ""):
+        self.futures_session = futures_session
         self.url = url
         self.username = username
         self.password = password
@@ -68,10 +69,10 @@ class ElasticsearchLogger():
     def post_request(self, endpoint, payload):
         the_url = "{}{}".format(self.url, endpoint)
         if self.username and self.password:
-            post = requests.post(the_url, auth=(self.username, self.password), json=payload)
+            future = self.futures_session.post(the_url, auth=(self.username, self.password), json=payload)
         else:
-            post = requests.post(the_url, json=payload)
-        self.log.debug(post.text)
+            future = self.futures_session.post(the_url, json=payload)
+        self.log.debug(future.result().text)
 
 
 class ConnectionReason(Enum):

minecraft_discord_bridge/minecraft_discord_bridge.py

@@ -32,7 +32,7 @@ import asyncio
 from threading import Thread
 from datetime import datetime, timedelta, timezone
 
-import requests
+from requests import RequestException
 from minecraft import authentication
 from minecraft.exceptions import YggdrasilError
 from minecraft.networking.connection import Connection
@@ -40,6 +40,7 @@ from minecraft.networking.packets import clientbound, serverbound
 import discord
 from mcstatus import MinecraftServer
 from bidict import bidict
+from requests_futures.sessions import FuturesSession
 
 import minecraft_discord_bridge
 from .database_session import DatabaseSession
@@ -72,15 +73,20 @@ class MinecraftDiscordBridge():
         self.database_session.initialize(self.config)
         self.bot_perms = discord.Permissions()
         self.bot_perms.update(manage_messages=True, manage_webhooks=True)
+        # Async http request pool
+        self.req_future_session = FuturesSession(max_workers=100)
         # We need to import twisted after setting up the logger because twisted hijacks our logging
         from . import auth_server
         auth_server.DATABASE_SESSION = self.database_session
         if self.config.es_enabled:
             if self.config.es_auth:
                 self.es_logger = ElasticsearchLogger(
-                    self.config.es_url, self.config.es_username, self.config.es_password)
+                    self.req_future_session,
+                    self.config.es_url,
+                    self.config.es_username,
+                    self.config.es_password)
             else:
-                self.es_logger = ElasticsearchLogger(self.config.es_url)
+                self.es_logger = ElasticsearchLogger(self.req_future_session, self.config.es_url)
 
         @self.discord_bot.event
         async def on_ready():  # pylint: disable=W0612
@@ -493,8 +499,8 @@ class MinecraftDiscordBridge():
         if mc_uuid not in self.uuid_cache:
             try:
                 short_uuid = mc_uuid.replace("-", "")
-                mojang_response = requests.get("https://api.mojang.com/user/profiles/{}/names".format(
-                    short_uuid)).json()
+                mojang_response = self.req_future_session.get("https://api.mojang.com/user/profiles/{}/names".format(
+                    short_uuid)).result().json()
                 if len(mojang_response) > 1:
                     # Multiple name changes
                     player_username = mojang_response[-1]["name"]
@@ -503,7 +509,7 @@ class MinecraftDiscordBridge():
                     player_username = mojang_response[0]["name"]
                 self.uuid_cache[mc_uuid] = player_username
                 return player_username
-            except requests.RequestException as ex:
+            except RequestException as ex:
                 self.logger.error(ex, exc_info=True)
                 self.logger.error("Failed to lookup %s's username using the Mojang API.", mc_uuid)
         else:
@@ -512,12 +518,12 @@ class MinecraftDiscordBridge():
     def mc_username_to_uuid(self, username: str):
         if username not in self.uuid_cache.inv:
             try:
-                player_uuid = requests.get(
+                player_uuid = self.req_future_session.get(
                     "https://api.mojang.com/users/profiles/minecraft/{}".format(username)).json()["id"]
                 long_uuid = uuid.UUID(player_uuid)
                 self.uuid_cache.inv[username] = str(long_uuid)
                 return player_uuid
-            except requests.RequestException:
+            except RequestException:
                 self.logger.error("Failed to lookup %s's username using the Mojang API.", username)
         else:
             return self.uuid_cache.inv[username]
@@ -701,7 +707,7 @@ class MinecraftDiscordBridge():
                         'embeds': [{'color': 65280, 'title': '**Joined the game**'}]
                     }
                     for webhook in self.webhooks:
-                        requests.post(webhook, json=webhook_payload)
+                        self.req_future_session.post(webhook, json=webhook_payload)
                     if self.config.es_enabled:
                         self.es_logger.log_connection(
                             uuid=action.uuid, reason=ConnectionReason.CONNECTED, count=len(self.player_list))
@@ -734,7 +740,7 @@ class MinecraftDiscordBridge():
                     'embeds': [{'color': 16711680, 'title': '**Left the game**'}]
                 }
                 for webhook in self.webhooks:
-                    requests.post(webhook, json=webhook_payload)
+                    self.req_future_session.post(webhook, json=webhook_payload)
                 del self.uuid_cache[action.uuid]
                 if action.uuid in self.player_list:
                     del self.player_list[action.uuid]
@@ -785,7 +791,7 @@ class MinecraftDiscordBridge():
                 'content': '{}'.format(message)
             }
             for webhook in self.webhooks:
-                requests.post(webhook, json=webhook_payload)
+                self.req_future_session.post(webhook, json=webhook_payload)
             if self.config.es_enabled:
                 self.es_logger.log_chat_message(
                     uuid=player_uuid, display_name=username, message=original_message, message_unformatted=chat_string)