diff --git a/Pipfile b/Pipfile index 8a0a526..2d68b80 100644 --- a/Pipfile +++ b/Pipfile @@ -19,3 +19,4 @@ quarry = "*" discord = {py = "*"} bidict = "*" minecraft = {git = "https://github.com/ammaraskar/pyCraft.git"} +requests-futures = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 2a0b908..4efea5d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9dc84aff633d8c6a344ee0e25946c2f0f57c2588d5daee523d15a95c24a5e537" + "sha256": "6db8b05b7b2f00ffb63631f49356c434e46360c4ba848e2e83380f5bc93662ea" }, "pipfile-spec": 6, "requires": {}, @@ -43,10 +43,10 @@ }, "asn1crypto": { "hashes": [ - "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", - "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" + "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", + "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" ], - "version": "==0.24.0" + "version": "==1.0.1" }, "async-timeout": { "hashes": [ @@ -57,10 +57,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", + "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" ], - "version": "==19.1.0" + "version": "==19.2.0" }, "automat": { "hashes": [ @@ -71,11 +71,11 @@ }, "bidict": { "hashes": [ - "sha256:70af520df680b00a8ff554e60ce305644a59e8d693af1ab28bd89ad2b6a4232f", - "sha256:ba01c15664af7bd2b032a60fa21ed3575a66fd8b20b35436bedce45837ae556d" + "sha256:1742a25a9ef1b1ac4000683406879a3e1a6577faa02f31e482e6c84e2e3bf628", + "sha256:621db384daef94b7031897ca479f4feed025989a64e91c6fae88fb88a842488c" ], "index": "pypi", - "version": "==0.18.2" + "version": "==0.18.3" }, "cached-property": { "hashes": [ @@ -318,6 +318,13 @@ "index": "pypi", "version": "==2.22.0" }, + "requests-futures": { + "hashes": [ + "sha256:35547502bf1958044716a03a2f47092a89efe8f9789ab0c4c528d9c9c30bc148" + ], + "index": "pypi", + "version": "==1.0.0" + }, "service-identity": { "hashes": [ "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36", @@ -334,10 +341,10 @@ }, "sqlalchemy": { "hashes": [ - "sha256:2f8ff566a4d3a92246d367f2e9cd6ed3edeef670dcd6dda6dfdc9efed88bcd80" + "sha256:272a835758908412e75e87f75dd0179a51422715c125ce42109632910526b1fd" ], "index": "pypi", - "version": "==1.3.8" + "version": "==1.3.9" }, "twisted": { "hashes": [ @@ -365,10 +372,10 @@ }, "urllib3": { "hashes": [ - "sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb", - "sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47" + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], - "version": "==1.25.5" + "version": "==1.25.6" }, "websockets": { "hashes": [ @@ -450,10 +457,10 @@ "develop": { "astroid": { "hashes": [ - "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", - "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + "sha256:98c665ad84d10b18318c5ab7c3d203fe11714cbad2a4aef4f44651f415392754", + "sha256:b7546ffdedbf7abcfbff93cd1de9e9980b1ef744852689decc5aeada324238c6" ], - "version": "==2.2.5" + "version": "==2.3.1" }, "entrypoints": { "hashes": [ @@ -566,11 +573,11 @@ }, "pylint": { "hashes": [ - "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", - "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + "sha256:7edbae11476c2182708063ac387a8f97c760d9cfe36a5ede0ca996f90cf346c8", + "sha256:844ce067788028c1a35086a5c66bc5e599ddd851841c41d6ee1623b36774d9f2" ], "index": "pypi", - "version": "==2.3.1" + "version": "==2.4.2" }, "pyparsing": { "hashes": [ @@ -619,7 +626,7 @@ "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" ], - "markers": "implementation_name == 'cpython'", + "markers": "implementation_name == 'cpython' and python_version < '3.8'", "version": "==1.4.0" }, "virtualenv": { diff --git a/minecraft_discord_bridge/elasticsearch_logger.py b/minecraft_discord_bridge/elasticsearch_logger.py index c3cf991..b44b1f7 100644 --- a/minecraft_discord_bridge/elasticsearch_logger.py +++ b/minecraft_discord_bridge/elasticsearch_logger.py @@ -21,11 +21,12 @@ import time from enum import Enum import logging -import requests +import requests_futures class ElasticsearchLogger(): - def __init__(self, url: str, username: str = "", password: str = ""): + def __init__(self, futures_session: requests_futures.sessions, url: str, username: str = "", password: str = ""): + self.futures_session = futures_session self.url = url self.username = username self.password = password @@ -68,10 +69,10 @@ class ElasticsearchLogger(): def post_request(self, endpoint, payload): the_url = "{}{}".format(self.url, endpoint) if self.username and self.password: - post = requests.post(the_url, auth=(self.username, self.password), json=payload) + future = self.futures_session.post(the_url, auth=(self.username, self.password), json=payload) else: - post = requests.post(the_url, json=payload) - self.log.debug(post.text) + future = self.futures_session.post(the_url, json=payload) + self.log.debug(future.result().text) class ConnectionReason(Enum): diff --git a/minecraft_discord_bridge/minecraft_discord_bridge.py b/minecraft_discord_bridge/minecraft_discord_bridge.py index 9359776..35ce2ab 100755 --- a/minecraft_discord_bridge/minecraft_discord_bridge.py +++ b/minecraft_discord_bridge/minecraft_discord_bridge.py @@ -32,7 +32,7 @@ import asyncio from threading import Thread from datetime import datetime, timedelta, timezone -import requests +from requests import RequestException from minecraft import authentication from minecraft.exceptions import YggdrasilError from minecraft.networking.connection import Connection @@ -40,6 +40,7 @@ from minecraft.networking.packets import clientbound, serverbound import discord from mcstatus import MinecraftServer from bidict import bidict +from requests_futures.sessions import FuturesSession import minecraft_discord_bridge from .database_session import DatabaseSession @@ -72,15 +73,20 @@ class MinecraftDiscordBridge(): self.database_session.initialize(self.config) self.bot_perms = discord.Permissions() self.bot_perms.update(manage_messages=True, manage_webhooks=True) + # Async http request pool + self.req_future_session = FuturesSession(max_workers=100) # We need to import twisted after setting up the logger because twisted hijacks our logging from . import auth_server auth_server.DATABASE_SESSION = self.database_session if self.config.es_enabled: if self.config.es_auth: self.es_logger = ElasticsearchLogger( - self.config.es_url, self.config.es_username, self.config.es_password) + self.req_future_session, + self.config.es_url, + self.config.es_username, + self.config.es_password) else: - self.es_logger = ElasticsearchLogger(self.config.es_url) + self.es_logger = ElasticsearchLogger(self.req_future_session, self.config.es_url) @self.discord_bot.event async def on_ready(): # pylint: disable=W0612 @@ -493,8 +499,8 @@ class MinecraftDiscordBridge(): if mc_uuid not in self.uuid_cache: try: short_uuid = mc_uuid.replace("-", "") - mojang_response = requests.get("https://api.mojang.com/user/profiles/{}/names".format( - short_uuid)).json() + mojang_response = self.req_future_session.get("https://api.mojang.com/user/profiles/{}/names".format( + short_uuid)).result().json() if len(mojang_response) > 1: # Multiple name changes player_username = mojang_response[-1]["name"] @@ -503,7 +509,7 @@ class MinecraftDiscordBridge(): player_username = mojang_response[0]["name"] self.uuid_cache[mc_uuid] = player_username return player_username - except requests.RequestException as ex: + except RequestException as ex: self.logger.error(ex, exc_info=True) self.logger.error("Failed to lookup %s's username using the Mojang API.", mc_uuid) else: @@ -512,12 +518,12 @@ class MinecraftDiscordBridge(): def mc_username_to_uuid(self, username: str): if username not in self.uuid_cache.inv: try: - player_uuid = requests.get( + player_uuid = self.req_future_session.get( "https://api.mojang.com/users/profiles/minecraft/{}".format(username)).json()["id"] long_uuid = uuid.UUID(player_uuid) self.uuid_cache.inv[username] = str(long_uuid) return player_uuid - except requests.RequestException: + except RequestException: self.logger.error("Failed to lookup %s's username using the Mojang API.", username) else: return self.uuid_cache.inv[username] @@ -701,7 +707,7 @@ class MinecraftDiscordBridge(): 'embeds': [{'color': 65280, 'title': '**Joined the game**'}] } for webhook in self.webhooks: - requests.post(webhook, json=webhook_payload) + self.req_future_session.post(webhook, json=webhook_payload) if self.config.es_enabled: self.es_logger.log_connection( uuid=action.uuid, reason=ConnectionReason.CONNECTED, count=len(self.player_list)) @@ -734,7 +740,7 @@ class MinecraftDiscordBridge(): 'embeds': [{'color': 16711680, 'title': '**Left the game**'}] } for webhook in self.webhooks: - requests.post(webhook, json=webhook_payload) + self.req_future_session.post(webhook, json=webhook_payload) del self.uuid_cache[action.uuid] if action.uuid in self.player_list: del self.player_list[action.uuid] @@ -785,7 +791,7 @@ class MinecraftDiscordBridge(): 'content': '{}'.format(message) } for webhook in self.webhooks: - requests.post(webhook, json=webhook_payload) + self.req_future_session.post(webhook, json=webhook_payload) if self.config.es_enabled: self.es_logger.log_chat_message( uuid=player_uuid, display_name=username, message=original_message, message_unformatted=chat_string)