Merge branch 'feature/async-http'

master
Tristan Gosselin-Hane 6 years ago
commit 00d4ae028c
No known key found for this signature in database
GPG Key ID: 0B7E55B60DCA15D5
  1. 1
      Pipfile
  2. 51
      Pipfile.lock
  3. 11
      minecraft_discord_bridge/elasticsearch_logger.py
  4. 28
      minecraft_discord_bridge/minecraft_discord_bridge.py

@ -19,3 +19,4 @@ quarry = "*"
discord = {py = "*"}
bidict = "*"
minecraft = {git = "https://github.com/ammaraskar/pyCraft.git"}
requests-futures = "*"

51
Pipfile.lock generated

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "9dc84aff633d8c6a344ee0e25946c2f0f57c2588d5daee523d15a95c24a5e537"
"sha256": "6db8b05b7b2f00ffb63631f49356c434e46360c4ba848e2e83380f5bc93662ea"
},
"pipfile-spec": 6,
"requires": {},
@ -43,10 +43,10 @@
},
"asn1crypto": {
"hashes": [
"sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
"sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"
"sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292",
"sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f"
],
"version": "==0.24.0"
"version": "==1.0.1"
},
"async-timeout": {
"hashes": [
@ -57,10 +57,10 @@
},
"attrs": {
"hashes": [
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
],
"version": "==19.1.0"
"version": "==19.2.0"
},
"automat": {
"hashes": [
@ -71,11 +71,11 @@
},
"bidict": {
"hashes": [
"sha256:70af520df680b00a8ff554e60ce305644a59e8d693af1ab28bd89ad2b6a4232f",
"sha256:ba01c15664af7bd2b032a60fa21ed3575a66fd8b20b35436bedce45837ae556d"
"sha256:1742a25a9ef1b1ac4000683406879a3e1a6577faa02f31e482e6c84e2e3bf628",
"sha256:621db384daef94b7031897ca479f4feed025989a64e91c6fae88fb88a842488c"
],
"index": "pypi",
"version": "==0.18.2"
"version": "==0.18.3"
},
"cached-property": {
"hashes": [
@ -318,6 +318,13 @@
"index": "pypi",
"version": "==2.22.0"
},
"requests-futures": {
"hashes": [
"sha256:35547502bf1958044716a03a2f47092a89efe8f9789ab0c4c528d9c9c30bc148"
],
"index": "pypi",
"version": "==1.0.0"
},
"service-identity": {
"hashes": [
"sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
@ -334,10 +341,10 @@
},
"sqlalchemy": {
"hashes": [
"sha256:2f8ff566a4d3a92246d367f2e9cd6ed3edeef670dcd6dda6dfdc9efed88bcd80"
"sha256:272a835758908412e75e87f75dd0179a51422715c125ce42109632910526b1fd"
],
"index": "pypi",
"version": "==1.3.8"
"version": "==1.3.9"
},
"twisted": {
"hashes": [
@ -365,10 +372,10 @@
},
"urllib3": {
"hashes": [
"sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb",
"sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47"
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
],
"version": "==1.25.5"
"version": "==1.25.6"
},
"websockets": {
"hashes": [
@ -450,10 +457,10 @@
"develop": {
"astroid": {
"hashes": [
"sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4",
"sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4"
"sha256:98c665ad84d10b18318c5ab7c3d203fe11714cbad2a4aef4f44651f415392754",
"sha256:b7546ffdedbf7abcfbff93cd1de9e9980b1ef744852689decc5aeada324238c6"
],
"version": "==2.2.5"
"version": "==2.3.1"
},
"entrypoints": {
"hashes": [
@ -566,11 +573,11 @@
},
"pylint": {
"hashes": [
"sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09",
"sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1"
"sha256:7edbae11476c2182708063ac387a8f97c760d9cfe36a5ede0ca996f90cf346c8",
"sha256:844ce067788028c1a35086a5c66bc5e599ddd851841c41d6ee1623b36774d9f2"
],
"index": "pypi",
"version": "==2.3.1"
"version": "==2.4.2"
},
"pyparsing": {
"hashes": [
@ -619,7 +626,7 @@
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
],
"markers": "implementation_name == 'cpython'",
"markers": "implementation_name == 'cpython' and python_version < '3.8'",
"version": "==1.4.0"
},
"virtualenv": {

@ -21,11 +21,12 @@ import time
from enum import Enum
import logging
import requests
import requests_futures
class ElasticsearchLogger():
def __init__(self, url: str, username: str = "", password: str = ""):
def __init__(self, futures_session: requests_futures.sessions, url: str, username: str = "", password: str = ""):
self.futures_session = futures_session
self.url = url
self.username = username
self.password = password
@ -68,10 +69,10 @@ class ElasticsearchLogger():
def post_request(self, endpoint, payload):
the_url = "{}{}".format(self.url, endpoint)
if self.username and self.password:
post = requests.post(the_url, auth=(self.username, self.password), json=payload)
future = self.futures_session.post(the_url, auth=(self.username, self.password), json=payload)
else:
post = requests.post(the_url, json=payload)
self.log.debug(post.text)
future = self.futures_session.post(the_url, json=payload)
self.log.debug(future.result().text)
class ConnectionReason(Enum):

@ -32,7 +32,7 @@ import asyncio
from threading import Thread
from datetime import datetime, timedelta, timezone
import requests
from requests import RequestException
from minecraft import authentication
from minecraft.exceptions import YggdrasilError
from minecraft.networking.connection import Connection
@ -40,6 +40,7 @@ from minecraft.networking.packets import clientbound, serverbound
import discord
from mcstatus import MinecraftServer
from bidict import bidict
from requests_futures.sessions import FuturesSession
import minecraft_discord_bridge
from .database_session import DatabaseSession
@ -72,15 +73,20 @@ class MinecraftDiscordBridge():
self.database_session.initialize(self.config)
self.bot_perms = discord.Permissions()
self.bot_perms.update(manage_messages=True, manage_webhooks=True)
# Async http request pool
self.req_future_session = FuturesSession(max_workers=100)
# We need to import twisted after setting up the logger because twisted hijacks our logging
from . import auth_server
auth_server.DATABASE_SESSION = self.database_session
if self.config.es_enabled:
if self.config.es_auth:
self.es_logger = ElasticsearchLogger(
self.config.es_url, self.config.es_username, self.config.es_password)
self.req_future_session,
self.config.es_url,
self.config.es_username,
self.config.es_password)
else:
self.es_logger = ElasticsearchLogger(self.config.es_url)
self.es_logger = ElasticsearchLogger(self.req_future_session, self.config.es_url)
@self.discord_bot.event
async def on_ready(): # pylint: disable=W0612
@ -493,8 +499,8 @@ class MinecraftDiscordBridge():
if mc_uuid not in self.uuid_cache:
try:
short_uuid = mc_uuid.replace("-", "")
mojang_response = requests.get("https://api.mojang.com/user/profiles/{}/names".format(
short_uuid)).json()
mojang_response = self.req_future_session.get("https://api.mojang.com/user/profiles/{}/names".format(
short_uuid)).result().json()
if len(mojang_response) > 1:
# Multiple name changes
player_username = mojang_response[-1]["name"]
@ -503,7 +509,7 @@ class MinecraftDiscordBridge():
player_username = mojang_response[0]["name"]
self.uuid_cache[mc_uuid] = player_username
return player_username
except requests.RequestException as ex:
except RequestException as ex:
self.logger.error(ex, exc_info=True)
self.logger.error("Failed to lookup %s's username using the Mojang API.", mc_uuid)
else:
@ -512,12 +518,12 @@ class MinecraftDiscordBridge():
def mc_username_to_uuid(self, username: str):
if username not in self.uuid_cache.inv:
try:
player_uuid = requests.get(
player_uuid = self.req_future_session.get(
"https://api.mojang.com/users/profiles/minecraft/{}".format(username)).json()["id"]
long_uuid = uuid.UUID(player_uuid)
self.uuid_cache.inv[username] = str(long_uuid)
return player_uuid
except requests.RequestException:
except RequestException:
self.logger.error("Failed to lookup %s's username using the Mojang API.", username)
else:
return self.uuid_cache.inv[username]
@ -701,7 +707,7 @@ class MinecraftDiscordBridge():
'embeds': [{'color': 65280, 'title': '**Joined the game**'}]
}
for webhook in self.webhooks:
requests.post(webhook, json=webhook_payload)
self.req_future_session.post(webhook, json=webhook_payload)
if self.config.es_enabled:
self.es_logger.log_connection(
uuid=action.uuid, reason=ConnectionReason.CONNECTED, count=len(self.player_list))
@ -734,7 +740,7 @@ class MinecraftDiscordBridge():
'embeds': [{'color': 16711680, 'title': '**Left the game**'}]
}
for webhook in self.webhooks:
requests.post(webhook, json=webhook_payload)
self.req_future_session.post(webhook, json=webhook_payload)
del self.uuid_cache[action.uuid]
if action.uuid in self.player_list:
del self.player_list[action.uuid]
@ -785,7 +791,7 @@ class MinecraftDiscordBridge():
'content': '{}'.format(message)
}
for webhook in self.webhooks:
requests.post(webhook, json=webhook_payload)
self.req_future_session.post(webhook, json=webhook_payload)
if self.config.es_enabled:
self.es_logger.log_chat_message(
uuid=player_uuid, display_name=username, message=original_message, message_unformatted=chat_string)

Loading…
Cancel
Save