Merge branch 'feature/async-http'

master
Tristan Gosselin-Hane 6 years ago
commit 00d4ae028c
No known key found for this signature in database
GPG Key ID: 0B7E55B60DCA15D5
  1. 1
      Pipfile
  2. 51
      Pipfile.lock
  3. 11
      minecraft_discord_bridge/elasticsearch_logger.py
  4. 28
      minecraft_discord_bridge/minecraft_discord_bridge.py

@ -19,3 +19,4 @@ quarry = "*"
discord = {py = "*"} discord = {py = "*"}
bidict = "*" bidict = "*"
minecraft = {git = "https://github.com/ammaraskar/pyCraft.git"} minecraft = {git = "https://github.com/ammaraskar/pyCraft.git"}
requests-futures = "*"

51
Pipfile.lock generated

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "9dc84aff633d8c6a344ee0e25946c2f0f57c2588d5daee523d15a95c24a5e537" "sha256": "6db8b05b7b2f00ffb63631f49356c434e46360c4ba848e2e83380f5bc93662ea"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": {}, "requires": {},
@ -43,10 +43,10 @@
}, },
"asn1crypto": { "asn1crypto": {
"hashes": [ "hashes": [
"sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292",
"sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f"
], ],
"version": "==0.24.0" "version": "==1.0.1"
}, },
"async-timeout": { "async-timeout": {
"hashes": [ "hashes": [
@ -57,10 +57,10 @@
}, },
"attrs": { "attrs": {
"hashes": [ "hashes": [
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
], ],
"version": "==19.1.0" "version": "==19.2.0"
}, },
"automat": { "automat": {
"hashes": [ "hashes": [
@ -71,11 +71,11 @@
}, },
"bidict": { "bidict": {
"hashes": [ "hashes": [
"sha256:70af520df680b00a8ff554e60ce305644a59e8d693af1ab28bd89ad2b6a4232f", "sha256:1742a25a9ef1b1ac4000683406879a3e1a6577faa02f31e482e6c84e2e3bf628",
"sha256:ba01c15664af7bd2b032a60fa21ed3575a66fd8b20b35436bedce45837ae556d" "sha256:621db384daef94b7031897ca479f4feed025989a64e91c6fae88fb88a842488c"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.18.2" "version": "==0.18.3"
}, },
"cached-property": { "cached-property": {
"hashes": [ "hashes": [
@ -318,6 +318,13 @@
"index": "pypi", "index": "pypi",
"version": "==2.22.0" "version": "==2.22.0"
}, },
"requests-futures": {
"hashes": [
"sha256:35547502bf1958044716a03a2f47092a89efe8f9789ab0c4c528d9c9c30bc148"
],
"index": "pypi",
"version": "==1.0.0"
},
"service-identity": { "service-identity": {
"hashes": [ "hashes": [
"sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36", "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
@ -334,10 +341,10 @@
}, },
"sqlalchemy": { "sqlalchemy": {
"hashes": [ "hashes": [
"sha256:2f8ff566a4d3a92246d367f2e9cd6ed3edeef670dcd6dda6dfdc9efed88bcd80" "sha256:272a835758908412e75e87f75dd0179a51422715c125ce42109632910526b1fd"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.3.8" "version": "==1.3.9"
}, },
"twisted": { "twisted": {
"hashes": [ "hashes": [
@ -365,10 +372,10 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb", "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
"sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47" "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
], ],
"version": "==1.25.5" "version": "==1.25.6"
}, },
"websockets": { "websockets": {
"hashes": [ "hashes": [
@ -450,10 +457,10 @@
"develop": { "develop": {
"astroid": { "astroid": {
"hashes": [ "hashes": [
"sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", "sha256:98c665ad84d10b18318c5ab7c3d203fe11714cbad2a4aef4f44651f415392754",
"sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" "sha256:b7546ffdedbf7abcfbff93cd1de9e9980b1ef744852689decc5aeada324238c6"
], ],
"version": "==2.2.5" "version": "==2.3.1"
}, },
"entrypoints": { "entrypoints": {
"hashes": [ "hashes": [
@ -566,11 +573,11 @@
}, },
"pylint": { "pylint": {
"hashes": [ "hashes": [
"sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", "sha256:7edbae11476c2182708063ac387a8f97c760d9cfe36a5ede0ca996f90cf346c8",
"sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" "sha256:844ce067788028c1a35086a5c66bc5e599ddd851841c41d6ee1623b36774d9f2"
], ],
"index": "pypi", "index": "pypi",
"version": "==2.3.1" "version": "==2.4.2"
}, },
"pyparsing": { "pyparsing": {
"hashes": [ "hashes": [
@ -619,7 +626,7 @@
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
], ],
"markers": "implementation_name == 'cpython'", "markers": "implementation_name == 'cpython' and python_version < '3.8'",
"version": "==1.4.0" "version": "==1.4.0"
}, },
"virtualenv": { "virtualenv": {

@ -21,11 +21,12 @@ import time
from enum import Enum from enum import Enum
import logging import logging
import requests import requests_futures
class ElasticsearchLogger(): class ElasticsearchLogger():
def __init__(self, url: str, username: str = "", password: str = ""): def __init__(self, futures_session: requests_futures.sessions, url: str, username: str = "", password: str = ""):
self.futures_session = futures_session
self.url = url self.url = url
self.username = username self.username = username
self.password = password self.password = password
@ -68,10 +69,10 @@ class ElasticsearchLogger():
def post_request(self, endpoint, payload): def post_request(self, endpoint, payload):
the_url = "{}{}".format(self.url, endpoint) the_url = "{}{}".format(self.url, endpoint)
if self.username and self.password: if self.username and self.password:
post = requests.post(the_url, auth=(self.username, self.password), json=payload) future = self.futures_session.post(the_url, auth=(self.username, self.password), json=payload)
else: else:
post = requests.post(the_url, json=payload) future = self.futures_session.post(the_url, json=payload)
self.log.debug(post.text) self.log.debug(future.result().text)
class ConnectionReason(Enum): class ConnectionReason(Enum):

@ -32,7 +32,7 @@ import asyncio
from threading import Thread from threading import Thread
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
import requests from requests import RequestException
from minecraft import authentication from minecraft import authentication
from minecraft.exceptions import YggdrasilError from minecraft.exceptions import YggdrasilError
from minecraft.networking.connection import Connection from minecraft.networking.connection import Connection
@ -40,6 +40,7 @@ from minecraft.networking.packets import clientbound, serverbound
import discord import discord
from mcstatus import MinecraftServer from mcstatus import MinecraftServer
from bidict import bidict from bidict import bidict
from requests_futures.sessions import FuturesSession
import minecraft_discord_bridge import minecraft_discord_bridge
from .database_session import DatabaseSession from .database_session import DatabaseSession
@ -72,15 +73,20 @@ class MinecraftDiscordBridge():
self.database_session.initialize(self.config) self.database_session.initialize(self.config)
self.bot_perms = discord.Permissions() self.bot_perms = discord.Permissions()
self.bot_perms.update(manage_messages=True, manage_webhooks=True) self.bot_perms.update(manage_messages=True, manage_webhooks=True)
# Async http request pool
self.req_future_session = FuturesSession(max_workers=100)
# We need to import twisted after setting up the logger because twisted hijacks our logging # We need to import twisted after setting up the logger because twisted hijacks our logging
from . import auth_server from . import auth_server
auth_server.DATABASE_SESSION = self.database_session auth_server.DATABASE_SESSION = self.database_session
if self.config.es_enabled: if self.config.es_enabled:
if self.config.es_auth: if self.config.es_auth:
self.es_logger = ElasticsearchLogger( self.es_logger = ElasticsearchLogger(
self.config.es_url, self.config.es_username, self.config.es_password) self.req_future_session,
self.config.es_url,
self.config.es_username,
self.config.es_password)
else: else:
self.es_logger = ElasticsearchLogger(self.config.es_url) self.es_logger = ElasticsearchLogger(self.req_future_session, self.config.es_url)
@self.discord_bot.event @self.discord_bot.event
async def on_ready(): # pylint: disable=W0612 async def on_ready(): # pylint: disable=W0612
@ -493,8 +499,8 @@ class MinecraftDiscordBridge():
if mc_uuid not in self.uuid_cache: if mc_uuid not in self.uuid_cache:
try: try:
short_uuid = mc_uuid.replace("-", "") short_uuid = mc_uuid.replace("-", "")
mojang_response = requests.get("https://api.mojang.com/user/profiles/{}/names".format( mojang_response = self.req_future_session.get("https://api.mojang.com/user/profiles/{}/names".format(
short_uuid)).json() short_uuid)).result().json()
if len(mojang_response) > 1: if len(mojang_response) > 1:
# Multiple name changes # Multiple name changes
player_username = mojang_response[-1]["name"] player_username = mojang_response[-1]["name"]
@ -503,7 +509,7 @@ class MinecraftDiscordBridge():
player_username = mojang_response[0]["name"] player_username = mojang_response[0]["name"]
self.uuid_cache[mc_uuid] = player_username self.uuid_cache[mc_uuid] = player_username
return player_username return player_username
except requests.RequestException as ex: except RequestException as ex:
self.logger.error(ex, exc_info=True) self.logger.error(ex, exc_info=True)
self.logger.error("Failed to lookup %s's username using the Mojang API.", mc_uuid) self.logger.error("Failed to lookup %s's username using the Mojang API.", mc_uuid)
else: else:
@ -512,12 +518,12 @@ class MinecraftDiscordBridge():
def mc_username_to_uuid(self, username: str): def mc_username_to_uuid(self, username: str):
if username not in self.uuid_cache.inv: if username not in self.uuid_cache.inv:
try: try:
player_uuid = requests.get( player_uuid = self.req_future_session.get(
"https://api.mojang.com/users/profiles/minecraft/{}".format(username)).json()["id"] "https://api.mojang.com/users/profiles/minecraft/{}".format(username)).json()["id"]
long_uuid = uuid.UUID(player_uuid) long_uuid = uuid.UUID(player_uuid)
self.uuid_cache.inv[username] = str(long_uuid) self.uuid_cache.inv[username] = str(long_uuid)
return player_uuid return player_uuid
except requests.RequestException: except RequestException:
self.logger.error("Failed to lookup %s's username using the Mojang API.", username) self.logger.error("Failed to lookup %s's username using the Mojang API.", username)
else: else:
return self.uuid_cache.inv[username] return self.uuid_cache.inv[username]
@ -701,7 +707,7 @@ class MinecraftDiscordBridge():
'embeds': [{'color': 65280, 'title': '**Joined the game**'}] 'embeds': [{'color': 65280, 'title': '**Joined the game**'}]
} }
for webhook in self.webhooks: for webhook in self.webhooks:
requests.post(webhook, json=webhook_payload) self.req_future_session.post(webhook, json=webhook_payload)
if self.config.es_enabled: if self.config.es_enabled:
self.es_logger.log_connection( self.es_logger.log_connection(
uuid=action.uuid, reason=ConnectionReason.CONNECTED, count=len(self.player_list)) uuid=action.uuid, reason=ConnectionReason.CONNECTED, count=len(self.player_list))
@ -734,7 +740,7 @@ class MinecraftDiscordBridge():
'embeds': [{'color': 16711680, 'title': '**Left the game**'}] 'embeds': [{'color': 16711680, 'title': '**Left the game**'}]
} }
for webhook in self.webhooks: for webhook in self.webhooks:
requests.post(webhook, json=webhook_payload) self.req_future_session.post(webhook, json=webhook_payload)
del self.uuid_cache[action.uuid] del self.uuid_cache[action.uuid]
if action.uuid in self.player_list: if action.uuid in self.player_list:
del self.player_list[action.uuid] del self.player_list[action.uuid]
@ -785,7 +791,7 @@ class MinecraftDiscordBridge():
'content': '{}'.format(message) 'content': '{}'.format(message)
} }
for webhook in self.webhooks: for webhook in self.webhooks:
requests.post(webhook, json=webhook_payload) self.req_future_session.post(webhook, json=webhook_payload)
if self.config.es_enabled: if self.config.es_enabled:
self.es_logger.log_chat_message( self.es_logger.log_chat_message(
uuid=player_uuid, display_name=username, message=original_message, message_unformatted=chat_string) uuid=player_uuid, display_name=username, message=original_message, message_unformatted=chat_string)

Loading…
Cancel
Save