Create requests-async based TestClient, remove aiohttp dependency, drop Python 3.5
Update all tests to be compatible with requests-async Cleanup testing client changes with black and isort Remove Python 3.5 and other meta doc cleanup rename pyproject and fix pep517 error Add black config to tox.ini Cleanup tests and remove aiohttp tox.ini change for easier development commands Remove aiohttp from changelog and requirements Cleanup imports and Makefile
This commit is contained in:
Adam Hopkins
@@ -1,39 +1,143 @@
|
||||
from json import JSONDecodeError
|
||||
from sanic import Sanic
|
||||
import asyncio
|
||||
import functools
|
||||
import socket
|
||||
|
||||
from asyncio import sleep as aio_sleep
|
||||
from http.client import _encode
|
||||
from json import JSONDecodeError
|
||||
|
||||
import httpcore
|
||||
import requests_async as requests
|
||||
|
||||
from httpcore import PoolTimeout
|
||||
|
||||
from sanic import Sanic, server
|
||||
from sanic.response import text
|
||||
from sanic import server
|
||||
import aiohttp
|
||||
from aiohttp import TCPConnector
|
||||
from sanic.testing import SanicTestClient, HOST, PORT
|
||||
from sanic.testing import HOST, PORT, SanicTestClient
|
||||
|
||||
|
||||
# import traceback
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
CONFIG_FOR_TESTS = {"KEEP_ALIVE_TIMEOUT": 2, "KEEP_ALIVE": True}
|
||||
|
||||
old_conn = None
|
||||
|
||||
class ReuseableTCPConnector(TCPConnector):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ReuseableTCPConnector, self).__init__(*args, **kwargs)
|
||||
self.old_proto = None
|
||||
|
||||
async def connect(self, req, *args, **kwargs):
|
||||
new_conn = await super(ReuseableTCPConnector, self).connect(
|
||||
req, *args, **kwargs
|
||||
)
|
||||
if self.old_proto is not None:
|
||||
if self.old_proto != new_conn._protocol:
|
||||
class ReusableSanicConnectionPool(httpcore.ConnectionPool):
|
||||
async def acquire_connection(self, url, ssl, timeout):
|
||||
global old_conn
|
||||
if timeout.connect_timeout and not timeout.pool_timeout:
|
||||
timeout.pool_timeout = timeout.connect_timeout
|
||||
key = (url.scheme, url.hostname, url.port, ssl, timeout)
|
||||
try:
|
||||
connection = self._keepalive_connections[key].pop()
|
||||
if not self._keepalive_connections[key]:
|
||||
del self._keepalive_connections[key]
|
||||
self.num_keepalive_connections -= 1
|
||||
self.num_active_connections += 1
|
||||
|
||||
except (KeyError, IndexError):
|
||||
ssl_context = await self.get_ssl_context(url, ssl)
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._max_connections.acquire(), timeout.pool_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise PoolTimeout()
|
||||
release = functools.partial(self.release_connection, key=key)
|
||||
connection = httpcore.connections.Connection(
|
||||
timeout=timeout, on_release=release
|
||||
)
|
||||
self.num_active_connections += 1
|
||||
await connection.open(url.hostname, url.port, ssl=ssl_context)
|
||||
|
||||
if old_conn is not None:
|
||||
if old_conn != connection:
|
||||
raise RuntimeError(
|
||||
"We got a new connection, wanted the same one!"
|
||||
)
|
||||
print(new_conn.__dict__)
|
||||
self.old_proto = new_conn._protocol
|
||||
return new_conn
|
||||
old_conn = connection
|
||||
|
||||
return connection
|
||||
|
||||
|
||||
class ReusableSanicAdapter(requests.adapters.HTTPAdapter):
|
||||
def __init__(self):
|
||||
self.pool = ReusableSanicConnectionPool()
|
||||
|
||||
async def send(
|
||||
self,
|
||||
request,
|
||||
stream=False,
|
||||
timeout=None,
|
||||
verify=True,
|
||||
cert=None,
|
||||
proxies=None,
|
||||
):
|
||||
|
||||
method = request.method
|
||||
url = request.url
|
||||
headers = [
|
||||
(_encode(k), _encode(v)) for k, v in request.headers.items()
|
||||
]
|
||||
|
||||
if not request.body:
|
||||
body = b""
|
||||
elif isinstance(request.body, str):
|
||||
body = _encode(request.body)
|
||||
else:
|
||||
body = request.body
|
||||
|
||||
if isinstance(timeout, tuple):
|
||||
timeout_kwargs = {
|
||||
"connect_timeout": timeout[0],
|
||||
"read_timeout": timeout[1],
|
||||
}
|
||||
else:
|
||||
timeout_kwargs = {
|
||||
"connect_timeout": timeout,
|
||||
"read_timeout": timeout,
|
||||
}
|
||||
|
||||
ssl = httpcore.SSLConfig(cert=cert, verify=verify)
|
||||
timeout = httpcore.TimeoutConfig(**timeout_kwargs)
|
||||
|
||||
try:
|
||||
response = await self.pool.request(
|
||||
method,
|
||||
url,
|
||||
headers=headers,
|
||||
body=body,
|
||||
stream=stream,
|
||||
ssl=ssl,
|
||||
timeout=timeout,
|
||||
)
|
||||
except (httpcore.BadResponse, socket.error) as err:
|
||||
raise ConnectionError(err, request=request)
|
||||
except httpcore.ConnectTimeout as err:
|
||||
raise requests.exceptions.ConnectTimeout(err, request=request)
|
||||
except httpcore.ReadTimeout as err:
|
||||
raise requests.exceptions.ReadTimeout(err, request=request)
|
||||
|
||||
return self.build_response(request, response)
|
||||
|
||||
|
||||
class ResusableSanicSession(requests.Session):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
adapter = ReusableSanicAdapter()
|
||||
self.mount("http://", adapter)
|
||||
self.mount("https://", adapter)
|
||||
|
||||
|
||||
class ReuseableSanicTestClient(SanicTestClient):
|
||||
def __init__(self, app, loop=None):
|
||||
super(ReuseableSanicTestClient, self).__init__(app)
|
||||
super().__init__(app)
|
||||
if loop is None:
|
||||
loop = asyncio.get_event_loop()
|
||||
self._loop = loop
|
||||
@@ -51,12 +155,11 @@ class ReuseableSanicTestClient(SanicTestClient):
|
||||
debug=False,
|
||||
server_kwargs={"return_asyncio_server": True},
|
||||
*request_args,
|
||||
**request_kwargs
|
||||
**request_kwargs,
|
||||
):
|
||||
loop = self._loop
|
||||
results = [None, None]
|
||||
exceptions = []
|
||||
do_kill_server = request_kwargs.pop("end_server", False)
|
||||
if gather_request:
|
||||
|
||||
def _collect_request(request):
|
||||
@@ -65,21 +168,27 @@ class ReuseableSanicTestClient(SanicTestClient):
|
||||
|
||||
self.app.request_middleware.appendleft(_collect_request)
|
||||
|
||||
if uri.startswith(
|
||||
("http:", "https:", "ftp:", "ftps://", "//", "ws:", "wss:")
|
||||
):
|
||||
url = uri
|
||||
else:
|
||||
uri = uri if uri.startswith("/") else "/{uri}".format(uri=uri)
|
||||
scheme = "http"
|
||||
url = "{scheme}://{host}:{port}{uri}".format(
|
||||
scheme=scheme, host=HOST, port=PORT, uri=uri
|
||||
)
|
||||
|
||||
@self.app.listener("after_server_start")
|
||||
async def _collect_response(loop):
|
||||
try:
|
||||
if do_kill_server:
|
||||
request_kwargs["end_session"] = True
|
||||
response = await self._local_request(
|
||||
method, uri, *request_args, **request_kwargs
|
||||
method, url, *request_args, **request_kwargs
|
||||
)
|
||||
results[-1] = response
|
||||
except Exception as e2:
|
||||
import traceback
|
||||
|
||||
traceback.print_tb(e2.__traceback__)
|
||||
# traceback.print_tb(e2.__traceback__)
|
||||
exceptions.append(e2)
|
||||
# Don't stop here! self.app.stop()
|
||||
|
||||
if self._server is not None:
|
||||
_server = self._server
|
||||
@@ -94,27 +203,14 @@ class ReuseableSanicTestClient(SanicTestClient):
|
||||
|
||||
try:
|
||||
loop._stopping = False
|
||||
http_server = loop.run_until_complete(_server_co)
|
||||
_server = loop.run_until_complete(_server_co)
|
||||
except Exception as e1:
|
||||
import traceback
|
||||
|
||||
traceback.print_tb(e1.__traceback__)
|
||||
# traceback.print_tb(e1.__traceback__)
|
||||
raise e1
|
||||
self._server = _server = http_server
|
||||
self._server = _server
|
||||
server.trigger_events(self.app.listeners["after_server_start"], loop)
|
||||
self.app.listeners["after_server_start"].pop()
|
||||
|
||||
if do_kill_server:
|
||||
try:
|
||||
_server.close()
|
||||
self._server = None
|
||||
loop.run_until_complete(_server.wait_closed())
|
||||
self.app.stop()
|
||||
except Exception as e3:
|
||||
import traceback
|
||||
|
||||
traceback.print_tb(e3.__traceback__)
|
||||
exceptions.append(e3)
|
||||
if exceptions:
|
||||
raise ValueError("Exception during request: {}".format(exceptions))
|
||||
|
||||
@@ -137,59 +233,61 @@ class ReuseableSanicTestClient(SanicTestClient):
|
||||
"Request object expected, got ({})".format(results)
|
||||
)
|
||||
|
||||
def kill_server(self):
|
||||
try:
|
||||
if self._server:
|
||||
self._server.close()
|
||||
self._loop.run_until_complete(self._server.wait_closed())
|
||||
self._server = None
|
||||
self.app.stop()
|
||||
|
||||
if self._session:
|
||||
self._loop.run_until_complete(self._session.close())
|
||||
self._session = None
|
||||
|
||||
except Exception as e3:
|
||||
raise e3
|
||||
|
||||
# Copied from SanicTestClient, but with some changes to reuse the
|
||||
# same TCPConnection and the sane ClientSession more than once.
|
||||
# Note, you cannot use the same session if you are in a _different_
|
||||
# loop, so the changes above are required too.
|
||||
async def _local_request(self, method, uri, cookies=None, *args, **kwargs):
|
||||
async def _local_request(self, method, url, *args, **kwargs):
|
||||
raw_cookies = kwargs.pop("raw_cookies", None)
|
||||
request_keepalive = kwargs.pop(
|
||||
"request_keepalive", CONFIG_FOR_TESTS["KEEP_ALIVE_TIMEOUT"]
|
||||
)
|
||||
if uri.startswith(("http:", "https:", "ftp:", "ftps://" "//")):
|
||||
url = uri
|
||||
else:
|
||||
url = "http://{host}:{port}{uri}".format(
|
||||
host=HOST, port=self.port, uri=uri
|
||||
)
|
||||
do_kill_session = kwargs.pop("end_session", False)
|
||||
if self._session:
|
||||
session = self._session
|
||||
_session = self._session
|
||||
else:
|
||||
if self._tcp_connector:
|
||||
conn = self._tcp_connector
|
||||
else:
|
||||
conn = ReuseableTCPConnector(
|
||||
ssl=False,
|
||||
loop=self._loop,
|
||||
keepalive_timeout=request_keepalive,
|
||||
_session = ResusableSanicSession()
|
||||
self._session = _session
|
||||
async with _session as session:
|
||||
try:
|
||||
response = await getattr(session, method.lower())(
|
||||
url,
|
||||
verify=False,
|
||||
timeout=request_keepalive,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
self._tcp_connector = conn
|
||||
session = aiohttp.ClientSession(
|
||||
cookies=cookies, connector=conn, loop=self._loop
|
||||
)
|
||||
self._session = session
|
||||
|
||||
async with getattr(session, method.lower())(
|
||||
url, *args, **kwargs
|
||||
) as response:
|
||||
try:
|
||||
response.text = await response.text()
|
||||
except UnicodeDecodeError:
|
||||
response.text = None
|
||||
except NameError:
|
||||
raise Exception(response.status_code)
|
||||
|
||||
try:
|
||||
response.json = await response.json()
|
||||
except (
|
||||
JSONDecodeError,
|
||||
UnicodeDecodeError,
|
||||
aiohttp.ClientResponseError,
|
||||
):
|
||||
response.json = response.json()
|
||||
except (JSONDecodeError, UnicodeDecodeError):
|
||||
response.json = None
|
||||
|
||||
response.body = await response.read()
|
||||
if do_kill_session:
|
||||
await session.close()
|
||||
self._session = None
|
||||
response.status = response.status_code
|
||||
response.content_type = response.headers.get("content-type")
|
||||
|
||||
if raw_cookies:
|
||||
response.raw_cookies = {}
|
||||
for cookie in response.cookies:
|
||||
response.raw_cookies[cookie.name] = cookie
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@@ -229,9 +327,10 @@ def test_keep_alive_timeout_reuse():
|
||||
assert response.status == 200
|
||||
assert response.text == "OK"
|
||||
loop.run_until_complete(aio_sleep(1))
|
||||
request, response = client.get("/1", end_server=True)
|
||||
request, response = client.get("/1")
|
||||
assert response.status == 200
|
||||
assert response.text == "OK"
|
||||
client.kill_server()
|
||||
|
||||
|
||||
def test_keep_alive_client_timeout():
|
||||
@@ -241,20 +340,21 @@ def test_keep_alive_client_timeout():
|
||||
asyncio.set_event_loop(loop)
|
||||
client = ReuseableSanicTestClient(keep_alive_app_client_timeout, loop)
|
||||
headers = {"Connection": "keep-alive"}
|
||||
request, response = client.get("/1", headers=headers, request_keepalive=1)
|
||||
assert response.status == 200
|
||||
assert response.text == "OK"
|
||||
loop.run_until_complete(aio_sleep(2))
|
||||
exception = None
|
||||
try:
|
||||
request, response = client.get(
|
||||
"/1", end_server=True, request_keepalive=1
|
||||
"/1", headers=headers, request_keepalive=1
|
||||
)
|
||||
assert response.status == 200
|
||||
assert response.text == "OK"
|
||||
loop.run_until_complete(aio_sleep(2))
|
||||
exception = None
|
||||
request, response = client.get("/1", request_keepalive=1)
|
||||
except ValueError as e:
|
||||
exception = e
|
||||
assert exception is not None
|
||||
assert isinstance(exception, ValueError)
|
||||
assert "got a new connection" in exception.args[0]
|
||||
client.kill_server()
|
||||
|
||||
|
||||
def test_keep_alive_server_timeout():
|
||||
@@ -266,15 +366,15 @@ def test_keep_alive_server_timeout():
|
||||
asyncio.set_event_loop(loop)
|
||||
client = ReuseableSanicTestClient(keep_alive_app_server_timeout, loop)
|
||||
headers = {"Connection": "keep-alive"}
|
||||
request, response = client.get("/1", headers=headers, request_keepalive=60)
|
||||
assert response.status == 200
|
||||
assert response.text == "OK"
|
||||
loop.run_until_complete(aio_sleep(3))
|
||||
exception = None
|
||||
try:
|
||||
request, response = client.get(
|
||||
"/1", request_keepalive=60, end_server=True
|
||||
"/1", headers=headers, request_keepalive=60
|
||||
)
|
||||
assert response.status == 200
|
||||
assert response.text == "OK"
|
||||
loop.run_until_complete(aio_sleep(3))
|
||||
exception = None
|
||||
request, response = client.get("/1", request_keepalive=60)
|
||||
except ValueError as e:
|
||||
exception = e
|
||||
assert exception is not None
|
||||
@@ -283,3 +383,4 @@ def test_keep_alive_server_timeout():
|
||||
"Connection reset" in exception.args[0]
|
||||
or "got a new connection" in exception.args[0]
|
||||
)
|
||||
client.kill_server()
|
||||
|
||||
Reference in New Issue
Block a user