Vasanko/sanic
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Swap out requests-async for httpx (#1728)

Browse Source 
* Begin swap of requests-async for httpx

* Finalize httpx adoption and resolve tests

Resolve linting and formatting

* Remove documentation references to requests-async in favor of httpx
This commit is contained in:
Adam Hopkins 2019-12-21 05:23:52 +02:00 committed by 7
parent a6077a1790
commit 3f6a978328
13 changed files with 171 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/sanic/testing.rst
View File

@ -2,7 +2,7 @@ Testing
=======
Sanic endpoints can be tested locally using the `test_client` object, which
depends on the additional `requests-async <https://github.com/encode/requests-async>`_
depends on an additional package: `httpx <https://www.encode.io/httpx/>`_
library, which implements an API that mirrors the `requests` library.
The `test_client` exposes `get`, `post`, `put`, `delete`, `patch`, `head` and `options` methods
@ -22,7 +22,7 @@ for you to run against your application. A simple example (using pytest) is like
assert response.status == 405
Internally, each time you call one of the `test_client` methods, the Sanic app is run at `127.0.0.1:42101` and
your test request is executed against your application, using `requests-async`.
your test request is executed against your application, using `httpx`.
The `test_client` methods accept the following arguments and keyword arguments:
@ -55,8 +55,8 @@ And to supply data to a JSON POST request:
assert request.json.get('key1') == 'value1'
More information about
the available arguments to `requests-async` can be found
[in the documentation for `requests <https://2.python-requests.org/en/master/>`_.
the available arguments to `httpx` can be found
[in the documentation for `httpx <https://www.encode.io/httpx/>`_.
Using a random port

2
environment.yml
View File

@ -13,7 +13,7 @@ dependencies:
- sphinx==1.8.3
- sphinx_rtd_theme==0.4.2
- recommonmark==0.5.0
- requests-async==0.5.0
- httpx==0.9.3
- sphinxcontrib-asyncio>=0.2.0
- docutils==0.14
- pygments==2.3.1

13
sanic/asgi.py
View File

@ -15,8 +15,6 @@ from typing import (
)
from urllib.parse import quote
from requests_async import ASGISession # type: ignore
import sanic.app # noqa
from sanic.compat import Header
@ -189,7 +187,7 @@ class Lifespan:
class ASGIApp:
sanic_app: Union[ASGISession, "sanic.app.Sanic"]
sanic_app: "sanic.app.Sanic"
request: Request
transport: MockTransport
do_stream: bool
@ -223,8 +221,13 @@ class ASGIApp:
if scope["type"] == "lifespan":
await instance.lifespan(scope, receive, send)
else:
url_bytes = scope.get("root_path", "") + quote(scope["path"])
url_bytes = url_bytes.encode("latin-1")
path = (
scope["path"][1:]
if scope["path"].startswith("/")
else scope["path"]
)
url = "/".join([scope.get("root_path", ""), quote(path)])
url_bytes = url.encode("latin-1")
url_bytes += b"?" + scope["query_string"]
if scope["type"] == "http":

274
sanic/testing.py
View File

@ -1,14 +1,8 @@
import asyncio
import types
import typing
from json import JSONDecodeError
from socket import socket
from urllib.parse import unquote, urlsplit
import httpcore # type: ignore
import requests_async as requests # type: ignore
import websockets # type: ignore
import httpx
import websockets
from sanic.asgi import ASGIApp
from sanic.exceptions import MethodNotSupported
@ -29,7 +23,7 @@ class SanicTestClient:
self.host = host
def get_new_session(self):
return requests.Session()
return httpx.Client()
async def _local_request(self, method, url, *args, **kwargs):
logger.info(url)
@ -60,7 +54,8 @@ class SanicTestClient:
if raw_cookies:
response.raw_cookies = {}
for cookie in response.cookies:
for cookie in response.cookies.jar:
response.raw_cookies[cookie.name] = cookie
return response
@ -179,181 +174,6 @@ class SanicTestClient:
return self._sanic_endpoint_test("websocket", *args, **kwargs)
class SanicASGIAdapter(requests.asgi.ASGIAdapter): # noqa
async def send( # type: ignore
self,
request: requests.PreparedRequest,
gather_return: bool = False,
*args: typing.Any,
**kwargs: typing.Any,
) -> requests.Response:
"""This method is taken MOSTLY verbatim from requests-asyn. The
difference is the capturing of a response on the ASGI call and then
returning it on the response object. This is implemented to achieve:
request, response = await app.asgi_client.get("/")
You can see the original code here:
https://github.com/encode/requests-async/blob/614f40f77f19e6c6da8a212ae799107b0384dbf9/requests_async/asgi.py#L51""" # noqa
scheme, netloc, path, query, fragment = urlsplit(
request.url
) # type: ignore
default_port = {"http": 80, "ws": 80, "https": 443, "wss": 443}[scheme]
if ":" in netloc:
host, port_string = netloc.split(":", 1)
port = int(port_string)
else:
host = netloc
port = default_port
# Include the 'host' header.
if "host" in request.headers:
headers = [] # type: typing.List[typing.Tuple[bytes, bytes]]
elif port == default_port:
headers = [(b"host", host.encode())]
else:
headers = [(b"host", (f"{host}:{port}").encode())]
# Include other request headers.
headers += [
(key.lower().encode(), value.encode())
for key, value in request.headers.items()
]
no_response = False
if scheme in {"ws", "wss"}:
subprotocol = request.headers.get("sec-websocket-protocol", None)
if subprotocol is None:
subprotocols = [] # type: typing.Sequence[str]
else:
subprotocols = [
value.strip() for value in subprotocol.split(",")
]
scope = {
"type": "websocket",
"path": unquote(path),
"root_path": "",
"scheme": scheme,
"query_string": query.encode(),
"headers": headers,
"client": ["testclient", 50000],
"server": [host, port],
"subprotocols": subprotocols,
}
no_response = True
else:
scope = {
"type": "http",
"http_version": "1.1",
"method": request.method,
"path": unquote(path),
"root_path": "",
"scheme": scheme,
"query_string": query.encode(),
"headers": headers,
"client": ["testclient", 50000],
"server": [host, port],
"extensions": {"http.response.template": {}},
}
async def receive():
nonlocal request_complete, response_complete
if request_complete:
while not response_complete:
await asyncio.sleep(0.0001)
return {"type": "http.disconnect"}
body = request.body
if isinstance(body, str):
body_bytes = body.encode("utf-8") # type: bytes
elif body is None:
body_bytes = b""
elif isinstance(body, types.GeneratorType):
try:
chunk = body.send(None)
if isinstance(chunk, str):
chunk = chunk.encode("utf-8")
return {
"type": "http.request",
"body": chunk,
"more_body": True,
}
except StopIteration:
request_complete = True
return {"type": "http.request", "body": b""}
else:
body_bytes = body
request_complete = True
return {"type": "http.request", "body": body_bytes}
request_complete = False
response_started = False
response_complete = False
raw_kwargs = {"content": b""} # type: typing.Dict[str, typing.Any]
template = None
context = None
return_value = None
async def send(message) -> None:
nonlocal raw_kwargs, response_started, response_complete, template, context # noqa
if message["type"] == "http.response.start":
assert (
not response_started
), 'Received multiple "http.response.start" messages.'
raw_kwargs["status_code"] = message["status"]
raw_kwargs["headers"] = message["headers"]
response_started = True
elif message["type"] == "http.response.body":
assert response_started, (
'Received "http.response.body" '
'without "http.response.start".'
)
assert (
not response_complete
), 'Received "http.response.body" after response completed.'
body = message.get("body", b"")
more_body = message.get("more_body", False)
if request.method != "HEAD":
raw_kwargs["content"] += body
if not more_body:
response_complete = True
elif message["type"] == "http.response.template":
template = message["template"]
context = message["context"]
try:
return_value = await self.app(scope, receive, send)
except BaseException as exc:
if not self.suppress_exceptions:
raise exc from None
if no_response:
response_started = True
raw_kwargs = {"status_code": 204, "headers": []}
if not self.suppress_exceptions:
assert response_started, "TestClient did not receive any response."
elif not response_started:
raw_kwargs = {"status_code": 500, "headers": []}
raw = httpcore.Response(**raw_kwargs)
response = self.build_response(request, raw)
if template is not None:
response.template = template
response.context = context
if gather_return:
response.return_value = return_value
return response
class TestASGIApp(ASGIApp):
async def __call__(self):
await super().__call__()
@ -365,7 +185,11 @@ async def app_call_with_return(self, scope, receive, send):
return await asgi_app()
class SanicASGITestClient(requests.ASGISession):
class SanicASGIDispatch(httpx.dispatch.ASGIDispatch):
pass
class SanicASGITestClient(httpx.Client):
def __init__(
self,
app,
@ -374,18 +198,18 @@ class SanicASGITestClient(requests.ASGISession):
) -> None:
app.__class__.__call__ = app_call_with_return
app.asgi = True
super().__init__(app)
adapter = SanicASGIAdapter(
app, suppress_exceptions=suppress_exceptions
)
self.mount("http://", adapter)
self.mount("https://", adapter)
self.mount("ws://", adapter)
self.mount("wss://", adapter)
self.headers.update({"user-agent": "testclient"})
self.app = app
self.base_url = base_url
dispatch = SanicASGIDispatch(app=app, client=(ASGI_HOST, PORT))
super().__init__(dispatch=dispatch, base_url=base_url)
self.last_request = None
def _collect_request(request):
self.last_request = request
app.request_middleware.appendleft(_collect_request)
async def request(self, method, url, gather_request=True, *args, **kwargs):
@ -395,33 +219,39 @@ class SanicASGITestClient(requests.ASGISession):
response.body = response.content
response.content_type = response.headers.get("content-type")
if hasattr(response, "return_value"):
request = response.return_value
del response.return_value
return request, response
return response
def merge_environment_settings(self, *args, **kwargs):
settings = super().merge_environment_settings(*args, **kwargs)
settings.update({"gather_return": self.gather_request})
return settings
return self.last_request, response
async def websocket(self, uri, subprotocols=None, *args, **kwargs):
if uri.startswith(("ws:", "wss:")):
url = uri
else:
uri = uri if uri.startswith("/") else "/{uri}".format(uri=uri)
url = "ws://testserver{uri}".format(uri=uri)
scheme = "ws"
path = uri
root_path = "{}://{}".format(scheme, ASGI_HOST)
headers = kwargs.get("headers", {})
headers.setdefault("connection", "upgrade")
headers.setdefault("sec-websocket-key", "testserver==")
headers.setdefault("sec-websocket-version", "13")
if subprotocols is not None:
headers.setdefault(
"sec-websocket-protocol", ", ".join(subprotocols)
)
kwargs["headers"] = headers
headers = kwargs.get("headers", {})
headers.setdefault("connection", "upgrade")
headers.setdefault("sec-websocket-key", "testserver==")
headers.setdefault("sec-websocket-version", "13")
if subprotocols is not None:
headers.setdefault(
"sec-websocket-protocol", ", ".join(subprotocols)
)
return await self.request("websocket", url, **kwargs)
scope = {
"type": "websocket",
"asgi": {"version": "3.0"},
"http_version": "1.1",
"headers": [map(lambda y: y.encode(), x) for x in headers.items()],
"scheme": scheme,
"root_path": root_path,
"path": path,
"query_string": b"",
}
async def receive():
return {}
async def send(message):
pass
await self.app(scope, receive, send)
return None, {}

3
setup.py
View File

@ -5,6 +5,7 @@ import codecs
import os
import re
import sys
from distutils.util import strtobool
from setuptools import setup
@ -83,7 +84,7 @@ requirements = [
"aiofiles>=0.3.0",
"websockets>=7.0,<9.0",
"multidict>=4.0,<5.0",
"requests-async==0.5.0",
"httpx==0.9.3",
]
tests_require = [

27
tests/test_asgi.py
View File

@ -1,6 +1,6 @@
import asyncio
from collections import deque
from collections import deque, namedtuple
import pytest
import uvicorn
@ -245,17 +245,26 @@ async def test_cookie_customization(app):
return response
_, response = await app.asgi_client.get("/cookie")
CookieDef = namedtuple("CookieDef", ("value", "httponly"))
Cookie = namedtuple("Cookie", ("domain", "path", "value", "httponly"))
cookie_map = {
"test": {"value": "Cookie1", "HttpOnly": True},
"c2": {"value": "Cookie2", "HttpOnly": False},
"test": CookieDef("Cookie1", True),
"c2": CookieDef("Cookie2", False),
}
for k, v in (
response.cookies._cookies.get("mockserver.local").get("/").items()
):
assert cookie_map.get(k).get("value") == v.value
if cookie_map.get(k).get("HttpOnly"):
assert "HttpOnly" in v._rest.keys()
cookies = {
c.name: Cookie(c.domain, c.path, c.value, "HttpOnly" in c._rest.keys())
for c in response.cookies.jar
}
for name, definition in cookie_map.items():
cookie = cookies.get(name)
assert cookie
assert cookie.value == definition.value
assert cookie.domain == "mockserver.local"
assert cookie.path == "/"
assert cookie.httponly == definition.httponly
@pytest.mark.asyncio

53
tests/test_keep_alive_timeout.py
View File

@ -1,15 +1,9 @@
import asyncio
import functools
import socket
from asyncio import sleep as aio_sleep
from http.client import _encode
from json import JSONDecodeError
import httpcore
import requests_async as requests
from httpcore import PoolTimeout
import httpx
from sanic import Sanic, server
from sanic.response import text
@ -21,24 +15,28 @@ CONFIG_FOR_TESTS = {"KEEP_ALIVE_TIMEOUT": 2, "KEEP_ALIVE": True}
old_conn = None
class ReusableSanicConnectionPool(httpcore.ConnectionPool):
async def acquire_connection(self, origin):
class ReusableSanicConnectionPool(
httpx.dispatch.connection_pool.ConnectionPool
):
async def acquire_connection(self, origin, timeout):
global old_conn
connection = self.active_connections.pop_by_origin(
origin, http2_only=True
)
if connection is None:
connection = self.keepalive_connections.pop_by_origin(origin)
connection = self.pop_connection(origin)
if connection is None:
await self.max_connections.acquire()
connection = httpcore.HTTPConnection(
pool_timeout = None if timeout is None else timeout.pool_timeout
await self.max_connections.acquire(timeout=pool_timeout)
connection = httpx.dispatch.connection.HTTPConnection(
origin,
ssl=self.ssl,
timeout=self.timeout,
verify=self.verify,
cert=self.cert,
http2=self.http2,
backend=self.backend,
release_func=self.release_connection,
trust_env=self.trust_env,
uds=self.uds,
)
self.active_connections.add(connection)
if old_conn is not None:
@ -51,17 +49,10 @@ class ReusableSanicConnectionPool(httpcore.ConnectionPool):
return connection
class ReusableSanicAdapter(requests.adapters.HTTPAdapter):
def __init__(self):
self.pool = ReusableSanicConnectionPool()
class ResusableSanicSession(requests.Session):
class ResusableSanicSession(httpx.Client):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
adapter = ReusableSanicAdapter()
self.mount("http://", adapter)
self.mount("https://", adapter)
dispatch = ReusableSanicConnectionPool()
super().__init__(dispatch=dispatch, *args, **kwargs)
class ReuseableSanicTestClient(SanicTestClient):
@ -74,6 +65,9 @@ class ReuseableSanicTestClient(SanicTestClient):
self._tcp_connector = None
self._session = None
def get_new_session(self):
return ResusableSanicSession()
# Copied from SanicTestClient, but with some changes to reuse the
# same loop for the same app.
def _sanic_endpoint_test(
@ -167,7 +161,6 @@ class ReuseableSanicTestClient(SanicTestClient):
self._server.close()
self._loop.run_until_complete(self._server.wait_closed())
self._server = None
self.app.stop()
if self._session:
self._loop.run_until_complete(self._session.close())
@ -186,7 +179,7 @@ class ReuseableSanicTestClient(SanicTestClient):
"request_keepalive", CONFIG_FOR_TESTS["KEEP_ALIVE_TIMEOUT"]
)
if not self._session:
self._session = ResusableSanicSession()
self._session = self.get_new_session()
try:
response = await getattr(self._session, method.lower())(
url, verify=False, timeout=request_keepalive, *args, **kwargs

1
tests/test_request_buffer_queue_size.py
View File

@ -2,6 +2,7 @@ import io
from sanic.response import text
data = "abc" * 10_000_000

2
tests/test_request_stream.py
View File

@ -332,7 +332,7 @@ def test_request_stream_handle_exception(app):
assert response.text == "Error: Requested URL /in_valid_post not found"
# 405
request, response = app.test_client.get("/post/random_id", data=data)
request, response = app.test_client.get("/post/random_id")
assert response.status == 405
assert (
response.text == "Error: Method GET not allowed for URL"

81
tests/test_request_timeout.py
View File

@ -1,49 +1,70 @@
import asyncio
import httpcore
import requests_async as requests
import httpx
from sanic import Sanic
from sanic.response import text
from sanic.testing import SanicTestClient
class DelayableSanicConnectionPool(httpcore.ConnectionPool):
class DelayableHTTPConnection(httpx.dispatch.connection.HTTPConnection):
def __init__(self, *args, **kwargs):
self._request_delay = None
if "request_delay" in kwargs:
self._request_delay = kwargs.pop("request_delay")
super().__init__(*args, **kwargs)
async def send(self, request, verify=None, cert=None, timeout=None):
if self.h11_connection is None and self.h2_connection is None:
await self.connect(verify=verify, cert=cert, timeout=timeout)
if self._request_delay:
await asyncio.sleep(self._request_delay)
if self.h2_connection is not None:
response = await self.h2_connection.send(request, timeout=timeout)
else:
assert self.h11_connection is not None
response = await self.h11_connection.send(request, timeout=timeout)
return response
class DelayableSanicConnectionPool(
httpx.dispatch.connection_pool.ConnectionPool
):
def __init__(self, request_delay=None, *args, **kwargs):
self._request_delay = request_delay
super().__init__(*args, **kwargs)
async def send(self, request, stream=False, ssl=None, timeout=None):
connection = await self.acquire_connection(request.url.origin)
if (
connection.h11_connection is None
and connection.h2_connection is None
):
await connection.connect(ssl=ssl, timeout=timeout)
if self._request_delay:
await asyncio.sleep(self._request_delay)
try:
response = await connection.send(
request, stream=stream, ssl=ssl, timeout=timeout
async def acquire_connection(self, origin, timeout=None):
connection = self.pop_connection(origin)
if connection is None:
pool_timeout = None if timeout is None else timeout.pool_timeout
await self.max_connections.acquire(timeout=pool_timeout)
connection = DelayableHTTPConnection(
origin,
verify=self.verify,
cert=self.cert,
http2=self.http2,
backend=self.backend,
release_func=self.release_connection,
trust_env=self.trust_env,
uds=self.uds,
request_delay=self._request_delay,
)
except BaseException as exc:
self.active_connections.remove(connection)
self.max_connections.release()
raise exc
return response
self.active_connections.add(connection)
return connection
class DelayableSanicAdapter(requests.adapters.HTTPAdapter):
def __init__(self, request_delay=None):
self.pool = DelayableSanicConnectionPool(request_delay=request_delay)
class DelayableSanicSession(requests.Session):
class DelayableSanicSession(httpx.Client):
def __init__(self, request_delay=None, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
adapter = DelayableSanicAdapter(request_delay=request_delay)
self.mount("http://", adapter)
self.mount("https://", adapter)
dispatch = DelayableSanicConnectionPool(request_delay=request_delay)
super().__init__(dispatch=dispatch, *args, **kwargs)
class DelayableSanicTestClient(SanicTestClient):

34
tests/test_requests.py
View File

@ -10,7 +10,7 @@ import pytest
from sanic import Blueprint, Sanic
from sanic.exceptions import ServerError
from sanic.request import DEFAULT_HTTP_CONTENT_TYPE, RequestParameters
from sanic.request import DEFAULT_HTTP_CONTENT_TYPE, Request, RequestParameters
from sanic.response import json, text
from sanic.testing import ASGI_HOST, HOST, PORT
@ -55,11 +55,11 @@ def test_ip(app):
async def test_ip_asgi(app):
@app.route("/")
def handler(request):
return text("{}".format(request.ip))
return text("{}".format(request.url))
request, response = await app.asgi_client.get("/")
assert response.text == "mockserver"
assert response.text == "http://mockserver/"
def test_text(app):
@ -207,24 +207,24 @@ async def test_empty_json_asgi(app):
def test_invalid_json(app):
@app.route("/")
@app.post("/")
async def handler(request):
return json(request.json)
data = "I am not json"
request, response = app.test_client.get("/", data=data)
request, response = app.test_client.post("/", data=data)
assert response.status == 400
@pytest.mark.asyncio
async def test_invalid_json_asgi(app):
@app.route("/")
@app.post("/")
async def handler(request):
return json(request.json)
data = "I am not json"
request, response = await app.asgi_client.get("/", data=data)
request, response = await app.asgi_client.post("/", data=data)
assert response.status == 400
@ -1807,26 +1807,6 @@ def test_request_port(app):
assert hasattr(request, "_port")
@pytest.mark.asyncio
async def test_request_port_asgi(app):
@app.get("/")
def handler(request):
return text("OK")
request, response = await app.asgi_client.get("/")
port = request.port
assert isinstance(port, int)
delattr(request, "_socket")
delattr(request, "_port")
port = request.port
assert isinstance(port, int)
assert hasattr(request, "_socket")
assert hasattr(request, "_port")
def test_request_socket(app):
@app.get("/")
def handler(request):

4
tests/test_utf8.py
View File

@ -37,14 +37,14 @@ def skip_test_utf8_route(app):
def test_utf8_post_json(app):
@app.route("/")
@app.post("/")
async def handler(request):
return text("OK")
payload = {"test": "✓"}
headers = {"content-type": "application/json"}
request, response = app.test_client.get(
request, response = app.test_client.post(
"/", data=json_dumps(payload), headers=headers
)

2
tox.ini
View File

@ -13,7 +13,7 @@ deps =
pytest-sanic
pytest-sugar
httpcore==0.3.0
requests-async==0.5.0
httpx==0.9.3
chardet<=2.3.0
beautifulsoup4
gunicorn