From eb8df1fc183910c7ff4ad96b70d14fb8bfb412ce Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 27 Sep 2020 02:58:36 +0300 Subject: [PATCH] Upgrade httpx --- sanic/response.py | 5 +- sanic/testing.py | 17 ++--- setup.py | 2 +- tests/test_app.py | 2 +- tests/test_keep_alive_timeout.py | 112 +++++++++---------------------- tests/test_request_stream.py | 5 +- tests/test_request_timeout.py | 68 ++++++++----------- tests/test_requests.py | 24 +++++-- tests/test_unix_socket.py | 7 +- tox.ini | 2 +- 10 files changed, 94 insertions(+), 150 deletions(-) diff --git a/sanic/response.py b/sanic/response.py index 24033336..d4dd39c1 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -249,10 +249,7 @@ def raw( :param content_type: the content type (string) of the response. """ return HTTPResponse( - body=body, - status=status, - headers=headers, - content_type=content_type, + body=body, status=status, headers=headers, content_type=content_type, ) diff --git a/sanic/testing.py b/sanic/testing.py index 020b3c11..1d13970b 100644 --- a/sanic/testing.py +++ b/sanic/testing.py @@ -11,6 +11,8 @@ from sanic.response import text ASGI_HOST = "mockserver" +ASGI_PORT = 1234 +ASGI_BASE_URL = f"http://{ASGI_HOST}:{ASGI_PORT}" HOST = "127.0.0.1" PORT = None @@ -103,9 +105,7 @@ class SanicTestClient: if self.port: server_kwargs = dict( - host=host or self.host, - port=self.port, - **server_kwargs, + host=host or self.host, port=self.port, **server_kwargs, ) host, port = host or self.host, self.port else: @@ -195,24 +195,19 @@ async def app_call_with_return(self, scope, receive, send): return await asgi_app() -class SanicASGIDispatch(httpx.ASGIDispatch): - pass - - class SanicASGITestClient(httpx.AsyncClient): def __init__( self, app, - base_url: str = f"http://{ASGI_HOST}", + base_url: str = ASGI_BASE_URL, suppress_exceptions: bool = False, ) -> None: app.__class__.__call__ = app_call_with_return app.asgi = True self.app = app - - dispatch = SanicASGIDispatch(app=app, client=(ASGI_HOST, PORT or 0)) - super().__init__(dispatch=dispatch, base_url=base_url) + transport = httpx.ASGITransport(app=app, client=(ASGI_HOST, ASGI_PORT)) + super().__init__(transport=transport, base_url=base_url) self.last_request = None diff --git a/setup.py b/setup.py index db788803..e4ff74ee 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ requirements = [ "aiofiles>=0.3.0", "websockets>=8.1,<9.0", "multidict>=4.0,<5.0", - "httpx==0.11.1", + "httpx==0.15.4", ] tests_require = [ diff --git a/tests/test_app.py b/tests/test_app.py index 6e1d28ff..5a7afa59 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,9 +1,9 @@ import asyncio import logging import sys -from unittest.mock import patch from inspect import isawaitable +from unittest.mock import patch import pytest diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 4cc24f53..a6d5e2c6 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -3,6 +3,7 @@ import asyncio from asyncio import sleep as aio_sleep from json import JSONDecodeError +import httpcore import httpx from sanic import Sanic, server @@ -12,67 +13,26 @@ from sanic.testing import HOST, SanicTestClient CONFIG_FOR_TESTS = {"KEEP_ALIVE_TIMEOUT": 2, "KEEP_ALIVE": True} -old_conn = None PORT = 42101 # test_keep_alive_timeout_reuse doesn't work with random port +from httpcore._async.base import ConnectionState +from httpcore._async.connection import AsyncHTTPConnection +from httpcore._types import Origin -class ReusableSanicConnectionPool( - httpx.dispatch.connection_pool.ConnectionPool -): - @property - def cert(self): - return self.ssl.cert - @property - def verify(self): - return self.ssl.verify +class ReusableSanicConnectionPool(httpcore.AsyncConnectionPool): + last_reused_connection = None - @property - def trust_env(self): - return self.ssl.trust_env - - @property - def http2(self): - return self.ssl.http2 - - async def acquire_connection(self, origin, timeout): - global old_conn - connection = self.pop_connection(origin) - - if connection is None: - pool_timeout = None if timeout is None else timeout.pool_timeout - - await self.max_connections.acquire(timeout=pool_timeout) - ssl_config = httpx.config.SSLConfig( - cert=self.cert, - verify=self.verify, - trust_env=self.trust_env, - http2=self.http2, - ) - connection = httpx.dispatch.connection.HTTPConnection( - origin, - ssl=ssl_config, - backend=self.backend, - release_func=self.release_connection, - uds=self.uds, - ) - - self.active_connections.add(connection) - - if old_conn is not None: - if old_conn != connection: - raise RuntimeError( - "We got a new connection, wanted the same one!" - ) - old_conn = connection - - return connection + async def _get_connection_from_pool(self, *args, **kwargs): + conn = await super()._get_connection_from_pool(*args, **kwargs) + self.__class__.last_reused_connection = conn + return conn class ResusableSanicSession(httpx.AsyncClient): def __init__(self, *args, **kwargs) -> None: - dispatch = ReusableSanicConnectionPool() - super().__init__(dispatch=dispatch, *args, **kwargs) + transport = ReusableSanicConnectionPool() + super().__init__(transport=transport, *args, **kwargs) class ReuseableSanicTestClient(SanicTestClient): @@ -258,6 +218,7 @@ def test_keep_alive_timeout_reuse(): request, response = client.get("/1") assert response.status == 200 assert response.text == "OK" + assert ReusableSanicConnectionPool.last_reused_connection finally: client.kill_server() @@ -270,20 +231,15 @@ def test_keep_alive_client_timeout(): asyncio.set_event_loop(loop) client = ReuseableSanicTestClient(keep_alive_app_client_timeout, loop) headers = {"Connection": "keep-alive"} - try: - request, response = client.get( - "/1", headers=headers, request_keepalive=1 - ) - assert response.status == 200 - assert response.text == "OK" - loop.run_until_complete(aio_sleep(2)) - exception = None - request, response = client.get("/1", request_keepalive=1) - except ValueError as e: - exception = e - assert exception is not None - assert isinstance(exception, ValueError) - assert "got a new connection" in exception.args[0] + request, response = client.get( + "/1", headers=headers, request_keepalive=1 + ) + assert response.status == 200 + assert response.text == "OK" + loop.run_until_complete(aio_sleep(2)) + exception = None + request, response = client.get("/1", request_keepalive=1) + assert ReusableSanicConnectionPool.last_reused_connection is None finally: client.kill_server() @@ -298,22 +254,14 @@ def test_keep_alive_server_timeout(): asyncio.set_event_loop(loop) client = ReuseableSanicTestClient(keep_alive_app_server_timeout, loop) headers = {"Connection": "keep-alive"} - try: - request, response = client.get( - "/1", headers=headers, request_keepalive=60 - ) - assert response.status == 200 - assert response.text == "OK" - loop.run_until_complete(aio_sleep(3)) - exception = None - request, response = client.get("/1", request_keepalive=60) - except ValueError as e: - exception = e - assert exception is not None - assert isinstance(exception, ValueError) - assert ( - "Connection reset" in exception.args[0] - or "got a new connection" in exception.args[0] + request, response = client.get( + "/1", headers=headers, request_keepalive=60 ) + assert response.status == 200 + assert response.text == "OK" + loop.run_until_complete(aio_sleep(3)) + exception = None + request, response = client.get("/1", request_keepalive=60) + assert ReusableSanicConnectionPool.last_reused_connection is None finally: client.kill_server() diff --git a/tests/test_request_stream.py b/tests/test_request_stream.py index ff298868..d3354251 100644 --- a/tests/test_request_stream.py +++ b/tests/test_request_stream.py @@ -1,13 +1,14 @@ -import pytest import asyncio +import pytest + from sanic.blueprints import Blueprint from sanic.exceptions import HeaderExpectationFailed from sanic.request import StreamBuffer from sanic.response import json, stream, text +from sanic.server import HttpProtocol from sanic.views import CompositionView, HTTPMethodView from sanic.views import stream as stream_decorator -from sanic.server import HttpProtocol data = "abc" * 1_000_000 diff --git a/tests/test_request_timeout.py b/tests/test_request_timeout.py index 0b7d6405..d750dd1d 100644 --- a/tests/test_request_timeout.py +++ b/tests/test_request_timeout.py @@ -1,64 +1,54 @@ import asyncio +from typing import cast + +import httpcore import httpx +from httpcore._async.base import ( + AsyncByteStream, + AsyncHTTPTransport, + ConnectionState, + NewConnectionRequired, +) +from httpcore._async.connection import AsyncHTTPConnection +from httpcore._async.connection_pool import ResponseByteStream +from httpcore._exceptions import LocalProtocolError, UnsupportedProtocol +from httpcore._types import TimeoutDict +from httpcore._utils import url_to_origin + from sanic import Sanic from sanic.response import text from sanic.testing import SanicTestClient -class DelayableHTTPConnection(httpx.dispatch.connection.HTTPConnection): - def __init__(self, *args, **kwargs): - self._request_delay = None - if "request_delay" in kwargs: - self._request_delay = kwargs.pop("request_delay") - super().__init__(*args, **kwargs) - - async def send(self, request, timeout=None): - - if self.connection is None: - self.connection = await self.connect(timeout=timeout) +class DelayableHTTPConnection(httpcore._async.connection.AsyncHTTPConnection): + async def arequest(self, *args, **kwargs): + await asyncio.sleep(2) + return await super().arequest(*args, **kwargs) + async def _open_socket(self, *args, **kwargs): + retval = await super()._open_socket(*args, **kwargs) if self._request_delay: await asyncio.sleep(self._request_delay) - - response = await self.connection.send(request, timeout=timeout) - - return response + return retval -class DelayableSanicConnectionPool( - httpx.dispatch.connection_pool.ConnectionPool -): +class DelayableSanicConnectionPool(httpcore.AsyncConnectionPool): def __init__(self, request_delay=None, *args, **kwargs): self._request_delay = request_delay super().__init__(*args, **kwargs) - async def acquire_connection(self, origin, timeout=None): - connection = self.pop_connection(origin) - - if connection is None: - pool_timeout = None if timeout is None else timeout.pool_timeout - - await self.max_connections.acquire(timeout=pool_timeout) - connection = DelayableHTTPConnection( - origin, - ssl=self.ssl, - backend=self.backend, - release_func=self.release_connection, - uds=self.uds, - request_delay=self._request_delay, - ) - - self.active_connections.add(connection) - - return connection + async def _add_to_pool(self, connection, timeout): + connection.__class__ = DelayableHTTPConnection + connection._request_delay = self._request_delay + await super()._add_to_pool(connection, timeout) class DelayableSanicSession(httpx.AsyncClient): def __init__(self, request_delay=None, *args, **kwargs) -> None: - dispatch = DelayableSanicConnectionPool(request_delay=request_delay) - super().__init__(dispatch=dispatch, *args, **kwargs) + transport = DelayableSanicConnectionPool(request_delay=request_delay) + super().__init__(transport=transport, *args, **kwargs) class DelayableSanicTestClient(SanicTestClient): diff --git a/tests/test_requests.py b/tests/test_requests.py index 1ed94359..add05f4a 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -12,7 +12,14 @@ from sanic import Blueprint, Sanic from sanic.exceptions import ServerError from sanic.request import DEFAULT_HTTP_CONTENT_TYPE, Request, RequestParameters from sanic.response import html, json, text -from sanic.testing import ASGI_HOST, HOST, PORT, SanicTestClient +from sanic.testing import ( + ASGI_BASE_URL, + ASGI_HOST, + ASGI_PORT, + HOST, + PORT, + SanicTestClient, +) # ------------------------------------------------------------ # @@ -59,7 +66,10 @@ async def test_ip_asgi(app): request, response = await app.asgi_client.get("/") - assert response.text == "http://mockserver/" + if response.text.endswith("/") and not ASGI_BASE_URL.endswith("/"): + response.text[:-1] == ASGI_BASE_URL + else: + assert response.text == ASGI_BASE_URL def test_text(app): @@ -573,7 +583,7 @@ async def test_standard_forwarded_asgi(app): assert response.json() == {"for": "127.0.0.2", "proto": "ws"} assert request.remote_addr == "127.0.0.2" assert request.scheme == "ws" - assert request.server_port == 80 + assert request.server_port == ASGI_PORT app.config.FORWARDED_SECRET = "mySecret" request, response = await app.asgi_client.get("/", headers=headers) @@ -1044,9 +1054,9 @@ def test_url_attributes_no_ssl(app, path, query, expected_url): @pytest.mark.parametrize( "path,query,expected_url", [ - ("/foo", "", "http://{}/foo"), - ("/bar/baz", "", "http://{}/bar/baz"), - ("/moo/boo", "arg1=val1", "http://{}/moo/boo?arg1=val1"), + ("/foo", "", "{}/foo"), + ("/bar/baz", "", "{}/bar/baz"), + ("/moo/boo", "arg1=val1", "{}/moo/boo?arg1=val1"), ], ) @pytest.mark.asyncio @@ -1057,7 +1067,7 @@ async def test_url_attributes_no_ssl_asgi(app, path, query, expected_url): app.add_route(handler, path) request, response = await app.asgi_client.get(path + f"?{query}") - assert request.url == expected_url.format(ASGI_HOST) + assert request.url == expected_url.format(ASGI_BASE_URL) parsed = urlparse(request.url) diff --git a/tests/test_unix_socket.py b/tests/test_unix_socket.py index bbffc890..c592dcde 100644 --- a/tests/test_unix_socket.py +++ b/tests/test_unix_socket.py @@ -4,6 +4,7 @@ import os import subprocess import sys +import httpcore import httpx import pytest @@ -139,8 +140,9 @@ def test_unix_connection(): @app.listener("after_server_start") async def client(app, loop): + transport = httpcore.AsyncConnectionPool(uds=SOCKPATH) try: - async with httpx.AsyncClient(uds=SOCKPATH) as client: + async with httpx.AsyncClient(transport=transport) as client: r = await client.get("http://myhost.invalid/") assert r.status_code == 200 assert r.text == os.path.abspath(SOCKPATH) @@ -179,8 +181,9 @@ async def test_zero_downtime(): from time import monotonic as current_time async def client(): + transport = httpcore.AsyncConnectionPool(uds=SOCKPATH) for _ in range(40): - async with httpx.AsyncClient(uds=SOCKPATH) as client: + async with httpx.AsyncClient(transport=transport) as client: r = await client.get("http://localhost/sleep/0.1") assert r.status_code == 200 assert r.text == f"Slept 0.1 seconds.\n" diff --git a/tox.ini b/tox.ini index 09cbc0dd..69a8ce49 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = pytest-sanic pytest-sugar httpcore==0.3.0 - httpx==0.11.1 + httpx==0.15.4 chardet<=2.3.0 beautifulsoup4 gunicorn