Swap out requests-async for httpx (#1728)

* Begin swap of requests-async for httpx

* Finalize httpx adoption and resolve tests

Resolve linting and formatting

* Remove documentation references to requests-async in favor of httpx

tests/test_asgi.py

@@ -1,6 +1,6 @@
 import asyncio
 
-from collections import deque
+from collections import deque, namedtuple
 
 import pytest
 import uvicorn
@@ -245,17 +245,26 @@ async def test_cookie_customization(app):
         return response
 
     _, response = await app.asgi_client.get("/cookie")
+
+    CookieDef = namedtuple("CookieDef", ("value", "httponly"))
+    Cookie = namedtuple("Cookie", ("domain", "path", "value", "httponly"))
     cookie_map = {
-        "test": {"value": "Cookie1", "HttpOnly": True},
-        "c2": {"value": "Cookie2", "HttpOnly": False},
+        "test": CookieDef("Cookie1", True),
+        "c2": CookieDef("Cookie2", False),
     }
 
-    for k, v in (
-        response.cookies._cookies.get("mockserver.local").get("/").items()
-    ):
-        assert cookie_map.get(k).get("value") == v.value
-        if cookie_map.get(k).get("HttpOnly"):
-            assert "HttpOnly" in v._rest.keys()
+    cookies = {
+        c.name: Cookie(c.domain, c.path, c.value, "HttpOnly" in c._rest.keys())
+        for c in response.cookies.jar
+    }
+
+    for name, definition in cookie_map.items():
+        cookie = cookies.get(name)
+        assert cookie
+        assert cookie.value == definition.value
+        assert cookie.domain == "mockserver.local"
+        assert cookie.path == "/"
+        assert cookie.httponly == definition.httponly
 
 
 @pytest.mark.asyncio

tests/test_keep_alive_timeout.py

@@ -1,15 +1,9 @@
 import asyncio
-import functools
-import socket
 
 from asyncio import sleep as aio_sleep
-from http.client import _encode
 from json import JSONDecodeError
 
-import httpcore
-import requests_async as requests
-
-from httpcore import PoolTimeout
+import httpx
 
 from sanic import Sanic, server
 from sanic.response import text
@@ -21,24 +15,28 @@ CONFIG_FOR_TESTS = {"KEEP_ALIVE_TIMEOUT": 2, "KEEP_ALIVE": True}
 old_conn = None
 
 
-class ReusableSanicConnectionPool(httpcore.ConnectionPool):
-    async def acquire_connection(self, origin):
+class ReusableSanicConnectionPool(
+    httpx.dispatch.connection_pool.ConnectionPool
+):
+    async def acquire_connection(self, origin, timeout):
         global old_conn
-        connection = self.active_connections.pop_by_origin(
-            origin, http2_only=True
-        )
-        if connection is None:
-            connection = self.keepalive_connections.pop_by_origin(origin)
+        connection = self.pop_connection(origin)
 
         if connection is None:
-            await self.max_connections.acquire()
-            connection = httpcore.HTTPConnection(
+            pool_timeout = None if timeout is None else timeout.pool_timeout
+
+            await self.max_connections.acquire(timeout=pool_timeout)
+            connection = httpx.dispatch.connection.HTTPConnection(
                 origin,
-                ssl=self.ssl,
-                timeout=self.timeout,
+                verify=self.verify,
+                cert=self.cert,
+                http2=self.http2,
                 backend=self.backend,
                 release_func=self.release_connection,
+                trust_env=self.trust_env,
+                uds=self.uds,
             )
+
         self.active_connections.add(connection)
 
         if old_conn is not None:
@@ -51,17 +49,10 @@ class ReusableSanicConnectionPool(httpcore.ConnectionPool):
         return connection
 
 
-class ReusableSanicAdapter(requests.adapters.HTTPAdapter):
-    def __init__(self):
-        self.pool = ReusableSanicConnectionPool()
-
-
-class ResusableSanicSession(requests.Session):
+class ResusableSanicSession(httpx.Client):
     def __init__(self, *args, **kwargs) -> None:
-        super().__init__(*args, **kwargs)
-        adapter = ReusableSanicAdapter()
-        self.mount("http://", adapter)
-        self.mount("https://", adapter)
+        dispatch = ReusableSanicConnectionPool()
+        super().__init__(dispatch=dispatch, *args, **kwargs)
 
 
 class ReuseableSanicTestClient(SanicTestClient):
@@ -74,6 +65,9 @@ class ReuseableSanicTestClient(SanicTestClient):
         self._tcp_connector = None
         self._session = None
 
+    def get_new_session(self):
+        return ResusableSanicSession()
+
     # Copied from SanicTestClient, but with some changes to reuse the
     # same loop for the same app.
     def _sanic_endpoint_test(
@@ -167,7 +161,6 @@ class ReuseableSanicTestClient(SanicTestClient):
                 self._server.close()
                 self._loop.run_until_complete(self._server.wait_closed())
                 self._server = None
-                self.app.stop()
 
             if self._session:
                 self._loop.run_until_complete(self._session.close())
@@ -186,7 +179,7 @@ class ReuseableSanicTestClient(SanicTestClient):
             "request_keepalive", CONFIG_FOR_TESTS["KEEP_ALIVE_TIMEOUT"]
         )
         if not self._session:
-            self._session = ResusableSanicSession()
+            self._session = self.get_new_session()
         try:
             response = await getattr(self._session, method.lower())(
                 url, verify=False, timeout=request_keepalive, *args, **kwargs

tests/test_request_buffer_queue_size.py

@@ -2,6 +2,7 @@ import io
 
 from sanic.response import text
 
+
 data = "abc" * 10_000_000
 
 

tests/test_request_stream.py

@@ -332,7 +332,7 @@ def test_request_stream_handle_exception(app):
     assert response.text == "Error: Requested URL /in_valid_post not found"
 
     # 405
-    request, response = app.test_client.get("/post/random_id", data=data)
+    request, response = app.test_client.get("/post/random_id")
     assert response.status == 405
     assert (
         response.text == "Error: Method GET not allowed for URL"

tests/test_request_timeout.py

@@ -1,49 +1,70 @@
 import asyncio
 
-import httpcore
-import requests_async as requests
+import httpx
 
 from sanic import Sanic
 from sanic.response import text
 from sanic.testing import SanicTestClient
 
 
-class DelayableSanicConnectionPool(httpcore.ConnectionPool):
+class DelayableHTTPConnection(httpx.dispatch.connection.HTTPConnection):
+    def __init__(self, *args, **kwargs):
+        self._request_delay = None
+        if "request_delay" in kwargs:
+            self._request_delay = kwargs.pop("request_delay")
+        super().__init__(*args, **kwargs)
+
+    async def send(self, request, verify=None, cert=None, timeout=None):
+        if self.h11_connection is None and self.h2_connection is None:
+            await self.connect(verify=verify, cert=cert, timeout=timeout)
+
+        if self._request_delay:
+            await asyncio.sleep(self._request_delay)
+
+        if self.h2_connection is not None:
+            response = await self.h2_connection.send(request, timeout=timeout)
+        else:
+            assert self.h11_connection is not None
+            response = await self.h11_connection.send(request, timeout=timeout)
+
+        return response
+
+
+class DelayableSanicConnectionPool(
+    httpx.dispatch.connection_pool.ConnectionPool
+):
     def __init__(self, request_delay=None, *args, **kwargs):
         self._request_delay = request_delay
         super().__init__(*args, **kwargs)
 
-    async def send(self, request, stream=False, ssl=None, timeout=None):
-        connection = await self.acquire_connection(request.url.origin)
-        if (
-            connection.h11_connection is None
-            and connection.h2_connection is None
-        ):
-            await connection.connect(ssl=ssl, timeout=timeout)
-        if self._request_delay:
-            await asyncio.sleep(self._request_delay)
-        try:
-            response = await connection.send(
-                request, stream=stream, ssl=ssl, timeout=timeout
+    async def acquire_connection(self, origin, timeout=None):
+        connection = self.pop_connection(origin)
+
+        if connection is None:
+            pool_timeout = None if timeout is None else timeout.pool_timeout
+
+            await self.max_connections.acquire(timeout=pool_timeout)
+            connection = DelayableHTTPConnection(
+                origin,
+                verify=self.verify,
+                cert=self.cert,
+                http2=self.http2,
+                backend=self.backend,
+                release_func=self.release_connection,
+                trust_env=self.trust_env,
+                uds=self.uds,
+                request_delay=self._request_delay,
             )
-        except BaseException as exc:
-            self.active_connections.remove(connection)
-            self.max_connections.release()
-            raise exc
-        return response
+
+        self.active_connections.add(connection)
+
+        return connection
 
 
-class DelayableSanicAdapter(requests.adapters.HTTPAdapter):
-    def __init__(self, request_delay=None):
-        self.pool = DelayableSanicConnectionPool(request_delay=request_delay)
-
-
-class DelayableSanicSession(requests.Session):
+class DelayableSanicSession(httpx.Client):
     def __init__(self, request_delay=None, *args, **kwargs) -> None:
-        super().__init__(*args, **kwargs)
-        adapter = DelayableSanicAdapter(request_delay=request_delay)
-        self.mount("http://", adapter)
-        self.mount("https://", adapter)
+        dispatch = DelayableSanicConnectionPool(request_delay=request_delay)
+        super().__init__(dispatch=dispatch, *args, **kwargs)
 
 
 class DelayableSanicTestClient(SanicTestClient):

tests/test_requests.py

@@ -10,7 +10,7 @@ import pytest
 
 from sanic import Blueprint, Sanic
 from sanic.exceptions import ServerError
-from sanic.request import DEFAULT_HTTP_CONTENT_TYPE, RequestParameters
+from sanic.request import DEFAULT_HTTP_CONTENT_TYPE, Request, RequestParameters
 from sanic.response import json, text
 from sanic.testing import ASGI_HOST, HOST, PORT
 
@@ -55,11 +55,11 @@ def test_ip(app):
 async def test_ip_asgi(app):
     @app.route("/")
     def handler(request):
-        return text("{}".format(request.ip))
+        return text("{}".format(request.url))
 
     request, response = await app.asgi_client.get("/")
 
-    assert response.text == "mockserver"
+    assert response.text == "http://mockserver/"
 
 
 def test_text(app):
@@ -207,24 +207,24 @@ async def test_empty_json_asgi(app):
 
 
 def test_invalid_json(app):
-    @app.route("/")
+    @app.post("/")
     async def handler(request):
         return json(request.json)
 
     data = "I am not json"
-    request, response = app.test_client.get("/", data=data)
+    request, response = app.test_client.post("/", data=data)
 
     assert response.status == 400
 
 
 @pytest.mark.asyncio
 async def test_invalid_json_asgi(app):
-    @app.route("/")
+    @app.post("/")
     async def handler(request):
         return json(request.json)
 
     data = "I am not json"
-    request, response = await app.asgi_client.get("/", data=data)
+    request, response = await app.asgi_client.post("/", data=data)
 
     assert response.status == 400
 
@@ -1807,26 +1807,6 @@ def test_request_port(app):
     assert hasattr(request, "_port")
 
 
-@pytest.mark.asyncio
-async def test_request_port_asgi(app):
-    @app.get("/")
-    def handler(request):
-        return text("OK")
-
-    request, response = await app.asgi_client.get("/")
-
-    port = request.port
-    assert isinstance(port, int)
-
-    delattr(request, "_socket")
-    delattr(request, "_port")
-
-    port = request.port
-    assert isinstance(port, int)
-    assert hasattr(request, "_socket")
-    assert hasattr(request, "_port")
-
-
 def test_request_socket(app):
     @app.get("/")
     def handler(request):

tests/test_utf8.py

@@ -37,14 +37,14 @@ def skip_test_utf8_route(app):
 
 
 def test_utf8_post_json(app):
-    @app.route("/")
+    @app.post("/")
     async def handler(request):
         return text("OK")
 
     payload = {"test": "✓"}
     headers = {"content-type": "application/json"}
 
-    request, response = app.test_client.get(
+    request, response = app.test_client.post(
         "/", data=json_dumps(payload), headers=headers
     )