Merge branch '19.12LTS' of github.com:huge-success/sanic into 19.12LTS

Resolve headers as body in ASGI mode

* Bump version to 19.12.3

* Update multidict==5.0.0

Makefile

@@ -71,7 +71,7 @@ black:
 	black --config ./.black.toml sanic tests
 
 fix-import: black
-	isort -rc sanic tests
+	isort sanic tests
 
 
 docs-clean:

sanic/__version__.py

@@ -1 +1 @@
-__version__ = "19.12.0"
+__version__ = "19.12.3"

sanic/app.py

@@ -194,6 +194,12 @@ class Sanic:
             strict_slashes = self.strict_slashes
 
         def response(handler):
+            if isinstance(handler, tuple):
+                # if a handler fn is already wrapped in a route, the handler
+                # variable will be a tuple of (existing routes, handler fn)
+                routes, handler = handler
+            else:
+                routes = []
             args = list(signature(handler).parameters.keys())
 
             if not args:
@@ -205,14 +211,16 @@ class Sanic:
             if stream:
                 handler.is_stream = stream
 
-            routes = self.router.add(
-                uri=uri,
-                methods=methods,
-                handler=handler,
-                host=host,
-                strict_slashes=strict_slashes,
-                version=version,
-                name=name,
+            routes.extend(
+                self.router.add(
+                    uri=uri,
+                    methods=methods,
+                    handler=handler,
+                    host=host,
+                    strict_slashes=strict_slashes,
+                    version=version,
+                    name=name,
+                )
             )
             return routes, handler
 
@@ -476,6 +484,13 @@ class Sanic:
             strict_slashes = self.strict_slashes
 
         def response(handler):
+            if isinstance(handler, tuple):
+                # if a handler fn is already wrapped in a route, the handler
+                # variable will be a tuple of (existing routes, handler fn)
+                routes, handler = handler
+            else:
+                routes = []
+
             async def websocket_handler(request, *args, **kwargs):
                 request.app = self
                 if not getattr(handler, "__blueprintname__", False):
@@ -516,13 +531,15 @@ class Sanic:
                     self.websocket_tasks.remove(fut)
                 await ws.close()
 
-            routes = self.router.add(
-                uri=uri,
-                handler=websocket_handler,
-                methods=frozenset({"GET"}),
-                host=host,
-                strict_slashes=strict_slashes,
-                name=name,
+            routes.extend(
+                self.router.add(
+                    uri=uri,
+                    handler=websocket_handler,
+                    methods=frozenset({"GET"}),
+                    host=host,
+                    strict_slashes=strict_slashes,
+                    name=name,
+                )
             )
             return routes, handler
 
@@ -813,6 +830,14 @@ class Sanic:
                 "Endpoint with name `{}` was not found".format(view_name)
             )
 
+        # If the route has host defined, split that off
+        # TODO: Retain netloc and path separately in Route objects
+        host = uri.find("/")
+        if host > 0:
+            host, uri = uri[:host], uri[host:]
+        else:
+            host = None
+
         if view_name == "static" or view_name.endswith(".static"):
             filename = kwargs.pop("filename", None)
             # it's static folder
@@ -845,7 +870,7 @@ class Sanic:
 
         netloc = kwargs.pop("_server", None)
         if netloc is None and external:
-            netloc = self.config.get("SERVER_NAME", "")
+            netloc = host or self.config.get("SERVER_NAME", "")
 
         if external:
             if not scheme:

sanic/asgi.py

@@ -1,6 +1,5 @@
 import asyncio
 import warnings
-
 from inspect import isawaitable
 from typing import (
     Any,
@@ -16,7 +15,6 @@ from typing import (
 from urllib.parse import quote
 
 import sanic.app  # noqa
-
 from sanic.compat import Header
 from sanic.exceptions import InvalidUsage, ServerError
 from sanic.log import logger
@@ -25,7 +23,6 @@ from sanic.response import HTTPResponse, StreamingHTTPResponse
 from sanic.server import StreamBuffer
 from sanic.websocket import WebSocketConnection
 
-
 ASGIScope = MutableMapping[str, Any]
 ASGIMessage = MutableMapping[str, Any]
 ASGISend = Callable[[ASGIMessage], Awaitable[None]]
@@ -68,9 +65,7 @@ class MockProtocol:
 class MockTransport:
     _protocol: Optional[MockProtocol]
 
-    def __init__(
-        self, scope: ASGIScope, receive: ASGIReceive, send: ASGISend
-    ) -> None:
+    def __init__(self, scope: ASGIScope, receive: ASGIReceive, send: ASGISend) -> None:
         self.scope = scope
         self._receive = receive
         self._send = send
@@ -146,9 +141,7 @@ class Lifespan:
         ) + self.asgi_app.sanic_app.listeners.get("after_server_start", [])
 
         for handler in listeners:
-            response = handler(
-                self.asgi_app.sanic_app, self.asgi_app.sanic_app.loop
-            )
+            response = handler(self.asgi_app.sanic_app, self.asgi_app.sanic_app.loop)
             if isawaitable(response):
                 await response
 
@@ -166,9 +159,7 @@ class Lifespan:
         ) + self.asgi_app.sanic_app.listeners.get("after_server_stop", [])
 
         for handler in listeners:
-            response = handler(
-                self.asgi_app.sanic_app, self.asgi_app.sanic_app.loop
-            )
+            response = handler(self.asgi_app.sanic_app, self.asgi_app.sanic_app.loop)
             if isawaitable(response):
                 await response
 
@@ -213,19 +204,13 @@ class ASGIApp:
                 for key, value in scope.get("headers", [])
             ]
         )
-        instance.do_stream = (
-            True if headers.get("expect") == "100-continue" else False
-        )
+        instance.do_stream = True if headers.get("expect") == "100-continue" else False
         instance.lifespan = Lifespan(instance)
 
         if scope["type"] == "lifespan":
             await instance.lifespan(scope, receive, send)
         else:
-            path = (
-                scope["path"][1:]
-                if scope["path"].startswith("/")
-                else scope["path"]
-            )
+            path = scope["path"][1:] if scope["path"].startswith("/") else scope["path"]
             url = "/".join([scope.get("root_path", ""), quote(path)])
             url_bytes = url.encode("latin-1")
             url_bytes += b"?" + scope["query_string"]
@@ -248,18 +233,11 @@ class ASGIApp:
 
             request_class = sanic_app.request_class or Request
             instance.request = request_class(
-                url_bytes,
-                headers,
-                version,
-                method,
-                instance.transport,
-                sanic_app,
+                url_bytes, headers, version, method, instance.transport, sanic_app,
             )
 
             if sanic_app.is_request_stream:
-                is_stream_handler = sanic_app.router.is_stream_handler(
-                    instance.request
-                )
+                is_stream_handler = sanic_app.router.is_stream_handler(instance.request)
                 if is_stream_handler:
                     instance.request.stream = StreamBuffer(
                         sanic_app.config.REQUEST_BUFFER_QUEUE_SIZE
@@ -338,26 +316,22 @@ class ASGIApp:
                 type(response),
             )
             exception = ServerError("Invalid response type")
-            response = self.sanic_app.error_handler.response(
-                self.request, exception
-            )
+            response = self.sanic_app.error_handler.response(self.request, exception)
             headers = [
                 (str(name).encode("latin-1"), str(value).encode("latin-1"))
                 for name, value in response.headers.items()
                 if name not in (b"Set-Cookie",)
             ]
 
+        response.asgi = True
+
         if "content-length" not in response.headers and not isinstance(
             response, StreamingHTTPResponse
         ):
-            headers += [
-                (b"content-length", str(len(response.body)).encode("latin-1"))
-            ]
+            headers += [(b"content-length", str(len(response.body)).encode("latin-1"))]
 
         if "content-type" not in response.headers:
-            headers += [
-                (b"content-type", str(response.content_type).encode("latin-1"))
-            ]
+            headers += [(b"content-type", str(response.content_type).encode("latin-1"))]
 
         if response.cookies:
             cookies.update(
@@ -369,8 +343,7 @@ class ASGIApp:
             )
 
         headers += [
-            (b"set-cookie", cookie.encode("utf-8"))
-            for k, cookie in cookies.items()
+            (b"set-cookie", cookie.encode("utf-8")) for k, cookie in cookies.items()
         ]
 
         await self.transport.send(

sanic/request.py

@@ -129,27 +129,27 @@ class Request:
 
     def get(self, key, default=None):
         """.. deprecated:: 19.9
-           Custom context is now stored in `request.custom_context.yourkey`"""
+        Custom context is now stored in `request.custom_context.yourkey`"""
         return self.ctx.__dict__.get(key, default)
 
     def __contains__(self, key):
         """.. deprecated:: 19.9
-           Custom context is now stored in `request.custom_context.yourkey`"""
+        Custom context is now stored in `request.custom_context.yourkey`"""
         return key in self.ctx.__dict__
 
     def __getitem__(self, key):
         """.. deprecated:: 19.9
-           Custom context is now stored in `request.custom_context.yourkey`"""
+        Custom context is now stored in `request.custom_context.yourkey`"""
         return self.ctx.__dict__[key]
 
     def __delitem__(self, key):
         """.. deprecated:: 19.9
-           Custom context is now stored in `request.custom_context.yourkey`"""
+        Custom context is now stored in `request.custom_context.yourkey`"""
         del self.ctx.__dict__[key]
 
     def __setitem__(self, key, value):
         """.. deprecated:: 19.9
-           Custom context is now stored in `request.custom_context.yourkey`"""
+        Custom context is now stored in `request.custom_context.yourkey`"""
         setattr(self.ctx, key, value)
 
     def body_init(self):

sanic/response.py

@@ -22,6 +22,9 @@ except ImportError:
 
 
 class BaseHTTPResponse:
+    def __init__(self):
+        self.asgi = False
+
     def _encode_body(self, data):
         try:
             # Try to encode it regularly
@@ -59,12 +62,15 @@ class StreamingHTTPResponse(BaseHTTPResponse):
         content_type="text/plain",
         chunked=True,
     ):
+        super().__init__()
+
         self.content_type = content_type
         self.streaming_fn = streaming_fn
         self.status = status
         self.headers = Header(headers or {})
         self.chunked = chunked
         self._cookies = None
+        self.protocol = None
 
     async def write(self, data):
         """Writes a chunk of data to the streaming response.
@@ -93,8 +99,9 @@ class StreamingHTTPResponse(BaseHTTPResponse):
             keep_alive=keep_alive,
             keep_alive_timeout=keep_alive_timeout,
         )
-        await self.protocol.push_data(headers)
-        await self.protocol.drain()
+        if not getattr(self, "asgi", False):
+            await self.protocol.push_data(headers)
+            await self.protocol.drain()
         await self.streaming_fn(self)
         if self.chunked:
             await self.protocol.push_data(b"0\r\n\r\n")
@@ -144,6 +151,8 @@ class HTTPResponse(BaseHTTPResponse):
         content_type=None,
         body_bytes=b"",
     ):
+        super().__init__()
+
         self.content_type = content_type
 
         if body is not None:
@@ -202,16 +211,14 @@ class HTTPResponse(BaseHTTPResponse):
         return self._cookies
 
 
-def empty(
-    status=204, headers=None,
-):
+def empty(status=204, headers=None):
     """
     Returns an empty response to the client.
 
     :param status Response code.
     :param headers Custom Headers.
     """
-    return HTTPResponse(body_bytes=b"", status=status, headers=headers,)
+    return HTTPResponse(body_bytes=b"", status=status, headers=headers)
 
 
 def json(

sanic/router.py

@@ -484,7 +484,7 @@ class Router:
         return route_handler, [], kwargs, route.uri, route.name
 
     def is_stream_handler(self, request):
-        """ Handler for request is stream or not.
+        """Handler for request is stream or not.
         :param request: Request object
         :return: bool
         """

sanic/server.py

@@ -731,6 +731,26 @@ class AsyncioServer:
             task = asyncio.ensure_future(coro, loop=self.loop)
             return task
 
+    def start_serving(self):
+        if self.server:
+            try:
+                return self.server.start_serving()
+            except AttributeError:
+                raise NotImplementedError(
+                    "server.start_serving not available in this version "
+                    "of asyncio or uvloop."
+                )
+
+    def serve_forever(self):
+        if self.server:
+            try:
+                return self.server.serve_forever()
+            except AttributeError:
+                raise NotImplementedError(
+                    "server.serve_forever not available in this version "
+                    "of asyncio or uvloop."
+                )
+
     def __await__(self):
         """Starts the asyncio server, returns AsyncServerCoro"""
         task = asyncio.ensure_future(self.serve_coro)

sanic/worker.py

@@ -174,7 +174,7 @@ class GunicornWorker(base.Worker):
 
     @staticmethod
     def _create_ssl_context(cfg):
-        """ Creates SSLContext instance for usage in asyncio.create_server.
+        """Creates SSLContext instance for usage in asyncio.create_server.
         See ssl.SSLSocket.__init__ for more details.
         """
         ctx = ssl.SSLContext(cfg.ssl_version)

setup.py

@@ -5,7 +5,6 @@ import codecs
 import os
 import re
 import sys
-
 from distutils.util import strtobool
 
 from setuptools import setup
@@ -39,9 +38,7 @@ def open_local(paths, mode="r", encoding="utf8"):
 
 with open_local(["sanic", "__version__.py"], encoding="latin1") as fp:
     try:
-        version = re.findall(
-            r"^__version__ = \"([^']+)\"\r?$", fp.read(), re.M
-        )[0]
+        version = re.findall(r"^__version__ = \"([^']+)\"\r?$", fp.read(), re.M)[0]
     except IndexError:
         raise RuntimeError("Unable to determine version.")
 
@@ -71,9 +68,7 @@ setup_kwargs = {
     ],
 }
 
-env_dependency = (
-    '; sys_platform != "win32" ' 'and implementation_name == "cpython"'
-)
+env_dependency = '; sys_platform != "win32" ' 'and implementation_name == "cpython"'
 ujson = "ujson>=1.35" + env_dependency
 uvloop = "uvloop>=0.5.3" + env_dependency
 
@@ -83,13 +78,13 @@ requirements = [
     ujson,
     "aiofiles>=0.3.0",
     "websockets>=7.0,<9.0",
-    "multidict>=4.0,<5.0",
+    "multidict==5.0.0",
     "httpx==0.9.3",
 ]
 
 tests_require = [
     "pytest==5.2.1",
-    "multidict>=4.0,<5.0",
+    "multidict==5.0.0",
     "gunicorn",
     "pytest-cov",
     "httpcore==0.3.0",

tests/performance/wheezy/simple_server.py

@@ -39,6 +39,7 @@ main = WSGIApplication(
 
 if __name__ == "__main__":
     import sys
+
     from wsgiref.simple_server import make_server
 
     try:

tests/test_app.py

@@ -41,6 +41,20 @@ def test_create_asyncio_server(app):
         assert srv.is_serving() is True
 
 
+@pytest.mark.skipif(
+    sys.version_info < (3, 7), reason="requires python3.7 or higher"
+)
+def test_asyncio_server_no_start_serving(app):
+    if not uvloop_installed():
+        loop = asyncio.get_event_loop()
+        asyncio_srv_coro = app.create_server(
+            return_asyncio_server=True,
+            asyncio_server_kwargs=dict(start_serving=False),
+        )
+        srv = loop.run_until_complete(asyncio_srv_coro)
+        assert srv.is_serving() is False
+
+
 @pytest.mark.skipif(
     sys.version_info < (3, 7), reason="requires python3.7 or higher"
 )
@@ -53,6 +67,10 @@ def test_asyncio_server_start_serving(app):
         )
         srv = loop.run_until_complete(asyncio_srv_coro)
         assert srv.is_serving() is False
+        loop.run_until_complete(srv.start_serving())
+        assert srv.is_serving() is True
+        srv.close()
+        # Looks like we can't easily test `serve_forever()`
 
 
 def test_app_loop_not_running(app):

tests/test_keep_alive_timeout.py

@@ -230,8 +230,8 @@ async def handler3(request):
 
 def test_keep_alive_timeout_reuse():
     """If the server keep-alive timeout and client keep-alive timeout are
-     both longer than the delay, the client _and_ server will successfully
-     reuse the existing connection."""
+    both longer than the delay, the client _and_ server will successfully
+    reuse the existing connection."""
     try:
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)

tests/test_response.py

@@ -15,13 +15,13 @@ from aiofiles import os as async_os
 from sanic.response import (
     HTTPResponse,
     StreamingHTTPResponse,
+    empty,
     file,
     file_stream,
     json,
     raw,
     stream,
 )
-from sanic.response import empty
 from sanic.server import HttpProtocol
 from sanic.testing import HOST, PORT
 
@@ -232,6 +232,12 @@ def test_chunked_streaming_returns_correct_content(streaming_app):
     assert response.text == "foo,bar"
 
 
+@pytest.mark.asyncio
+async def test_chunked_streaming_returns_correct_content_asgi(streaming_app):
+    request, response = await streaming_app.asgi_client.get("/")
+    assert response.text == "4\r\nfoo,\r\n3\r\nbar\r\n0\r\n\r\n"
+
+
 def test_non_chunked_streaming_adds_correct_headers(non_chunked_streaming_app):
     request, response = non_chunked_streaming_app.test_client.get("/")
     assert "Transfer-Encoding" not in response.headers
@@ -239,8 +245,18 @@ def test_non_chunked_streaming_adds_correct_headers(non_chunked_streaming_app):
     assert response.headers["Content-Length"] == "7"
 
 
+@pytest.mark.asyncio
+async def test_non_chunked_streaming_adds_correct_headers_asgi(
+    non_chunked_streaming_app,
+):
+    request, response = await non_chunked_streaming_app.asgi_client.get("/")
+    assert "Transfer-Encoding" not in response.headers
+    assert response.headers["Content-Type"] == "text/csv"
+    assert response.headers["Content-Length"] == "7"
+
+
 def test_non_chunked_streaming_returns_correct_content(
-    non_chunked_streaming_app
+    non_chunked_streaming_app,
 ):
     request, response = non_chunked_streaming_app.test_client.get("/")
     assert response.text == "foo,bar"
@@ -255,7 +271,7 @@ def test_stream_response_status_returns_correct_headers(status):
 
 @pytest.mark.parametrize("keep_alive_timeout", [10, 20, 30])
 def test_stream_response_keep_alive_returns_correct_headers(
-    keep_alive_timeout
+    keep_alive_timeout,
 ):
     response = StreamingHTTPResponse(sample_streaming_fn)
     headers = response.get_headers(
@@ -284,7 +300,7 @@ def test_stream_response_does_not_include_chunked_header_if_disabled():
 
 
 def test_stream_response_writes_correct_content_to_transport_when_chunked(
-    streaming_app
+    streaming_app,
 ):
     response = StreamingHTTPResponse(sample_streaming_fn)
     response.protocol = MagicMock(HttpProtocol)

tests/test_routes.py

@@ -551,6 +551,35 @@ def test_route_duplicate(app):
             pass
 
 
+def test_double_stack_route(app):
+    @app.route("/test/1")
+    @app.route("/test/2")
+    async def handler1(request):
+        return text("OK")
+
+    request, response = app.test_client.get("/test/1")
+    assert response.status == 200
+    request, response = app.test_client.get("/test/2")
+    assert response.status == 200
+
+
+@pytest.mark.asyncio
+async def test_websocket_route_asgi(app):
+    ev = asyncio.Event()
+
+    @app.websocket("/test/1")
+    @app.websocket("/test/2")
+    async def handler(request, ws):
+        ev.set()
+
+    request, response = await app.asgi_client.websocket("/test/1")
+    first_set = ev.is_set()
+    ev.clear()
+    request, response = await app.asgi_client.websocket("/test/1")
+    second_set = ev.is_set()
+    assert first_set and second_set
+
+
 def test_method_not_allowed(app):
     @app.route("/test", methods=["GET"])
     async def handler(request):