diff --git a/.github/stale.yml b/.github/stale.yml index 1ddb59e7..15893644 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -8,6 +8,7 @@ exemptLabels: - urgent - necessary - help wanted + - RFC # Label to use when marking an issue as stale staleLabel: stale # Comment to post when marking an issue as stale. Set to `false` to disable diff --git a/.travis.yml b/.travis.yml index 4553e325..5980de49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,19 @@ matrix: dist: xenial sudo: true name: "Python 3.7 Documentation tests" + - env: TOX_ENV=pyNightly + python: 'nightly' + name: "Python nightly with Extensions" + - env: TOX_ENV=pyNightly-no-ext + python: 'nightly' + name: "Python nightly Extensions" + allow_failures: + - env: TOX_ENV=pyNightly + python: 'nightly' + name: "Python nightly with Extensions" + - env: TOX_ENV=pyNightly-no-ext + python: 'nightly' + name: "Python nightly Extensions" install: - pip install -U tox - pip install codecov diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0c9153b1..be827e00 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,38 @@ +Version 19.12.0 +=============== + +Bugfixes +******** + +- Fix blueprint middleware application + + Currently, any blueprint middleware registered, irrespective of which blueprint was used to do so, was + being applied to all of the routes created by the :code:`@app` and :code:`@blueprint` alike. + + As part of this change, the blueprint based middleware application is enforced based on where they are + registered. + + - If you register a middleware via :code:`@blueprint.middleware` then it will apply only to the routes defined by the blueprint. + - If you register a middleware via :code:`@blueprint_group.middleware` then it will apply to all blueprint based routes that are part of the group. + - If you define a middleware via :code:`@app.middleware` then it will be applied on all available routes (`#37 `__) +- Fix `url_for` behavior with missing SERVER_NAME + + If the `SERVER_NAME` was missing in the `app.config` entity, the `url_for` on the `request` and `app` were failing + due to an `AttributeError`. This fix makes the availability of `SERVER_NAME` on our `app.config` an optional behavior. (`#1707 `__) + + +Improved Documentation +********************** + +- Move docs from RST to MD + + Moved all docs from markdown to restructured text like the rest of the docs to unify the scheme and make it easier in + the future to update documentation. (`#1691 `__) +- Fix documentation for `get` and `getlist` of the `request.args` + + Add additional example for showing the usage of `getlist` and fix the documentation string for `request.args` behavior (`#1704 `__) + + Version 19.6.3 ============== diff --git a/changelogs/1691.doc.rst b/changelogs/1691.doc.rst deleted file mode 100644 index e4a9e3de..00000000 --- a/changelogs/1691.doc.rst +++ /dev/null @@ -1,4 +0,0 @@ -Move docs from RST to MD - -Moved all docs from markdown to restructured text like the rest of the docs to unify the scheme and make it easier in -the future to update documentation. diff --git a/changelogs/1704.doc.rst b/changelogs/1704.doc.rst deleted file mode 100644 index 873ce539..00000000 --- a/changelogs/1704.doc.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix documentation for `get` and `getlist` of the `request.args` - -Add additional example for showing the usage of `getlist` and fix the documentation string for `request.args` behavior diff --git a/changelogs/1707.bugfix.rst b/changelogs/1707.bugfix.rst deleted file mode 100644 index a98cd6b4..00000000 --- a/changelogs/1707.bugfix.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix `url_for` behavior with missing SERVER_NAME - -If the `SERVER_NAME` was missing in the `app.config` entity, the `url_for` on the `request` and `app` were failing -due to an `AttributeError`. This fix makes the availability of `SERVER_NAME` on our `app.config` an optional behavior. diff --git a/docs/sanic/response.rst b/docs/sanic/response.rst index 75a44425..a241b7ac 100644 --- a/docs/sanic/response.rst +++ b/docs/sanic/response.rst @@ -107,6 +107,19 @@ Response without encoding the body def handle_request(request): return response.raw(b'raw data') +Empty +-------------- + +For responding with an empty message as defined by `RFC 2616 `_ + +.. code-block:: python + + from sanic import response + + @app.route('/empty') + async def handle_request(request): + return response.empty() + Modify headers or status ------------------------ diff --git a/docs/sanic/testing.rst b/docs/sanic/testing.rst index 5dba2ab4..67506edc 100644 --- a/docs/sanic/testing.rst +++ b/docs/sanic/testing.rst @@ -2,7 +2,7 @@ Testing ======= Sanic endpoints can be tested locally using the `test_client` object, which -depends on the additional `requests-async `_ +depends on an additional package: `httpx `_ library, which implements an API that mirrors the `requests` library. The `test_client` exposes `get`, `post`, `put`, `delete`, `patch`, `head` and `options` methods @@ -22,7 +22,7 @@ for you to run against your application. A simple example (using pytest) is like assert response.status == 405 Internally, each time you call one of the `test_client` methods, the Sanic app is run at `127.0.0.1:42101` and -your test request is executed against your application, using `requests-async`. +your test request is executed against your application, using `httpx`. The `test_client` methods accept the following arguments and keyword arguments: @@ -55,8 +55,8 @@ And to supply data to a JSON POST request: assert request.json.get('key1') == 'value1' More information about -the available arguments to `requests-async` can be found -[in the documentation for `requests `_. +the available arguments to `httpx` can be found +[in the documentation for `httpx `_. Using a random port diff --git a/environment.yml b/environment.yml index e42a97ef..dc88e646 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - sphinx==1.8.3 - sphinx_rtd_theme==0.4.2 - recommonmark==0.5.0 - - requests-async==0.5.0 + - httpx==0.9.3 - sphinxcontrib-asyncio>=0.2.0 - docutils==0.14 - pygments==2.3.1 diff --git a/sanic/__version__.py b/sanic/__version__.py index 08f1a61d..3124890b 100644 --- a/sanic/__version__.py +++ b/sanic/__version__.py @@ -1 +1 @@ -__version__ = "19.9.0" +__version__ = "19.12.0" diff --git a/sanic/app.py b/sanic/app.py index 58c785b0..abdd36fb 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -85,7 +85,8 @@ class Sanic: self.is_request_stream = False self.websocket_enabled = False self.websocket_tasks = set() - + self.named_request_middleware = {} + self.named_response_middleware = {} # Register alternative method names self.go_fast = self.run @@ -178,7 +179,7 @@ class Sanic: :param stream: :param version: :param name: user defined route name for url_for - :return: decorated function + :return: tuple of routes, decorated function """ # Fix case where the user did not prefix the URL with a / @@ -193,6 +194,12 @@ class Sanic: strict_slashes = self.strict_slashes def response(handler): + if isinstance(handler, tuple): + # if a handler fn is already wrapped in a route, the handler + # variable will be a tuple of (existing routes, handler fn) + routes, handler = handler + else: + routes = [] args = list(signature(handler).parameters.keys()) if not args: @@ -204,16 +211,18 @@ class Sanic: if stream: handler.is_stream = stream - self.router.add( - uri=uri, - methods=methods, - handler=handler, - host=host, - strict_slashes=strict_slashes, - version=version, - name=name, + routes.extend( + self.router.add( + uri=uri, + methods=methods, + handler=handler, + host=host, + strict_slashes=strict_slashes, + version=version, + name=name, + ) ) - return handler + return routes, handler return response @@ -462,7 +471,7 @@ class Sanic: :param subprotocols: optional list of str with supported subprotocols :param name: A unique name assigned to the URL so that it can be used with :func:`url_for` - :return: decorated function + :return: tuple of routes, decorated function """ self.enable_websocket() @@ -475,6 +484,13 @@ class Sanic: strict_slashes = self.strict_slashes def response(handler): + if isinstance(handler, tuple): + # if a handler fn is already wrapped in a route, the handler + # variable will be a tuple of (existing routes, handler fn) + routes, handler = handler + else: + routes = [] + async def websocket_handler(request, *args, **kwargs): request.app = self if not getattr(handler, "__blueprintname__", False): @@ -515,15 +531,17 @@ class Sanic: self.websocket_tasks.remove(fut) await ws.close() - self.router.add( - uri=uri, - handler=websocket_handler, - methods=frozenset({"GET"}), - host=host, - strict_slashes=strict_slashes, - name=name, + routes.extend( + self.router.add( + uri=uri, + handler=websocket_handler, + methods=frozenset({"GET"}), + host=host, + strict_slashes=strict_slashes, + name=name, + ) ) - return handler + return routes, handler return response @@ -544,6 +562,7 @@ class Sanic: :param host: Host IP or FQDN details :param uri: URL path that will be mapped to the websocket handler + handler :param strict_slashes: If the API endpoint needs to terminate with a "/" or not :param subprotocols: Subprotocols to be used with websocket @@ -645,6 +664,22 @@ class Sanic: self.response_middleware.appendleft(middleware) return middleware + def register_named_middleware( + self, middleware, route_names, attach_to="request" + ): + if attach_to == "request": + for _rn in route_names: + if _rn not in self.named_request_middleware: + self.named_request_middleware[_rn] = deque() + if middleware not in self.named_request_middleware[_rn]: + self.named_request_middleware[_rn].append(middleware) + if attach_to == "response": + for _rn in route_names: + if _rn not in self.named_response_middleware: + self.named_response_middleware[_rn] = deque() + if middleware not in self.named_response_middleware[_rn]: + self.named_response_middleware[_rn].append(middleware) + # Decorator def middleware(self, middleware_or_request): """ @@ -916,20 +951,23 @@ class Sanic: # allocation before assignment below. response = None cancelled = False + name = None try: + # Fetch handler from router + handler, args, kwargs, uri, name = self.router.get(request) + # -------------------------------------------- # # Request Middleware # -------------------------------------------- # - response = await self._run_request_middleware(request) + response = await self._run_request_middleware( + request, request_name=name + ) # No middleware results if not response: # -------------------------------------------- # # Execute Handler # -------------------------------------------- # - # Fetch handler from router - handler, args, kwargs, uri = self.router.get(request) - request.uri_template = uri if handler is None: raise ServerError( @@ -993,7 +1031,7 @@ class Sanic: if response is not None: try: response = await self._run_response_middleware( - request, response + request, response, request_name=name ) except CancelledError: # Response middleware can timeout too, as above. @@ -1265,10 +1303,14 @@ class Sanic: if isawaitable(result): await result - async def _run_request_middleware(self, request): + async def _run_request_middleware(self, request, request_name=None): # The if improves speed. I don't know why - if self.request_middleware: - for middleware in self.request_middleware: + named_middleware = self.named_request_middleware.get( + request_name, deque() + ) + applicable_middleware = self.request_middleware + named_middleware + if applicable_middleware: + for middleware in applicable_middleware: response = middleware(request) if isawaitable(response): response = await response @@ -1276,9 +1318,15 @@ class Sanic: return response return None - async def _run_response_middleware(self, request, response): - if self.response_middleware: - for middleware in self.response_middleware: + async def _run_response_middleware( + self, request, response, request_name=None + ): + named_middleware = self.named_response_middleware.get( + request_name, deque() + ) + applicable_middleware = self.response_middleware + named_middleware + if applicable_middleware: + for middleware in applicable_middleware: _response = middleware(request, response) if isawaitable(_response): _response = await _response diff --git a/sanic/asgi.py b/sanic/asgi.py index f9e49005..f08cc454 100644 --- a/sanic/asgi.py +++ b/sanic/asgi.py @@ -15,8 +15,6 @@ from typing import ( ) from urllib.parse import quote -from requests_async import ASGISession # type: ignore - import sanic.app # noqa from sanic.compat import Header @@ -189,7 +187,7 @@ class Lifespan: class ASGIApp: - sanic_app: Union[ASGISession, "sanic.app.Sanic"] + sanic_app: "sanic.app.Sanic" request: Request transport: MockTransport do_stream: bool @@ -223,8 +221,13 @@ class ASGIApp: if scope["type"] == "lifespan": await instance.lifespan(scope, receive, send) else: - url_bytes = scope.get("root_path", "") + quote(scope["path"]) - url_bytes = url_bytes.encode("latin-1") + path = ( + scope["path"][1:] + if scope["path"].startswith("/") + else scope["path"] + ) + url = "/".join([scope.get("root_path", ""), quote(path)]) + url_bytes = url.encode("latin-1") url_bytes += b"?" + scope["query_string"] if scope["type"] == "http": diff --git a/sanic/blueprints.py b/sanic/blueprints.py index b4894f62..511e5162 100644 --- a/sanic/blueprints.py +++ b/sanic/blueprints.py @@ -104,6 +104,8 @@ class Blueprint: url_prefix = options.get("url_prefix", self.url_prefix) + routes = [] + # Routes for future in self.routes: # attach the blueprint name to the handler so that it can be @@ -114,7 +116,7 @@ class Blueprint: version = future.version or self.version - app.route( + _routes, _ = app.route( uri=uri[1:] if uri.startswith("//") else uri, methods=future.methods, host=future.host or self.host, @@ -123,6 +125,8 @@ class Blueprint: version=version, name=future.name, )(future.handler) + if _routes: + routes += _routes for future in self.websocket_routes: # attach the blueprint name to the handler so that it can be @@ -130,21 +134,27 @@ class Blueprint: future.handler.__blueprintname__ = self.name # Prepend the blueprint URI prefix if available uri = url_prefix + future.uri if url_prefix else future.uri - app.websocket( + _routes, _ = app.websocket( uri=uri, host=future.host or self.host, strict_slashes=future.strict_slashes, name=future.name, )(future.handler) + if _routes: + routes += _routes + route_names = [route.name for route in routes] # Middleware for future in self.middlewares: if future.args or future.kwargs: - app.register_middleware( - future.middleware, *future.args, **future.kwargs + app.register_named_middleware( + future.middleware, + route_names, + *future.args, + **future.kwargs ) else: - app.register_middleware(future.middleware) + app.register_named_middleware(future.middleware, route_names) # Exceptions for future in self.exceptions: diff --git a/sanic/headers.py b/sanic/headers.py index 9f50c797..c8fdfcc5 100644 --- a/sanic/headers.py +++ b/sanic/headers.py @@ -5,7 +5,6 @@ from urllib.parse import unquote from sanic.helpers import STATUS_CODES - HeaderIterable = Iterable[Tuple[str, Any]] # Values convertible to str Options = Dict[str, Union[int, str]] # key=value fields in various headers OptionsIterable = Iterable[Tuple[str, str]] # May contain duplicate keys @@ -192,8 +191,6 @@ def format_http1_response( - If `body` is included, content-length must be specified in headers. """ headerbytes = format_http1(headers) - if status == 200: - return b"HTTP/1.1 200 OK\r\n%b\r\n%b" % (headerbytes, body) return b"HTTP/1.1 %d %b\r\n%b\r\n%b" % ( status, STATUS_CODES.get(status, b"UNKNOWN"), diff --git a/sanic/response.py b/sanic/response.py index 96d87baf..d572d951 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -98,7 +98,10 @@ class StreamingHTTPResponse(BaseHTTPResponse): def get_headers( self, version="1.1", keep_alive=False, keep_alive_timeout=None ): - if "Content-Type" not in self.headers: + assert version == "1.1", "No other versions are currently supported" + + # self.headers get priority over content_type + if self.content_type and "Content-Type" not in self.headers: self.headers["Content-Type"] = self.content_type if keep_alive and keep_alive_timeout is not None: @@ -119,7 +122,7 @@ class HTTPResponse(BaseHTTPResponse): body=None, status=200, headers=None, - content_type="text/plain; charset=utf-8", + content_type=None, body_bytes=b"", ): self.content_type = content_type @@ -129,9 +132,7 @@ class HTTPResponse(BaseHTTPResponse): self._cookies = None def output(self, version="1.1", keep_alive=False, keep_alive_timeout=None): - if "Content-Type" not in self.headers: - self.headers["Content-Type"] = self.content_type - + assert version == "1.1", "No other versions are currently supported" body = b"" if has_message_body(self.status): body = self.body @@ -139,15 +140,13 @@ class HTTPResponse(BaseHTTPResponse): "Content-Length", len(self.body) ) + # self.headers get priority over content_type + if self.content_type and "Content-Type" not in self.headers: + self.headers["Content-Type"] = self.content_type + if self.status in (304, 412): self.headers = remove_entity_headers(self.headers) - if keep_alive and keep_alive_timeout is not None: - self.headers["Connection"] = "keep-alive" - self.headers["Keep-Alive"] = keep_alive_timeout - elif not keep_alive: - self.headers["Connection"] = "close" - return format_http1_response(self.status, self.headers.items(), body) @property @@ -157,6 +156,16 @@ class HTTPResponse(BaseHTTPResponse): return self._cookies +def empty(status=204, headers=None): + """ + Returns an empty response to the client. + + :param status Response code. + :param headers Custom Headers. + """ + return HTTPResponse(body_bytes=b"", status=status, headers=headers) + + def json( body, status=200, diff --git a/sanic/router.py b/sanic/router.py index 63119446..2d8817a3 100644 --- a/sanic/router.py +++ b/sanic/router.py @@ -140,21 +140,22 @@ class Router: docs for further details. :return: Nothing """ + routes = [] if version is not None: version = re.escape(str(version).strip("/").lstrip("v")) uri = "/".join(["/v{}".format(version), uri.lstrip("/")]) # add regular version - self._add(uri, methods, handler, host, name) + routes.append(self._add(uri, methods, handler, host, name)) if strict_slashes: - return + return routes if not isinstance(host, str) and host is not None: # we have gotten back to the top of the recursion tree where the # host was originally a list. By now, we've processed the strict # slashes logic on the leaf nodes (the individual host strings in # the list of host) - return + return routes # Add versions with and without trailing / slashed_methods = self.routes_all.get(uri + "/", frozenset({})) @@ -176,10 +177,12 @@ class Router: ) # add version with trailing slash if slash_is_missing: - self._add(uri + "/", methods, handler, host, name) + routes.append(self._add(uri + "/", methods, handler, host, name)) # add version without trailing slash elif without_slash_is_missing: - self._add(uri[:-1], methods, handler, host, name) + routes.append(self._add(uri[:-1], methods, handler, host, name)) + + return routes def _add(self, uri, methods, handler, host=None, name=None): """Add a handler to the route list @@ -328,6 +331,7 @@ class Router: self.routes_dynamic[url_hash(uri)].append(route) else: self.routes_static[uri] = route + return route @staticmethod def check_dynamic_route_exists(pattern, routes_to_check, parameters): @@ -442,6 +446,7 @@ class Router: method=method, allowed_methods=self.get_supported_methods(url), ) + if route: if route.methods and method not in route.methods: raise method_not_supported @@ -476,7 +481,7 @@ class Router: route_handler = route.handler if hasattr(route_handler, "handlers"): route_handler = route_handler.handlers[method] - return route_handler, [], kwargs, route.uri + return route_handler, [], kwargs, route.uri, route.name def is_stream_handler(self, request): """ Handler for request is stream or not. diff --git a/sanic/server.py b/sanic/server.py index 41af81c0..47ebcd95 100644 --- a/sanic/server.py +++ b/sanic/server.py @@ -1,5 +1,6 @@ import asyncio import os +import sys import traceback from collections import deque @@ -87,6 +88,7 @@ class HttpProtocol(asyncio.Protocol): "_header_fragment", "state", "_debug", + "_body_chunks", ) def __init__( @@ -133,7 +135,10 @@ class HttpProtocol(asyncio.Protocol): self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False - self._not_paused = asyncio.Event(loop=loop) + if sys.version_info.minor >= 8: + self._not_paused = asyncio.Event() + else: + self._not_paused = asyncio.Event(loop=loop) self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None @@ -364,6 +369,21 @@ class HttpProtocol(asyncio.Protocol): else: self.request.body_push(body) + async def body_append(self, body): + if ( + self.request is None + or self._request_stream_task is None + or self._request_stream_task.cancelled() + ): + return + + if self.request.stream.is_full(): + self.transport.pause_reading() + await self.request.stream.put(body) + self.transport.resume_reading() + else: + await self.request.stream.put(body) + async def stream_append(self): while self._body_chunks: body = self._body_chunks.popleft() @@ -935,7 +955,10 @@ def serve( else: conn.close() - _shutdown = asyncio.gather(*coros, loop=loop) + if sys.version_info.minor >= 8: + _shutdown = asyncio.gather(*coros, loop=loop) + else: + _shutdown = asyncio.gather(*coros) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) diff --git a/sanic/testing.py b/sanic/testing.py index f4925b7f..c836a943 100644 --- a/sanic/testing.py +++ b/sanic/testing.py @@ -1,14 +1,8 @@ -import asyncio -import types -import typing - from json import JSONDecodeError from socket import socket -from urllib.parse import unquote, urlsplit -import httpcore # type: ignore -import requests_async as requests # type: ignore -import websockets # type: ignore +import httpx +import websockets from sanic.asgi import ASGIApp from sanic.exceptions import MethodNotSupported @@ -22,13 +16,14 @@ PORT = 42101 class SanicTestClient: - def __init__(self, app, port=PORT): + def __init__(self, app, port=PORT, host=HOST): """Use port=None to bind to a random port""" self.app = app self.port = port + self.host = host def get_new_session(self): - return requests.Session() + return httpx.Client() async def _local_request(self, method, url, *args, **kwargs): logger.info(url) @@ -59,7 +54,8 @@ class SanicTestClient: if raw_cookies: response.raw_cookies = {} - for cookie in response.cookies: + + for cookie in response.cookies.jar: response.raw_cookies[cookie.name] = cookie return response @@ -71,6 +67,7 @@ class SanicTestClient: gather_request=True, debug=False, server_kwargs={"auto_reload": False}, + host=None, *request_args, **request_kwargs, ): @@ -95,11 +92,13 @@ class SanicTestClient: return self.app.error_handler.default(request, exception) if self.port: - server_kwargs = dict(host=HOST, port=self.port, **server_kwargs) - host, port = HOST, self.port + server_kwargs = dict( + host=host or self.host, port=self.port, **server_kwargs + ) + host, port = host or self.host, self.port else: sock = socket() - sock.bind((HOST, 0)) + sock.bind((host or self.host, 0)) server_kwargs = dict(sock=sock, **server_kwargs) host, port = sock.getsockname() @@ -175,181 +174,6 @@ class SanicTestClient: return self._sanic_endpoint_test("websocket", *args, **kwargs) -class SanicASGIAdapter(requests.asgi.ASGIAdapter): # noqa - async def send( # type: ignore - self, - request: requests.PreparedRequest, - gather_return: bool = False, - *args: typing.Any, - **kwargs: typing.Any, - ) -> requests.Response: - """This method is taken MOSTLY verbatim from requests-asyn. The - difference is the capturing of a response on the ASGI call and then - returning it on the response object. This is implemented to achieve: - - request, response = await app.asgi_client.get("/") - - You can see the original code here: - https://github.com/encode/requests-async/blob/614f40f77f19e6c6da8a212ae799107b0384dbf9/requests_async/asgi.py#L51""" # noqa - scheme, netloc, path, query, fragment = urlsplit( - request.url - ) # type: ignore - - default_port = {"http": 80, "ws": 80, "https": 443, "wss": 443}[scheme] - - if ":" in netloc: - host, port_string = netloc.split(":", 1) - port = int(port_string) - else: - host = netloc - port = default_port - - # Include the 'host' header. - if "host" in request.headers: - headers = [] # type: typing.List[typing.Tuple[bytes, bytes]] - elif port == default_port: - headers = [(b"host", host.encode())] - else: - headers = [(b"host", (f"{host}:{port}").encode())] - - # Include other request headers. - headers += [ - (key.lower().encode(), value.encode()) - for key, value in request.headers.items() - ] - - no_response = False - if scheme in {"ws", "wss"}: - subprotocol = request.headers.get("sec-websocket-protocol", None) - if subprotocol is None: - subprotocols = [] # type: typing.Sequence[str] - else: - subprotocols = [ - value.strip() for value in subprotocol.split(",") - ] - - scope = { - "type": "websocket", - "path": unquote(path), - "root_path": "", - "scheme": scheme, - "query_string": query.encode(), - "headers": headers, - "client": ["testclient", 50000], - "server": [host, port], - "subprotocols": subprotocols, - } - no_response = True - - else: - scope = { - "type": "http", - "http_version": "1.1", - "method": request.method, - "path": unquote(path), - "root_path": "", - "scheme": scheme, - "query_string": query.encode(), - "headers": headers, - "client": ["testclient", 50000], - "server": [host, port], - "extensions": {"http.response.template": {}}, - } - - async def receive(): - nonlocal request_complete, response_complete - - if request_complete: - while not response_complete: - await asyncio.sleep(0.0001) - return {"type": "http.disconnect"} - - body = request.body - if isinstance(body, str): - body_bytes = body.encode("utf-8") # type: bytes - elif body is None: - body_bytes = b"" - elif isinstance(body, types.GeneratorType): - try: - chunk = body.send(None) - if isinstance(chunk, str): - chunk = chunk.encode("utf-8") - return { - "type": "http.request", - "body": chunk, - "more_body": True, - } - except StopIteration: - request_complete = True - return {"type": "http.request", "body": b""} - else: - body_bytes = body - - request_complete = True - return {"type": "http.request", "body": body_bytes} - - request_complete = False - response_started = False - response_complete = False - raw_kwargs = {"content": b""} # type: typing.Dict[str, typing.Any] - template = None - context = None - return_value = None - - async def send(message) -> None: - nonlocal raw_kwargs, response_started, response_complete, template, context # noqa - - if message["type"] == "http.response.start": - assert ( - not response_started - ), 'Received multiple "http.response.start" messages.' - raw_kwargs["status_code"] = message["status"] - raw_kwargs["headers"] = message["headers"] - response_started = True - elif message["type"] == "http.response.body": - assert response_started, ( - 'Received "http.response.body" ' - 'without "http.response.start".' - ) - assert ( - not response_complete - ), 'Received "http.response.body" after response completed.' - body = message.get("body", b"") - more_body = message.get("more_body", False) - if request.method != "HEAD": - raw_kwargs["content"] += body - if not more_body: - response_complete = True - elif message["type"] == "http.response.template": - template = message["template"] - context = message["context"] - - try: - return_value = await self.app(scope, receive, send) - except BaseException as exc: - if not self.suppress_exceptions: - raise exc from None - - if no_response: - response_started = True - raw_kwargs = {"status_code": 204, "headers": []} - - if not self.suppress_exceptions: - assert response_started, "TestClient did not receive any response." - elif not response_started: - raw_kwargs = {"status_code": 500, "headers": []} - - raw = httpcore.Response(**raw_kwargs) - response = self.build_response(request, raw) - if template is not None: - response.template = template - response.context = context - - if gather_return: - response.return_value = return_value - return response - - class TestASGIApp(ASGIApp): async def __call__(self): await super().__call__() @@ -361,7 +185,11 @@ async def app_call_with_return(self, scope, receive, send): return await asgi_app() -class SanicASGITestClient(requests.ASGISession): +class SanicASGIDispatch(httpx.dispatch.ASGIDispatch): + pass + + +class SanicASGITestClient(httpx.Client): def __init__( self, app, @@ -370,18 +198,18 @@ class SanicASGITestClient(requests.ASGISession): ) -> None: app.__class__.__call__ = app_call_with_return app.asgi = True - super().__init__(app) - adapter = SanicASGIAdapter( - app, suppress_exceptions=suppress_exceptions - ) - self.mount("http://", adapter) - self.mount("https://", adapter) - self.mount("ws://", adapter) - self.mount("wss://", adapter) - self.headers.update({"user-agent": "testclient"}) self.app = app - self.base_url = base_url + + dispatch = SanicASGIDispatch(app=app, client=(ASGI_HOST, PORT)) + super().__init__(dispatch=dispatch, base_url=base_url) + + self.last_request = None + + def _collect_request(request): + self.last_request = request + + app.request_middleware.appendleft(_collect_request) async def request(self, method, url, gather_request=True, *args, **kwargs): @@ -391,33 +219,39 @@ class SanicASGITestClient(requests.ASGISession): response.body = response.content response.content_type = response.headers.get("content-type") - if hasattr(response, "return_value"): - request = response.return_value - del response.return_value - return request, response - - return response - - def merge_environment_settings(self, *args, **kwargs): - settings = super().merge_environment_settings(*args, **kwargs) - settings.update({"gather_return": self.gather_request}) - return settings + return self.last_request, response async def websocket(self, uri, subprotocols=None, *args, **kwargs): - if uri.startswith(("ws:", "wss:")): - url = uri - else: - uri = uri if uri.startswith("/") else "/{uri}".format(uri=uri) - url = "ws://testserver{uri}".format(uri=uri) + scheme = "ws" + path = uri + root_path = "{}://{}".format(scheme, ASGI_HOST) - headers = kwargs.get("headers", {}) - headers.setdefault("connection", "upgrade") - headers.setdefault("sec-websocket-key", "testserver==") - headers.setdefault("sec-websocket-version", "13") - if subprotocols is not None: - headers.setdefault( - "sec-websocket-protocol", ", ".join(subprotocols) - ) - kwargs["headers"] = headers + headers = kwargs.get("headers", {}) + headers.setdefault("connection", "upgrade") + headers.setdefault("sec-websocket-key", "testserver==") + headers.setdefault("sec-websocket-version", "13") + if subprotocols is not None: + headers.setdefault( + "sec-websocket-protocol", ", ".join(subprotocols) + ) - return await self.request("websocket", url, **kwargs) + scope = { + "type": "websocket", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "headers": [map(lambda y: y.encode(), x) for x in headers.items()], + "scheme": scheme, + "root_path": root_path, + "path": path, + "query_string": b"", + } + + async def receive(): + return {} + + async def send(message): + pass + + await self.app(scope, receive, send) + + return None, {} diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index 34acf03f..0b2a417d 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -1,6 +1,6 @@ [tool.towncrier] package = "sanic" -package_dir = "." +package_dir = ".." filename = "../CHANGELOG.rst" directory = "./changelogs" underlines = ["=", "*", "~"] diff --git a/setup.cfg b/setup.cfg index d0efa01a..ac75014e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ multi_line_output = 3 not_skip = __init__.py [version] -current_version = 19.9.0 +current_version = 19.12.0 files = sanic/__version__.py current_version_pattern = __version__ = "{current_version}" new_version_pattern = __version__ = "{new_version}" diff --git a/setup.py b/setup.py index bd48d994..019769cf 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ import codecs import os import re import sys + from distutils.util import strtobool from setuptools import setup @@ -83,7 +84,7 @@ requirements = [ "aiofiles>=0.3.0", "websockets>=7.0,<9.0", "multidict>=4.0,<5.0", - "requests-async==0.5.0", + "httpx==0.9.3", ] tests_require = [ diff --git a/tests/test_app.py b/tests/test_app.py index deb050b6..77171094 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -94,7 +94,7 @@ def test_app_route_raise_value_error(app): def test_app_handle_request_handler_is_none(app, monkeypatch): def mockreturn(*args, **kwargs): - return None, [], {}, "" + return None, [], {}, "", "" # Not sure how to make app.router.get() return None, so use mock here. monkeypatch.setattr(app.router, "get", mockreturn) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index bf024252..4b9ed58b 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,6 +1,6 @@ import asyncio -from collections import deque +from collections import deque, namedtuple import pytest import uvicorn @@ -245,17 +245,26 @@ async def test_cookie_customization(app): return response _, response = await app.asgi_client.get("/cookie") + + CookieDef = namedtuple("CookieDef", ("value", "httponly")) + Cookie = namedtuple("Cookie", ("domain", "path", "value", "httponly")) cookie_map = { - "test": {"value": "Cookie1", "HttpOnly": True}, - "c2": {"value": "Cookie2", "HttpOnly": False}, + "test": CookieDef("Cookie1", True), + "c2": CookieDef("Cookie2", False), } - for k, v in ( - response.cookies._cookies.get("mockserver.local").get("/").items() - ): - assert cookie_map.get(k).get("value") == v.value - if cookie_map.get(k).get("HttpOnly"): - assert "HttpOnly" in v._rest.keys() + cookies = { + c.name: Cookie(c.domain, c.path, c.value, "HttpOnly" in c._rest.keys()) + for c in response.cookies.jar + } + + for name, definition in cookie_map.items(): + cookie = cookies.get(name) + assert cookie + assert cookie.value == definition.value + assert cookie.domain == "mockserver.local" + assert cookie.path == "/" + assert cookie.httponly == definition.httponly @pytest.mark.asyncio diff --git a/tests/test_blueprint_group.py b/tests/test_blueprint_group.py index 32729a49..5a4b805c 100644 --- a/tests/test_blueprint_group.py +++ b/tests/test_blueprint_group.py @@ -83,7 +83,7 @@ def test_bp_group_with_additional_route_params(app: Sanic): _, response = app.test_client.patch("/api/bp2/route/bp2", headers=header) assert response.text == "PATCH_bp2" - _, response = app.test_client.get("/v2/api/bp1/request_path") + _, response = app.test_client.put("/v2/api/bp1/request_path") assert response.status == 401 @@ -141,8 +141,8 @@ def test_bp_group(app: Sanic): _, response = app.test_client.get("/api/bp3") assert response.text == "BP3_OK" - assert MIDDLEWARE_INVOKE_COUNTER["response"] == 4 - assert MIDDLEWARE_INVOKE_COUNTER["request"] == 4 + assert MIDDLEWARE_INVOKE_COUNTER["response"] == 3 + assert MIDDLEWARE_INVOKE_COUNTER["request"] == 2 def test_bp_group_list_operations(app: Sanic): diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py index 16a97309..c21f3807 100644 --- a/tests/test_blueprints.py +++ b/tests/test_blueprints.py @@ -268,7 +268,7 @@ def test_bp_middleware(app): request, response = app.test_client.get("/") assert response.status == 200 - assert response.text == "OK" + assert response.text == "FAIL" def test_bp_exception_handler(app): diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 672d78ac..a59d6c5b 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -1,15 +1,9 @@ import asyncio -import functools -import socket from asyncio import sleep as aio_sleep -from http.client import _encode from json import JSONDecodeError -import httpcore -import requests_async as requests - -from httpcore import PoolTimeout +import httpx from sanic import Sanic, server from sanic.response import text @@ -21,24 +15,28 @@ CONFIG_FOR_TESTS = {"KEEP_ALIVE_TIMEOUT": 2, "KEEP_ALIVE": True} old_conn = None -class ReusableSanicConnectionPool(httpcore.ConnectionPool): - async def acquire_connection(self, origin): +class ReusableSanicConnectionPool( + httpx.dispatch.connection_pool.ConnectionPool +): + async def acquire_connection(self, origin, timeout): global old_conn - connection = self.active_connections.pop_by_origin( - origin, http2_only=True - ) - if connection is None: - connection = self.keepalive_connections.pop_by_origin(origin) + connection = self.pop_connection(origin) if connection is None: - await self.max_connections.acquire() - connection = httpcore.HTTPConnection( + pool_timeout = None if timeout is None else timeout.pool_timeout + + await self.max_connections.acquire(timeout=pool_timeout) + connection = httpx.dispatch.connection.HTTPConnection( origin, - ssl=self.ssl, - timeout=self.timeout, + verify=self.verify, + cert=self.cert, + http2=self.http2, backend=self.backend, release_func=self.release_connection, + trust_env=self.trust_env, + uds=self.uds, ) + self.active_connections.add(connection) if old_conn is not None: @@ -51,17 +49,10 @@ class ReusableSanicConnectionPool(httpcore.ConnectionPool): return connection -class ReusableSanicAdapter(requests.adapters.HTTPAdapter): - def __init__(self): - self.pool = ReusableSanicConnectionPool() - - -class ResusableSanicSession(requests.Session): +class ResusableSanicSession(httpx.Client): def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - adapter = ReusableSanicAdapter() - self.mount("http://", adapter) - self.mount("https://", adapter) + dispatch = ReusableSanicConnectionPool() + super().__init__(dispatch=dispatch, *args, **kwargs) class ReuseableSanicTestClient(SanicTestClient): @@ -74,6 +65,9 @@ class ReuseableSanicTestClient(SanicTestClient): self._tcp_connector = None self._session = None + def get_new_session(self): + return ResusableSanicSession() + # Copied from SanicTestClient, but with some changes to reuse the # same loop for the same app. def _sanic_endpoint_test( @@ -167,7 +161,6 @@ class ReuseableSanicTestClient(SanicTestClient): self._server.close() self._loop.run_until_complete(self._server.wait_closed()) self._server = None - self.app.stop() if self._session: self._loop.run_until_complete(self._session.close()) @@ -186,7 +179,7 @@ class ReuseableSanicTestClient(SanicTestClient): "request_keepalive", CONFIG_FOR_TESTS["KEEP_ALIVE_TIMEOUT"] ) if not self._session: - self._session = ResusableSanicSession() + self._session = self.get_new_session() try: response = await getattr(self._session, method.lower())( url, verify=False, timeout=request_keepalive, *args, **kwargs diff --git a/tests/test_request_buffer_queue_size.py b/tests/test_request_buffer_queue_size.py index 1a9cfdf3..8e42c79a 100644 --- a/tests/test_request_buffer_queue_size.py +++ b/tests/test_request_buffer_queue_size.py @@ -2,6 +2,7 @@ import io from sanic.response import text + data = "abc" * 10_000_000 diff --git a/tests/test_request_stream.py b/tests/test_request_stream.py index 8f893e2b..404299c0 100644 --- a/tests/test_request_stream.py +++ b/tests/test_request_stream.py @@ -332,7 +332,7 @@ def test_request_stream_handle_exception(app): assert response.text == "Error: Requested URL /in_valid_post not found" # 405 - request, response = app.test_client.get("/post/random_id", data=data) + request, response = app.test_client.get("/post/random_id") assert response.status == 405 assert ( response.text == "Error: Method GET not allowed for URL" diff --git a/tests/test_request_timeout.py b/tests/test_request_timeout.py index e3e02d7c..4025c71b 100644 --- a/tests/test_request_timeout.py +++ b/tests/test_request_timeout.py @@ -1,49 +1,70 @@ import asyncio -import httpcore -import requests_async as requests +import httpx from sanic import Sanic from sanic.response import text from sanic.testing import SanicTestClient -class DelayableSanicConnectionPool(httpcore.ConnectionPool): +class DelayableHTTPConnection(httpx.dispatch.connection.HTTPConnection): + def __init__(self, *args, **kwargs): + self._request_delay = None + if "request_delay" in kwargs: + self._request_delay = kwargs.pop("request_delay") + super().__init__(*args, **kwargs) + + async def send(self, request, verify=None, cert=None, timeout=None): + if self.h11_connection is None and self.h2_connection is None: + await self.connect(verify=verify, cert=cert, timeout=timeout) + + if self._request_delay: + await asyncio.sleep(self._request_delay) + + if self.h2_connection is not None: + response = await self.h2_connection.send(request, timeout=timeout) + else: + assert self.h11_connection is not None + response = await self.h11_connection.send(request, timeout=timeout) + + return response + + +class DelayableSanicConnectionPool( + httpx.dispatch.connection_pool.ConnectionPool +): def __init__(self, request_delay=None, *args, **kwargs): self._request_delay = request_delay super().__init__(*args, **kwargs) - async def send(self, request, stream=False, ssl=None, timeout=None): - connection = await self.acquire_connection(request.url.origin) - if ( - connection.h11_connection is None - and connection.h2_connection is None - ): - await connection.connect(ssl=ssl, timeout=timeout) - if self._request_delay: - await asyncio.sleep(self._request_delay) - try: - response = await connection.send( - request, stream=stream, ssl=ssl, timeout=timeout + async def acquire_connection(self, origin, timeout=None): + connection = self.pop_connection(origin) + + if connection is None: + pool_timeout = None if timeout is None else timeout.pool_timeout + + await self.max_connections.acquire(timeout=pool_timeout) + connection = DelayableHTTPConnection( + origin, + verify=self.verify, + cert=self.cert, + http2=self.http2, + backend=self.backend, + release_func=self.release_connection, + trust_env=self.trust_env, + uds=self.uds, + request_delay=self._request_delay, ) - except BaseException as exc: - self.active_connections.remove(connection) - self.max_connections.release() - raise exc - return response + + self.active_connections.add(connection) + + return connection -class DelayableSanicAdapter(requests.adapters.HTTPAdapter): - def __init__(self, request_delay=None): - self.pool = DelayableSanicConnectionPool(request_delay=request_delay) - - -class DelayableSanicSession(requests.Session): +class DelayableSanicSession(httpx.Client): def __init__(self, request_delay=None, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - adapter = DelayableSanicAdapter(request_delay=request_delay) - self.mount("http://", adapter) - self.mount("https://", adapter) + dispatch = DelayableSanicConnectionPool(request_delay=request_delay) + super().__init__(dispatch=dispatch, *args, **kwargs) class DelayableSanicTestClient(SanicTestClient): diff --git a/tests/test_requests.py b/tests/test_requests.py index f24fdcb5..ace172eb 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -55,11 +55,11 @@ def test_ip(app): async def test_ip_asgi(app): @app.route("/") def handler(request): - return text("{}".format(request.ip)) + return text("{}".format(request.url)) request, response = await app.asgi_client.get("/") - assert response.text == "mockserver" + assert response.text == "http://mockserver/" def test_text(app): @@ -242,24 +242,24 @@ async def test_empty_json_asgi(app): def test_invalid_json(app): - @app.route("/") + @app.post("/") async def handler(request): return json(request.json) data = "I am not json" - request, response = app.test_client.get("/", data=data) + request, response = app.test_client.post("/", data=data) assert response.status == 400 @pytest.mark.asyncio async def test_invalid_json_asgi(app): - @app.route("/") + @app.post("/") async def handler(request): return json(request.json) data = "I am not json" - request, response = await app.asgi_client.get("/", data=data) + request, response = await app.asgi_client.post("/", data=data) assert response.status == 400 @@ -1842,26 +1842,6 @@ def test_request_port(app): assert hasattr(request, "_port") -@pytest.mark.asyncio -async def test_request_port_asgi(app): - @app.get("/") - def handler(request): - return text("OK") - - request, response = await app.asgi_client.get("/") - - port = request.port - assert isinstance(port, int) - - delattr(request, "_socket") - delattr(request, "_port") - - port = request.port - assert isinstance(port, int) - assert hasattr(request, "_socket") - assert hasattr(request, "_port") - - def test_request_socket(app): @app.get("/") def handler(request): diff --git a/tests/test_response.py b/tests/test_response.py index 011d853d..ca508af7 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -22,6 +22,7 @@ from sanic.response import ( stream, text, ) +from sanic.response import empty from sanic.server import HttpProtocol from sanic.testing import HOST, PORT @@ -592,3 +593,13 @@ def test_raw_response(app): request, response = app.test_client.get("/test") assert response.content_type == "application/octet-stream" assert response.body == b"raw_response" + + +def test_empty_response(app): + @app.get("/test") + def handler(request): + return empty() + + request, response = app.test_client.get("/test") + assert response.content_type is None + assert response.body == b"" diff --git a/tests/test_routes.py b/tests/test_routes.py index 3b24389f..31fa1a56 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -551,6 +551,35 @@ def test_route_duplicate(app): pass +def test_double_stack_route(app): + @app.route("/test/1") + @app.route("/test/2") + async def handler1(request): + return text("OK") + + request, response = app.test_client.get("/test/1") + assert response.status == 200 + request, response = app.test_client.get("/test/2") + assert response.status == 200 + + +@pytest.mark.asyncio +async def test_websocket_route_asgi(app): + ev = asyncio.Event() + + @app.websocket("/test/1") + @app.websocket("/test/2") + async def handler(request, ws): + ev.set() + + request, response = await app.asgi_client.websocket("/test/1") + first_set = ev.is_set() + ev.clear() + request, response = await app.asgi_client.websocket("/test/1") + second_set = ev.is_set() + assert(first_set and second_set) + + def test_method_not_allowed(app): @app.route("/test", methods=["GET"]) async def handler(request): diff --git a/tests/test_utf8.py b/tests/test_utf8.py index 8fd072a4..a2bf893e 100644 --- a/tests/test_utf8.py +++ b/tests/test_utf8.py @@ -37,14 +37,14 @@ def skip_test_utf8_route(app): def test_utf8_post_json(app): - @app.route("/") + @app.post("/") async def handler(request): return text("OK") payload = {"test": "✓"} headers = {"content-type": "application/json"} - request, response = app.test_client.get( + request, response = app.test_client.post( "/", data=json_dumps(payload), headers=headers ) diff --git a/tox.ini b/tox.ini index a1a99268..d4ed4cb6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ [tox] -envlist = py36, py37, {py36,py37}-no-ext, lint, check, security, docs +envlist = py36, py37, pyNightly, {py36,py37,pyNightly}-no-ext, lint, check, security, docs [testenv] usedevelop = True setenv = - {py36,py37}-no-ext: SANIC_NO_UJSON=1 - {py36,py37}-no-ext: SANIC_NO_UVLOOP=1 + {py36,py37,pyNightly}-no-ext: SANIC_NO_UJSON=1 + {py36,py37,pyNightly}-no-ext: SANIC_NO_UVLOOP=1 deps = coverage pytest==5.2.1 @@ -13,7 +13,7 @@ deps = pytest-sanic pytest-sugar httpcore==0.3.0 - requests-async==0.5.0 + httpx==0.9.3 chardet<=2.3.0 beautifulsoup4 gunicorn