From 377c9890a3fbd322f5f4cc91633c972ef720bf35 Mon Sep 17 00:00:00 2001 From: Cosmo Borsky Date: Fri, 20 Jul 2018 16:39:10 -0400 Subject: [PATCH 01/16] Support status code for file reponse (#1269) Fixes #1268 --- sanic/response.py | 13 ++++++------- tests/test_response.py | 8 +++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sanic/response.py b/sanic/response.py index 2281766e..2d2f5b96 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -233,8 +233,8 @@ def html(body, status=200, headers=None): content_type="text/html; charset=utf-8") -async def file( - location, mime_type=None, headers=None, filename=None, _range=None): +async def file(location, status=200, mime_type=None, headers=None, + filename=None, _range=None): """Return a response object with file data. :param location: Location of file on system. @@ -260,15 +260,14 @@ async def file( out_stream = await _file.read() mime_type = mime_type or guess_type(filename)[0] or 'text/plain' - return HTTPResponse(status=200, + return HTTPResponse(status=status, headers=headers, content_type=mime_type, body_bytes=out_stream) -async def file_stream( - location, chunk_size=4096, mime_type=None, headers=None, - filename=None, _range=None): +async def file_stream(location, status=200, chunk_size=4096, mime_type=None, + headers=None, filename=None, _range=None): """Return a streaming response object with file data. :param location: Location of file on system. @@ -315,7 +314,7 @@ async def file_stream( headers['Content-Range'] = 'bytes %s-%s/%s' % ( _range.start, _range.end, _range.total) return StreamingHTTPResponse(streaming_fn=_streaming_fn, - status=200, + status=status, headers=headers, content_type=mime_type) diff --git a/tests/test_response.py b/tests/test_response.py index 36259970..96c06ce6 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -227,17 +227,19 @@ def get_file_content(static_file_directory, file_name): @pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) -def test_file_response(file_name, static_file_directory): +@pytest.mark.parametrize('status', [200, 401]) +def test_file_response(file_name, static_file_directory, status): app = Sanic('test_file_helper') @app.route('/files/', methods=['GET']) def file_route(request, filename): file_path = os.path.join(static_file_directory, filename) file_path = os.path.abspath(unquote(file_path)) - return file(file_path, mime_type=guess_type(file_path)[0] or 'text/plain') + return file(file_path, status=status, + mime_type=guess_type(file_path)[0] or 'text/plain') request, response = app.test_client.get('/files/{}'.format(file_name)) - assert response.status == 200 + assert response.status == status assert response.body == get_file_content(static_file_directory, file_name) assert 'Content-Disposition' not in response.headers From b238be54a4d13e37954e025e76472c30029390af Mon Sep 17 00:00:00 2001 From: Cosmo Borsky Date: Sat, 21 Jul 2018 01:31:15 -0400 Subject: [PATCH 02/16] Add content_type flag to Sanic.static (#1267) * Add content_type flag to Sanic.static Fixes #1266 * Fix flake8 error in travis Add line to document `content_type` arg * Fix content_type for file streams Update tests herp derp * Remove content_type as an arg to HTTPResponse `response.HTTPResponse` will default to `headers['Content-Type']` instead of `content_type` https://github.com/channelcat/sanic/pull/1267#discussion_r204190913 --- sanic/app.py | 5 ++-- sanic/static.py | 9 ++++---- tests/static/test.html | 26 +++++++++++++++++++++ tests/test_blueprints.py | 49 ++++++++++++++++++++++++++++++++-------- tests/test_static.py | 15 ++++++++++++ 5 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 tests/static/test.html diff --git a/sanic/app.py b/sanic/app.py index 7af82111..f9eda2d4 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -386,13 +386,14 @@ class Sanic: def static(self, uri, file_or_directory, pattern=r'/?.+', use_modified_since=True, use_content_range=False, stream_large_files=False, name='static', host=None, - strict_slashes=None): + strict_slashes=None, content_type=None): """Register a root to serve files from. The input can either be a file or a directory. See """ static_register(self, uri, file_or_directory, pattern, use_modified_since, use_content_range, - stream_large_files, name, host, strict_slashes) + stream_large_files, name, host, strict_slashes, + content_type) def blueprint(self, blueprint, **options): """Register a blueprint on the application. diff --git a/sanic/static.py b/sanic/static.py index f2d02ab0..07831390 100644 --- a/sanic/static.py +++ b/sanic/static.py @@ -19,7 +19,7 @@ from sanic.response import file, file_stream, HTTPResponse def register(app, uri, file_or_directory, pattern, use_modified_since, use_content_range, stream_large_files, name='static', host=None, - strict_slashes=None): + strict_slashes=None, content_type=None): # TODO: Though sanic is not a file server, I feel like we should at least # make a good effort here. Modified-since is nice, but we could # also look into etags, expires, and caching @@ -41,6 +41,7 @@ def register(app, uri, file_or_directory, pattern, If this is an integer, this represents the threshold size to switch to file_stream() :param name: user defined name used for url_for + :param content_type: user defined content type for header """ # If we're not trying to match a file directly, # serve from the folder @@ -95,10 +96,10 @@ def register(app, uri, file_or_directory, pattern, del headers['Content-Length'] for key, value in _range.headers.items(): headers[key] = value + headers['Content-Type'] = content_type \ + or guess_type(file_path)[0] or 'text/plain' if request.method == 'HEAD': - return HTTPResponse( - headers=headers, - content_type=guess_type(file_path)[0] or 'text/plain') + return HTTPResponse(headers=headers) else: if stream_large_files: if isinstance(stream_large_files, int): diff --git a/tests/static/test.html b/tests/static/test.html new file mode 100644 index 00000000..4ba71873 --- /dev/null +++ b/tests/static/test.html @@ -0,0 +1,26 @@ + + +
+                 ▄▄▄▄▄
+        ▀▀▀██████▄▄▄       _______________
+      ▄▄▄▄▄  █████████▄  /                 \
+     ▀▀▀▀█████▌ ▀▐▄ ▀▐█ |   Gotta go fast!  |
+   ▀▀█████▄▄ ▀██████▄██ | _________________/
+   ▀▄▄▄▄▄  ▀▀█▄▀█════█▀ |/
+        ▀▀▀▄  ▀▀███ ▀       ▄▄
+     ▄███▀▀██▄████████▄ ▄▀▀▀▀▀▀█▌
+   ██▀▄▄▄██▀▄███▀ ▀▀████      ▄██
+▄▀▀▀▄██▄▀▀▌████▒▒▒▒▒▒███     ▌▄▄▀
+▌    ▐▀████▐███▒▒▒▒▒▐██▌
+▀▄▄▄▄▀   ▀▀████▒▒▒▒▄██▀
+          ▀▀█████████▀
+        ▄▄██▀██████▀█
+      ▄██▀     ▀▀▀  █
+     ▄█             ▐▌
+ ▄▄▄▄█▌              ▀█▄▄▄▄▀▀▄
+▌     ▐                ▀▀▄▄▄▀
+ ▀▀▄▄▀
+
+
+ + diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py index 4c321646..37756085 100644 --- a/tests/test_blueprints.py +++ b/tests/test_blueprints.py @@ -1,5 +1,6 @@ import asyncio import inspect +import os import pytest from sanic import Sanic @@ -13,6 +14,14 @@ from sanic.constants import HTTP_METHODS # GET # ------------------------------------------------------------ # +def get_file_path(static_file_directory, file_name): + return os.path.join(static_file_directory, file_name) + +def get_file_content(static_file_directory, file_name): + """The content of the static file to check""" + with open(get_file_path(static_file_directory, file_name), 'rb') as file: + return file.read() + @pytest.mark.parametrize('method', HTTP_METHODS) def test_versioned_routes_get(method): app = Sanic('test_shorhand_routes_get') @@ -348,6 +357,28 @@ def test_bp_static(): assert response.status == 200 assert response.body == current_file_contents +@pytest.mark.parametrize('file_name', ['test.html']) +def test_bp_static_content_type(file_name): + # This is done here, since no other test loads a file here + current_file = inspect.getfile(inspect.currentframe()) + current_directory = os.path.dirname(os.path.abspath(current_file)) + static_directory = os.path.join(current_directory, 'static') + + app = Sanic('test_static') + blueprint = Blueprint('test_static') + blueprint.static( + '/testing.file', + get_file_path(static_directory, file_name), + content_type='text/html; charset=utf-8' + ) + + app.blueprint(blueprint) + + request, response = app.test_client.get('/testing.file') + assert response.status == 200 + assert response.body == get_file_content(static_directory, file_name) + assert response.headers['Content-Type'] == 'text/html; charset=utf-8' + def test_bp_shorthand(): app = Sanic('test_shorhand_routes') blueprint = Blueprint('test_shorhand_routes') @@ -449,41 +480,41 @@ def test_bp_shorthand(): def test_bp_group(): app = Sanic('test_nested_bp_groups') - + deep_0 = Blueprint('deep_0', url_prefix='/deep') deep_1 = Blueprint('deep_1', url_prefix = '/deep1') @deep_0.route('/') def handler(request): return text('D0_OK') - + @deep_1.route('/bottom') def handler(request): return text('D1B_OK') mid_0 = Blueprint.group(deep_0, deep_1, url_prefix='/mid') mid_1 = Blueprint('mid_tier', url_prefix='/mid1') - + @mid_1.route('/') def handler(request): return text('M1_OK') top = Blueprint.group(mid_0, mid_1) - + app.blueprint(top) - + @app.route('/') def handler(request): return text('TOP_OK') - + request, response = app.test_client.get('/') assert response.text == 'TOP_OK' - + request, response = app.test_client.get('/mid1') assert response.text == 'M1_OK' - + request, response = app.test_client.get('/mid/deep') assert response.text == 'D0_OK' - + request, response = app.test_client.get('/mid/deep1/bottom') assert response.text == 'D1B_OK' diff --git a/tests/test_static.py b/tests/test_static.py index 276001cc..3335e248 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -36,6 +36,21 @@ def test_static_file(static_file_directory, file_name): assert response.body == get_file_content(static_file_directory, file_name) +@pytest.mark.parametrize('file_name', ['test.html']) +def test_static_file_content_type(static_file_directory, file_name): + app = Sanic('test_static') + app.static( + '/testing.file', + get_file_path(static_file_directory, file_name), + content_type='text/html; charset=utf-8' + ) + + request, response = app.test_client.get('/testing.file') + assert response.status == 200 + assert response.body == get_file_content(static_file_directory, file_name) + assert response.headers['Content-Type'] == 'text/html; charset=utf-8' + + @pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt']) @pytest.mark.parametrize('base_uri', ['/static', '', '/dir']) def test_static_directory(file_name, base_uri, static_file_directory): From 39ff02b6e4b2f3af9eafa52a2e9ba5d4fd5009e5 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Mon, 6 Aug 2018 14:12:30 +1000 Subject: [PATCH 03/16] Modifications the `handle_request` function to detect and gracefully handle the case that the request_handler Task is canceled by the sanic server while it is handling the request. One common occurrence of this is when the server issues a ResponseTimeout error, it also cancels the response_handler Task. The Canceled exception handler purposely sets `response` to `None` to drop references to the handler coroutine, in an attempt to preemptively release resources. This commit also fixes a possible reference-before-assignment of the `response` variable in the `handle_request` function. Finally, another byproduct of this change is that ResponseMiddleware will no longer run if the `response` is `None`. --- sanic/app.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index f9eda2d4..e4e4ff20 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -571,6 +571,10 @@ class Sanic: :return: Nothing """ + # Define `response` var here to remove warnings about + # allocation before assignment below. + response = None + cancelled = False try: # -------------------------------------------- # # Request Middleware @@ -597,6 +601,13 @@ class Sanic: response = handler(request, *args, **kwargs) if isawaitable(response): response = await response + except CancelledError: + # If response handler times out, the server handles the error + # and cancels the handle_request job. + # In this case, the transport is already closed and we cannot + # issue a response. + response = None + cancelled = True except Exception as e: # -------------------------------------------- # # Response Generation Failed @@ -622,13 +633,22 @@ class Sanic: # -------------------------------------------- # # Response Middleware # -------------------------------------------- # - try: - response = await self._run_response_middleware(request, - response) - except BaseException: - error_logger.exception( - 'Exception occurred in one of response middleware handlers' - ) + # Don't run response middleware if response is None + if response is not None: + try: + response = await self._run_response_middleware(request, + response) + except CancelledError: + # Response middleware can timeout too, as above. + response = None + cancelled = True + except BaseException: + error_logger.exception( + 'Exception occurred in one of response ' + 'middleware handlers' + ) + if cancelled: + raise CancelledError() # pass the response to the correct callback if isinstance(response, StreamingHTTPResponse): From afea15e4a7834a83ff95ddf974852272e4c94b80 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Mon, 6 Aug 2018 15:02:12 +1000 Subject: [PATCH 04/16] Add a test for the graceful CancelledError handling. The user app should _never_ see a CancelledError bubble up, nor should they be able to catch it, because the response is already sent at that point. --- tests/test_response_timeout.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_response_timeout.py b/tests/test_response_timeout.py index bf55a42e..44204b5e 100644 --- a/tests/test_response_timeout.py +++ b/tests/test_response_timeout.py @@ -7,6 +7,7 @@ from sanic.config import Config Config.RESPONSE_TIMEOUT = 1 response_timeout_app = Sanic('test_response_timeout') response_timeout_default_app = Sanic('test_response_timeout_default') +response_handler_cancelled_app = Sanic('test_response_handler_cancelled') @response_timeout_app.route('/1') @@ -36,3 +37,29 @@ def test_default_server_error_response_timeout(): request, response = response_timeout_default_app.test_client.get('/1') assert response.status == 503 assert response.text == 'Error: Response Timeout' + + +response_handler_cancelled_app.flag = False + + +@response_handler_cancelled_app.exception(asyncio.CancelledError) +def handler_cancelled(request, exception): + # If we get a CancelledError, it means sanic has already sent a response, + # we should not ever have to handle a CancelledError. + response_handler_cancelled_app.flag = True + return text("App received CancelledError!", 500) + # The client will never receive this response, because the socket + # is already closed when we get a CancelledError. + + +@response_handler_cancelled_app.route('/1') +async def handler_3(request): + await asyncio.sleep(2) + return text('OK') + + +def test_response_handler_cancelled(): + request, response = response_handler_cancelled_app.test_client.get('/1') + assert response.status == 503 + assert response.text == 'Error: Response Timeout' + assert response_handler_cancelled_app.flag is False From 212da1029e9a0ed786b5a1bfaf10183bfef9a790 Mon Sep 17 00:00:00 2001 From: abuckenheimer Date: Tue, 7 Aug 2018 14:48:18 -0400 Subject: [PATCH 05/16] disabled auto_reload by default in windows (#1280) --- sanic/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index f9eda2d4..e82028f4 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -671,8 +671,8 @@ class Sanic: """ # Default auto_reload to false auto_reload = False - # If debug is set, default it to true - if debug: + # If debug is set, default it to true (unless on windows) + if debug and os.name == 'posix': auto_reload = True # Allow for overriding either of the defaults auto_reload = kwargs.get("auto_reload", auto_reload) From 6abdf9f9c162140a4981c21361d334323ed47d65 Mon Sep 17 00:00:00 2001 From: hqy Date: Thu, 16 Aug 2018 01:23:04 +0800 Subject: [PATCH 06/16] fixed #1143 (#1276) * fixed #1143 * fixed build failed with create_serve call _helper failed --- sanic/app.py | 16 ++++++++-------- sanic/config.py | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index e82028f4..367acc5a 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -688,11 +688,12 @@ class Sanic: warnings.simplefilter('default') warnings.warn("stop_event will be removed from future versions.", DeprecationWarning) + # compatibility old access_log params + self.config.ACCESS_LOG = access_log server_settings = self._helper( host=host, port=port, debug=debug, ssl=ssl, sock=sock, workers=workers, protocol=protocol, backlog=backlog, - register_sys_signals=register_sys_signals, - access_log=access_log, auto_reload=auto_reload) + register_sys_signals=register_sys_signals, auto_reload=auto_reload) try: self.is_running = True @@ -746,12 +747,12 @@ class Sanic: warnings.simplefilter('default') warnings.warn("stop_event will be removed from future versions.", DeprecationWarning) - + # compatibility old access_log params + self.config.ACCESS_LOG = access_log server_settings = self._helper( host=host, port=port, debug=debug, ssl=ssl, sock=sock, loop=get_event_loop(), protocol=protocol, - backlog=backlog, run_async=True, - access_log=access_log) + backlog=backlog, run_async=True) # Trigger before_start events await self.trigger_events( @@ -796,8 +797,7 @@ class Sanic: def _helper(self, host=None, port=None, debug=False, ssl=None, sock=None, workers=1, loop=None, protocol=HttpProtocol, backlog=100, stop_event=None, - register_sys_signals=True, run_async=False, access_log=True, - auto_reload=False): + register_sys_signals=True, run_async=False, auto_reload=False): """Helper function used by `run` and `create_server`.""" if isinstance(ssl, dict): # try common aliaseses @@ -838,7 +838,7 @@ class Sanic: 'loop': loop, 'register_sys_signals': register_sys_signals, 'backlog': backlog, - 'access_log': access_log, + 'access_log': self.config.ACCESS_LOG, 'websocket_max_size': self.config.WEBSOCKET_MAX_SIZE, 'websocket_max_queue': self.config.WEBSOCKET_MAX_QUEUE, 'websocket_read_limit': self.config.WEBSOCKET_READ_LIMIT, diff --git a/sanic/config.py b/sanic/config.py index 8e1f383c..c5e42de5 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -39,6 +39,7 @@ class Config(dict): self.WEBSOCKET_READ_LIMIT = 2 ** 16 self.WEBSOCKET_WRITE_LIMIT = 2 ** 16 self.GRACEFUL_SHUTDOWN_TIMEOUT = 15.0 # 15 sec + self.ACCESS_LOG = True if load_env: prefix = SANIC_PREFIX if load_env is True else load_env From ec226e33cb3a75af3407fca56dbf0aa4b7078db3 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Thu, 16 Aug 2018 15:00:23 +1000 Subject: [PATCH 07/16] Pin aiohttp <= 3.2.1 in requirements-dev.txt (fixes errors for new contributors checking out the code and setting up a dev environment) Future-proof the some test cases so they work with aiohttp >= 3.3.0, in case we bump the aiohttp version in the future. --- requirements-dev.txt | 2 +- tests/test_keep_alive_timeout.py | 30 +++++++++++-- tests/test_request_timeout.py | 73 ++++++++++++++++++++++++++------ 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3d94c51d..004f6f9e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ aiofiles -aiohttp>=2.3.0 +aiohttp>=2.3.0,<=3.2.1 chardet<=2.3.0 beautifulsoup4 coverage diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 2a9e93a2..53a2872e 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -9,14 +9,39 @@ import aiohttp from aiohttp import TCPConnector from sanic.testing import SanicTestClient, HOST, PORT +try: + try: + import packaging # direct use + except ImportError: + # setuptools v39.0 and above. + try: + from setuptools.extern import packaging + except ImportError: + # Before setuptools v39.0 + from pkg_resources.extern import packaging + version = packaging.version +except ImportError: + raise RuntimeError("The 'packaging' library is missing.") + +aiohttp_version = version.parse(aiohttp.__version__) class ReuseableTCPConnector(TCPConnector): def __init__(self, *args, **kwargs): super(ReuseableTCPConnector, self).__init__(*args, **kwargs) self.old_proto = None - if aiohttp.__version__ >= '3.0': - + if aiohttp_version >= version.parse('3.3.0'): + async def connect(self, req, traces, timeout): + new_conn = await super(ReuseableTCPConnector, self)\ + .connect(req, traces, timeout) + if self.old_proto is not None: + if self.old_proto != new_conn._protocol: + raise RuntimeError( + "We got a new connection, wanted the same one!") + print(new_conn.__dict__) + self.old_proto = new_conn._protocol + return new_conn + elif aiohttp_version >= version.parse('3.0.0'): async def connect(self, req, traces=None): new_conn = await super(ReuseableTCPConnector, self)\ .connect(req, traces=traces) @@ -28,7 +53,6 @@ class ReuseableTCPConnector(TCPConnector): self.old_proto = new_conn._protocol return new_conn else: - async def connect(self, req): new_conn = await super(ReuseableTCPConnector, self)\ .connect(req) diff --git a/tests/test_request_timeout.py b/tests/test_request_timeout.py index b3eb78aa..672d0588 100644 --- a/tests/test_request_timeout.py +++ b/tests/test_request_timeout.py @@ -5,9 +5,24 @@ import asyncio from sanic.response import text from sanic.config import Config import aiohttp -from aiohttp import TCPConnector +from aiohttp import TCPConnector, ClientResponse from sanic.testing import SanicTestClient, HOST, PORT +try: + try: + import packaging # direct use + except ImportError: + # setuptools v39.0 and above. + try: + from setuptools.extern import packaging + except ImportError: + # Before setuptools v39.0 + from pkg_resources.extern import packaging + version = packaging.version +except ImportError: + raise RuntimeError("The 'packaging' library is missing.") + +aiohttp_version = version.parse(aiohttp.__version__) class DelayableTCPConnector(TCPConnector): @@ -38,8 +53,11 @@ class DelayableTCPConnector(TCPConnector): self.orig_start = getattr(resp, 'start') try: - ret = await self.orig_start(connection, - read_until_eof) + if aiohttp_version >= version.parse("3.3.0"): + ret = await self.orig_start(connection) + else: + ret = await self.orig_start(connection, + read_until_eof) except Exception as e: raise e return ret @@ -57,15 +75,31 @@ class DelayableTCPConnector(TCPConnector): await asyncio.sleep(self.delay) t = req.loop.time() print("sending at {}".format(t), flush=True) - conn = next(iter(args)) # first arg is connection - if aiohttp.__version__ >= "3.1.0": + conn = next(iter(args)) # first arg is connection + + if aiohttp_version >= version.parse("3.1.0"): try: delayed_resp = await self.orig_send(*args, **kwargs) except Exception as e: - return aiohttp.ClientResponse(req.method, req.url, - writer=None, continue100=None, timer=None, - request_info=None, auto_decompress=None, traces=[], - loop=req.loop, session=None) + if aiohttp_version >= version.parse("3.3.0"): + return aiohttp.ClientResponse(req.method, req.url, + writer=None, + continue100=None, + timer=None, + request_info=None, + traces=[], + loop=req.loop, + session=None) + else: + return aiohttp.ClientResponse(req.method, req.url, + writer=None, + continue100=None, + timer=None, + request_info=None, + auto_decompress=None, + traces=[], + loop=req.loop, + session=None) else: try: delayed_resp = self.orig_send(*args, **kwargs) @@ -73,7 +107,7 @@ class DelayableTCPConnector(TCPConnector): return aiohttp.ClientResponse(req.method, req.url) return delayed_resp - if aiohttp.__version__ >= "3.1.0": + if aiohttp_version >= version.parse("3.1.0"): # aiohttp changed the request.send method to async async def send(self, *args, **kwargs): gen = self.delayed_send(*args, **kwargs) @@ -96,12 +130,25 @@ class DelayableTCPConnector(TCPConnector): self._post_connect_delay = _post_connect_delay self._pre_request_delay = _pre_request_delay - if aiohttp.__version__ >= '3.0': - + if aiohttp_version >= version.parse("3.3.0"): + async def connect(self, req, traces, timeout): + d_req = DelayableTCPConnector.\ + RequestContextManager(req, self._pre_request_delay) + conn = await super(DelayableTCPConnector, self).\ + connect(req, traces, timeout) + if self._post_connect_delay and self._post_connect_delay > 0: + await asyncio.sleep(self._post_connect_delay, + loop=self._loop) + req.send = d_req.send + t = req.loop.time() + print("Connected at {}".format(t), flush=True) + return conn + elif aiohttp_version >= version.parse("3.0.0"): async def connect(self, req, traces=None): d_req = DelayableTCPConnector.\ RequestContextManager(req, self._pre_request_delay) - conn = await super(DelayableTCPConnector, self).connect(req, traces=traces) + conn = await super(DelayableTCPConnector, self).\ + connect(req, traces=traces) if self._post_connect_delay and self._post_connect_delay > 0: await asyncio.sleep(self._post_connect_delay, loop=self._loop) From 1814ff05f4fde9f875aa7f55b27b59b9a0fcb798 Mon Sep 17 00:00:00 2001 From: Innokenty Lebedev Date: Thu, 16 Aug 2018 21:59:58 +0300 Subject: [PATCH 08/16] Add sse extension (#1288) --- docs/sanic/extensions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sanic/extensions.md b/docs/sanic/extensions.md index 01c89f95..c0728627 100644 --- a/docs/sanic/extensions.md +++ b/docs/sanic/extensions.md @@ -31,3 +31,4 @@ A list of Sanic extensions created by the community. - [Sanic-Auth](https://github.com/pyx/sanic-auth): A minimal backend agnostic session-based user authentication mechanism for Sanic. - [Sanic-CookieSession](https://github.com/pyx/sanic-cookiesession): A client-side only, cookie-based session, similar to the built-in session in Flask. - [Sanic-WTF](https://github.com/pyx/sanic-wtf): Sanic-WTF makes using WTForms with Sanic and CSRF (Cross-Site Request Forgery) protection a little bit easier. +- [sanic-sse](https://github.com/inn0kenty/sanic_sse): [Server-Sent Events](https://en.wikipedia.org/wiki/Server-sent_events) implementation for Sanic. From 79e35bbdf600d62707b6d3a5d57051e3c608daf2 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Fri, 17 Aug 2018 16:30:03 +1000 Subject: [PATCH 09/16] Fix auto_reload in Linux (#1286) * Fix two problems with the auto_reloader in Linux. 1) Change 'posix' to 'linux' in sys.plaform check, because 'posix' is an invalid value and 'linux' is the correct value to use here. 2) In kill_process_children, don't just kill the 2nd level procs, also kill the 1st level procs. Also in kill_process_children, catch and ignore errors in the case that the child proc is already killed. * Fix flake8 formatting on PR --- sanic/reloader_helpers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sanic/reloader_helpers.py b/sanic/reloader_helpers.py index 73759124..e0cb42e0 100644 --- a/sanic/reloader_helpers.py +++ b/sanic/reloader_helpers.py @@ -74,7 +74,14 @@ def kill_process_children_unix(pid): with open(children_proc_path) as children_list_file_2: children_list_pid_2 = children_list_file_2.read().split() for _pid in children_list_pid_2: - os.kill(int(_pid), signal.SIGTERM) + try: + os.kill(int(_pid), signal.SIGTERM) + except ProcessLookupError: + continue + try: + os.kill(int(child_pid), signal.SIGTERM) + except ProcessLookupError: + continue def kill_process_children_osx(pid): @@ -94,7 +101,7 @@ def kill_process_children(pid): """ if sys.platform == 'darwin': kill_process_children_osx(pid) - elif sys.platform == 'posix': + elif sys.platform == 'linux': kill_process_children_unix(pid) else: pass # should signal error here @@ -136,8 +143,8 @@ def watchdog(sleep_interval): continue elif mtime > old_time: kill_process_children(worker_process.pid) + worker_process.terminate() worker_process = restart_with_reloader() - mtimes[filename] = mtime break From b398c1fe72977583e1c663b7a4c07044e6f272f1 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Fri, 17 Aug 2018 11:43:15 -0700 Subject: [PATCH 10/16] Increment to 0.8.0 Signed-off-by: Eli Uriegas --- sanic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanic/__init__.py b/sanic/__init__.py index 78bc7bd9..5e6ff4da 100644 --- a/sanic/__init__.py +++ b/sanic/__init__.py @@ -1,6 +1,6 @@ from sanic.app import Sanic from sanic.blueprints import Blueprint -__version__ = '0.7.0' +__version__ = '0.8.0' __all__ = ['Sanic', 'Blueprint'] From 30e6a310f132752669a74927530e8bc52a51e98e Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Sun, 19 Aug 2018 11:12:13 +1000 Subject: [PATCH 11/16] Pausable response streams (#1179) * This commit adds handlers for the asyncio/uvloop protocol callbacks for pause_writing and resume_writing. These are needed for the correct functioning of built-in tcp flow-control provided by uvloop and asyncio. This is somewhat of a breaking change, because the `write` function in user streaming callbacks now must be `await`ed. This is necessary because it is possible now that the http protocol may be paused, and any calls to write may need to wait on an async event to be called to become unpaused. Updated examples and tests to reflect this change. This change does not apply to websocket connections. A change to websocket connections may be required to match this change. * Fix a couple of PEP8 errors caused by previous rebase. * update docs add await syntax to response.write in response-streaming docs. * remove commented out code from a test file --- docs/sanic/streaming.md | 8 ++++---- examples/request_stream/server.py | 2 +- sanic/response.py | 19 +++++++++++-------- sanic/server.py | 20 ++++++++++++++++++-- tests/test_request_stream.py | 16 ++++++++-------- tests/test_response.py | 24 ++++++++++++++++++------ 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/docs/sanic/streaming.md b/docs/sanic/streaming.md index a785322a..bf3ca664 100644 --- a/docs/sanic/streaming.md +++ b/docs/sanic/streaming.md @@ -37,7 +37,7 @@ async def handler(request): if body is None: break body = body.decode('utf-8').replace('1', 'A') - response.write(body) + await response.write(body) return stream(streaming) @@ -85,8 +85,8 @@ app = Sanic(__name__) @app.route("/") async def test(request): async def sample_streaming_fn(response): - response.write('foo,') - response.write('bar') + await response.write('foo,') + await response.write('bar') return stream(sample_streaming_fn, content_type='text/csv') ``` @@ -100,7 +100,7 @@ async def index(request): conn = await asyncpg.connect(database='test') async with conn.transaction(): async for record in conn.cursor('SELECT generate_series(0, 10)'): - response.write(record[0]) + await response.write(record[0]) return stream(stream_from_db) ``` diff --git a/examples/request_stream/server.py b/examples/request_stream/server.py index e53a224c..d3d35aef 100644 --- a/examples/request_stream/server.py +++ b/examples/request_stream/server.py @@ -30,7 +30,7 @@ async def handler(request): if body is None: break body = body.decode('utf-8').replace('1', 'A') - response.write(body) + await response.write(body) return stream(streaming) diff --git a/sanic/response.py b/sanic/response.py index 2d2f5b96..f169b4f2 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -46,7 +46,7 @@ class BaseHTTPResponse: class StreamingHTTPResponse(BaseHTTPResponse): __slots__ = ( - 'transport', 'streaming_fn', 'status', + 'protocol', 'streaming_fn', 'status', 'content_type', 'headers', '_cookies' ) @@ -58,7 +58,7 @@ class StreamingHTTPResponse(BaseHTTPResponse): self.headers = CIMultiDict(headers or {}) self._cookies = None - def write(self, data): + async def write(self, data): """Writes a chunk of data to the streaming response. :param data: bytes-ish data to be written. @@ -66,8 +66,9 @@ class StreamingHTTPResponse(BaseHTTPResponse): if type(data) != bytes: data = self._encode_body(data) - self.transport.write( + self.protocol.push_data( b"%x\r\n%b\r\n" % (len(data), data)) + await self.protocol.drain() async def stream( self, version="1.1", keep_alive=False, keep_alive_timeout=None): @@ -77,10 +78,12 @@ class StreamingHTTPResponse(BaseHTTPResponse): headers = self.get_headers( version, keep_alive=keep_alive, keep_alive_timeout=keep_alive_timeout) - self.transport.write(headers) - + self.protocol.push_data(headers) + await self.protocol.drain() await self.streaming_fn(self) - self.transport.write(b'0\r\n\r\n') + self.protocol.push_data(b'0\r\n\r\n') + # no need to await drain here after this write, because it is the + # very last thing we write and nothing needs to wait for it. def get_headers( self, version="1.1", keep_alive=False, keep_alive_timeout=None): @@ -298,13 +301,13 @@ async def file_stream(location, status=200, chunk_size=4096, mime_type=None, if len(content) < 1: break to_send -= len(content) - response.write(content) + await response.write(content) else: while True: content = await _file.read(chunk_size) if len(content) < 1: break - response.write(content) + await response.write(content) finally: await _file.close() return # Returning from this fn closes the stream diff --git a/sanic/server.py b/sanic/server.py index 11e54edc..d5a8f211 100644 --- a/sanic/server.py +++ b/sanic/server.py @@ -55,7 +55,8 @@ class HttpProtocol(asyncio.Protocol): # connection management '_total_request_size', '_request_timeout_handler', '_response_timeout_handler', '_keep_alive_timeout_handler', - '_last_request_time', '_last_response_time', '_is_stream_handler') + '_last_request_time', '_last_response_time', '_is_stream_handler', + '_not_paused') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, @@ -82,6 +83,7 @@ class HttpProtocol(asyncio.Protocol): self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False + self._not_paused = asyncio.Event(loop=loop) self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None @@ -96,6 +98,7 @@ class HttpProtocol(asyncio.Protocol): if 'requests_count' not in self.state: self.state['requests_count'] = 0 self._debug = debug + self._not_paused.set() @property def keep_alive(self): @@ -124,6 +127,12 @@ class HttpProtocol(asyncio.Protocol): if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() + def pause_writing(self): + self._not_paused.clear() + + def resume_writing(self): + self._not_paused.set() + def request_timeout_callback(self): # See the docstring in the RequestTimeout exception, to see # exactly what this timeout is checking for. @@ -351,6 +360,12 @@ class HttpProtocol(asyncio.Protocol): self._last_response_time = current_time self.cleanup() + async def drain(self): + await self._not_paused.wait() + + def push_data(self, data): + self.transport.write(data) + async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches @@ -360,9 +375,10 @@ class HttpProtocol(asyncio.Protocol): if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None + try: keep_alive = self.keep_alive - response.transport = self.transport + response.protocol = self await response.stream( self.request.version, keep_alive, self.keep_alive_timeout) self.log_response(response) diff --git a/tests/test_request_stream.py b/tests/test_request_stream.py index 4ca4e44e..b14aa519 100644 --- a/tests/test_request_stream.py +++ b/tests/test_request_stream.py @@ -83,7 +83,7 @@ def test_request_stream_app(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) @app.put('/_put') @@ -100,7 +100,7 @@ def test_request_stream_app(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) @app.patch('/_patch') @@ -117,7 +117,7 @@ def test_request_stream_app(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) assert app.is_request_stream is True @@ -177,7 +177,7 @@ def test_request_stream_handle_exception(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) # 404 @@ -231,7 +231,7 @@ def test_request_stream_blueprint(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) @bp.put('/_put') @@ -248,7 +248,7 @@ def test_request_stream_blueprint(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) @bp.patch('/_patch') @@ -265,7 +265,7 @@ def test_request_stream_blueprint(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) app.blueprint(bp) @@ -380,7 +380,7 @@ def test_request_stream(): body = await request.stream.get() if body is None: break - response.write(body.decode('utf-8')) + await response.write(body.decode('utf-8')) return stream(streaming) @app.get('/get') diff --git a/tests/test_response.py b/tests/test_response.py index 96c06ce6..6dcd2ea6 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -10,6 +10,7 @@ from random import choice from sanic import Sanic from sanic.response import HTTPResponse, stream, StreamingHTTPResponse, file, file_stream, json +from sanic.server import HttpProtocol from sanic.testing import HOST, PORT from unittest.mock import MagicMock @@ -30,9 +31,10 @@ def test_response_body_not_a_string(): async def sample_streaming_fn(response): - response.write('foo,') + await response.write('foo,') await asyncio.sleep(.001) - response.write('bar') + await response.write('bar') + def test_method_not_allowed(): @@ -189,20 +191,30 @@ def test_stream_response_includes_chunked_header(): def test_stream_response_writes_correct_content_to_transport(streaming_app): response = StreamingHTTPResponse(sample_streaming_fn) - response.transport = MagicMock(asyncio.Transport) + response.protocol = MagicMock(HttpProtocol) + response.protocol.transport = MagicMock(asyncio.Transport) + + async def mock_drain(): + pass + + def mock_push_data(data): + response.protocol.transport.write(data) + + response.protocol.push_data = mock_push_data + response.protocol.drain = mock_drain @streaming_app.listener('after_server_start') async def run_stream(app, loop): await response.stream() - assert response.transport.write.call_args_list[1][0][0] == ( + assert response.protocol.transport.write.call_args_list[1][0][0] == ( b'4\r\nfoo,\r\n' ) - assert response.transport.write.call_args_list[2][0][0] == ( + assert response.protocol.transport.write.call_args_list[2][0][0] == ( b'3\r\nbar\r\n' ) - assert response.transport.write.call_args_list[3][0][0] == ( + assert response.protocol.transport.write.call_args_list[3][0][0] == ( b'0\r\n\r\n' ) From 9082eb56a73d8bf2715fa9a46841567f9ac8173c Mon Sep 17 00:00:00 2001 From: Channel Cat Date: Thu, 6 Sep 2018 13:51:31 -0700 Subject: [PATCH 12/16] Update version to circumvent pypi upload errors --- sanic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanic/__init__.py b/sanic/__init__.py index 5e6ff4da..e675195a 100644 --- a/sanic/__init__.py +++ b/sanic/__init__.py @@ -1,6 +1,6 @@ from sanic.app import Sanic from sanic.blueprints import Blueprint -__version__ = '0.8.0' +__version__ = '0.8.1' __all__ = ['Sanic', 'Blueprint'] From 7ae0eb0dc3c28a459ee757c0fa6f17a0921b2ab5 Mon Sep 17 00:00:00 2001 From: Channel Cat Date: Thu, 13 Sep 2018 01:39:24 -0700 Subject: [PATCH 13/16] Transfer ownership --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 01801ddd..19b0a9f8 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ Sanic is a Flask-like Python 3.5+ web server that's written to go fast. It's ba On top of being Flask-like, Sanic supports async request handlers. This means you can use the new shiny async/await syntax from Python 3.5, making your code non-blocking and speedy. -Sanic is developed `on GitHub `_. Contributions are welcome! +Sanic is developed `on GitHub `_. Contributions are welcome! -If you have a project that utilizes Sanic make sure to comment on the `issue `_ that we use to track those projects! +If you have a project that utilizes Sanic make sure to comment on the `issue `_ that we use to track those projects! Hello World Example ------------------- @@ -47,8 +47,8 @@ Documentation .. |Join the chat at https://gitter.im/sanic-python/Lobby| image:: https://badges.gitter.im/sanic-python/Lobby.svg :target: https://gitter.im/sanic-python/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -.. |Build Status| image:: https://travis-ci.org/channelcat/sanic.svg?branch=master - :target: https://travis-ci.org/channelcat/sanic +.. |Build Status| image:: https://travis-ci.org/huge-success/sanic.svg?branch=master + :target: https://travis-ci.org/huge-success/sanic .. |Documentation| image:: https://readthedocs.org/projects/sanic/badge/?version=latest :target: http://sanic.readthedocs.io/en/latest/?badge=latest .. |PyPI| image:: https://img.shields.io/pypi/v/sanic.svg @@ -59,11 +59,11 @@ Documentation Examples -------- -`Non-Core examples `_. Examples of plugins and Sanic that are outside the scope of Sanic core. +`Non-Core examples `_. Examples of plugins and Sanic that are outside the scope of Sanic core. -`Extensions `_. Sanic extensions created by the community. +`Extensions `_. Sanic extensions created by the community. -`Projects `_. Sanic in production use. +`Projects `_. Sanic in production use. TODO From d38fc1719127851afad9be18b94dc845b5049f01 Mon Sep 17 00:00:00 2001 From: Channel Cat Date: Thu, 13 Sep 2018 01:50:32 -0700 Subject: [PATCH 14/16] Update version to test pypi --- sanic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanic/__init__.py b/sanic/__init__.py index e675195a..912a151c 100644 --- a/sanic/__init__.py +++ b/sanic/__init__.py @@ -1,6 +1,6 @@ from sanic.app import Sanic from sanic.blueprints import Blueprint -__version__ = '0.8.1' +__version__ = '0.8.2' __all__ = ['Sanic', 'Blueprint'] From 3e616b599a59c91bbaeef17145c400f96be9474d Mon Sep 17 00:00:00 2001 From: Channel Cat Date: Thu, 13 Sep 2018 02:17:27 -0700 Subject: [PATCH 15/16] update encrypted creds for new org --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c18e895b..83f13caf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ deploy: provider: pypi user: channelcat password: - secure: OgADRQH3+dTL5swGzXkeRJDNbLpFzwqYnXB4iLD0Npvzj9QnKyQVvkbaeq6VmV9dpEFb5ULaAKYQq19CrXYDm28yanUSn6jdJ4SukaHusi7xt07U6H7pmoX/uZ2WZYqCSLM8cSp8TXY/3oV3rY5Jfj/AibE5XTbim5/lrhsvW6NR+ALzxc0URRPAHDZEPpojTCjSTjpY0aDsaKWg4mXVRMFfY3O68j6KaIoukIZLuoHfePLKrbZxaPG5VxNhMHEaICdxVxE/dO+7pQmQxXuIsEOHK1QiVJ9YrSGcNqgEqhN36kYP8dqMeVB07sv8Xa6o/Uax2/wXS2HEJvuwP1YD6WkoZuo9ZB85bcMdg7BV9jJDbVFVPJwc75BnTLHrMa3Q1KrRlKRDBUXBUsQivPuWhFNwUgvEayq2qSI3aRQR4Z0O+DfboEhXYojSoD64/EWBTZ7vhgbvOTGEdukUQSYrKj9P8jc1s8exomTsAiqdFxTUpzfiammUSL+M93lP4urtahl1jjXFX7gd3DzdEEb0NsGkx5lm/qdsty8/TeAvKUmC+RVU6T856W6MqN0P+yGbpWUARcSE7fwztC3SPxwAuxvIN3BHmRhOUHoORPNG2VpfbnscIzBKJR4v0JKzbpi0IDa66K+tCGsCEvQuL4cxVOtoUySPWNSUAyUWWUrGM2k= + secure: h7oNDjA/ObDBGK7xt55SV0INHOclMJW/byxMrNxvCZ0JxiRk7WBNtWYt0WJjyf5lO/L0/sfgiAk0GIdFon57S24njSLPAq/a4ptkWZ68s2A+TaF6ezJSZvE9V8khivjoeub90TzfX6P5aukRja1CSxXKJm+v0V8hGE4CZGyCgEDvK3JqIakpXllSDl19DhVftCS/lQZD7AXrZlg1kZnPCMtB5IbCVR4L2bfrSJVNptBi2CqqxacY2MOLu+jv5FzJ2BGVIJ2zoIJS2T+JmGJzpiamF6y8Amv0667i9lg2DXWCtI3PsQzCmwa3F/ZsI+ohUAvJC5yvzP7SyTJyXifRBdJ9O137QkNAHFoJOOY3B4GSnTo8/boajKXEqGiV4h2EgwNjBaR0WJl0pB7HHUCBMkNRWqo6ACB8eCr04tXWXPvkGIc+wPjq960hsUZea1O31MuktYc9Ot6eiFqm7OKoItdi7LxCen1eTj93ePgkiEnVZ+p/04Hh1U7CX31UJMNu5kCvZPIANnAuDsS2SK7Qkr88OAuWL0wmrBcXKOcnVkJtZ5mzx8T54bI1RrSYtFDBLFfOPb0GucSziMBtQpE76qPEauVwIXBk3RnR8N57xBR/lvTaIk758tf+haO0llEO5rVls1zLNZ+VlTzXy7hX0OZbdopIAcCFBFWqWMAdXQc= on: tags: true distributions: "sdist bdist_wheel" From d8f9986089311c9fe07e1970f39043cf211af79a Mon Sep 17 00:00:00 2001 From: Channel Cat Date: Thu, 13 Sep 2018 02:24:31 -0700 Subject: [PATCH 16/16] Re-releasing with updated credentials --- sanic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanic/__init__.py b/sanic/__init__.py index 912a151c..51c8268e 100644 --- a/sanic/__init__.py +++ b/sanic/__init__.py @@ -1,6 +1,6 @@ from sanic.app import Sanic from sanic.blueprints import Blueprint -__version__ = '0.8.2' +__version__ = '0.8.3' __all__ = ['Sanic', 'Blueprint']