From 2979e03148540d4d16e797ebb1999da58843d87d Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Mon, 11 Sep 2017 17:17:33 +1000 Subject: [PATCH 01/15] WIP - Split RequestTimeout, ResponseTimout, and KeepAliveTimeout into different timeouts, with different callbacks. --- sanic/app.py | 2 + sanic/config.py | 8 ++ sanic/exceptions.py | 14 +++ sanic/server.py | 136 +++++++++++++++++++++++------ tests/test_keep_alive_timeout.py | 142 +++++++++++++++++++++++++++++++ tests/test_request_timeout.py | 101 +++++++++++++++++----- tests/test_response_timeout.py | 38 +++++++++ 7 files changed, 395 insertions(+), 46 deletions(-) create mode 100644 tests/test_keep_alive_timeout.py create mode 100644 tests/test_response_timeout.py diff --git a/sanic/app.py b/sanic/app.py index 20c02a5c..4a2ea01c 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -745,6 +745,8 @@ class Sanic: 'request_handler': self.handle_request, 'error_handler': self.error_handler, 'request_timeout': self.config.REQUEST_TIMEOUT, + 'response_timeout': self.config.RESPONSE_TIMEOUT, + 'keep_alive_timeout': self.config.KEEP_ALIVE_TIMEOUT, 'request_max_size': self.config.REQUEST_MAX_SIZE, 'keep_alive': self.config.KEEP_ALIVE, 'loop': loop, diff --git a/sanic/config.py b/sanic/config.py index 0c2cc701..560fa2ec 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -125,7 +125,15 @@ class Config(dict): """ self.REQUEST_MAX_SIZE = 100000000 # 100 megabytes self.REQUEST_TIMEOUT = 60 # 60 seconds + self.RESPONSE_TIMEOUT = 60 # 60 seconds self.KEEP_ALIVE = keep_alive + # Apache httpd server default keepalive timeout = 5 seconds + # Nginx server default keepalive timeout = 75 seconds + # Nginx performance tuning guidelines uses keepalive timeout = 15 seconds + # IE client hard keepalive limit = 60 seconds + # Firefox client hard keepalive limit = 115 seconds + + self.KEEP_ALIVE_TIMEOUT = 5 # 5 seconds self.WEBSOCKET_MAX_SIZE = 2 ** 20 # 1 megabytes self.WEBSOCKET_MAX_QUEUE = 32 self.GRACEFUL_SHUTDOWN_TIMEOUT = 15.0 # 15 sec diff --git a/sanic/exceptions.py b/sanic/exceptions.py index 9663ea7c..e2d808f7 100644 --- a/sanic/exceptions.py +++ b/sanic/exceptions.py @@ -155,6 +155,13 @@ class ServerError(SanicException): pass +@add_status_code(503) +class ServiceUnavailable(SanicException): + """The server is currently unavailable (because it is overloaded or + down for maintenance). Generally, this is a temporary state.""" + pass + + class URLBuildError(ServerError): pass @@ -170,6 +177,13 @@ class FileNotFound(NotFound): @add_status_code(408) class RequestTimeout(SanicException): + """The Web server (running the Web site) thinks that there has been too + long an interval of time between 1) the establishment of an IP + connection (socket) between the client and the server and + 2) the receipt of any data on that socket, so the server has dropped + the connection. The socket connection has actually been lost - the Web + server has 'timed out' on that particular socket connection. + """ pass diff --git a/sanic/server.py b/sanic/server.py index f62ba654..bcef8a91 100644 --- a/sanic/server.py +++ b/sanic/server.py @@ -28,7 +28,8 @@ from sanic.log import log, netlog from sanic.response import HTTPResponse from sanic.request import Request from sanic.exceptions import ( - RequestTimeout, PayloadTooLarge, InvalidUsage, ServerError) + RequestTimeout, PayloadTooLarge, InvalidUsage, ServerError, + ServiceUnavailable) current_time = None @@ -63,16 +64,19 @@ class HttpProtocol(asyncio.Protocol): # request params 'parser', 'request', 'url', 'headers', # request config - 'request_handler', 'request_timeout', 'request_max_size', - 'request_class', 'is_request_stream', 'router', + 'request_handler', 'request_timeout', 'response_timeout', + 'keep_alive_timeout', 'request_max_size', 'request_class', + 'is_request_stream', 'router', # enable or disable access log / error log purpose 'has_log', # connection management - '_total_request_size', '_timeout_handler', '_last_communication_time', - '_is_stream_handler') + '_total_request_size', '_request_timeout_handler', + '_response_timeout_handler', '_keep_alive_timeout_handler', + '_last_request_time', '_last_response_time', '_is_stream_handler') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, + response_timeout=60, keep_alive_timeout=15, request_max_size=None, request_class=None, has_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): @@ -89,13 +93,18 @@ class HttpProtocol(asyncio.Protocol): self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout + self.response_timeout = response_timeout + self.keep_alive_timeout = keep_alive_timeout self.request_max_size = request_max_size self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False self._total_request_size = 0 - self._timeout_handler = None + self._request_timeout_handler = None + self._response_timeout_handler = None + self._keep_alive_timeout_handler = None self._last_request_time = None + self._last_response_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive @@ -118,22 +127,32 @@ class HttpProtocol(asyncio.Protocol): def connection_made(self, transport): self.connections.add(self) - self._timeout_handler = self.loop.call_later( - self.request_timeout, self.connection_timeout) + self._request_timeout_handler = self.loop.call_later( + self.request_timeout, self.request_timeout_callback) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) - self._timeout_handler.cancel() + if self._request_timeout_handler: + self._request_timeout_handler.cancel() + if self._response_timeout_handler: + self._response_timeout_handler.cancel() + if self._keep_alive_timeout_handler: + self._keep_alive_timeout_handler.cancel() - def connection_timeout(self): - # Check if + def request_timeout_callback(self): + # See the docstring in the RequestTimeout exception, to see + # exactly what this timeout is checking for. + # Check if elapsed time since request initiated exceeds our + # configured maximum request timeout value time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed - self._timeout_handler = ( - self.loop.call_later(time_left, self.connection_timeout)) + self._request_timeout_handler = ( + self.loop.call_later(time_left, + self.request_timeout_callback) + ) else: if self._request_stream_task: self._request_stream_task.cancel() @@ -144,6 +163,37 @@ class HttpProtocol(asyncio.Protocol): except RequestTimeout as exception: self.write_error(exception) + def response_timeout_callback(self): + # Check if elapsed time since response was initiated exceeds our + # configured maximum request timeout value + time_elapsed = current_time - self._last_request_time + if time_elapsed < self.response_timeout: + time_left = self.response_timeout - time_elapsed + self._response_timeout_handler = ( + self.loop.call_later(time_left, + self.response_timeout_callback) + ) + else: + try: + raise ServiceUnavailable('Response Timeout') + except ServiceUnavailable as exception: + self.write_error(exception) + + def keep_alive_timeout_callback(self): + # Check if elapsed time since last response exceeds our configured + # maximum keep alive timeout value + time_elapsed = current_time - self._last_response_time + if time_elapsed < self.keep_alive_timeout: + time_left = self.keep_alive_timeout - time_elapsed + self._keep_alive_timeout_handler = ( + self.loop.call_later(time_left, + self.keep_alive_timeout_callback) + ) + else: + log.info('KeepAlive Timeout. Closing connection.') + self.transport.close() + + # -------------------------------------------- # # Parsing # -------------------------------------------- # @@ -204,6 +254,11 @@ class HttpProtocol(asyncio.Protocol): method=self.parser.get_method().decode(), transport=self.transport ) + # Remove any existing KeepAlive handler here, + # It will be recreated if required on the new request. + if self._keep_alive_timeout_handler: + self._keep_alive_timeout_handler.cancel() + self._keep_alive_timeout_handler = None if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) @@ -219,6 +274,11 @@ class HttpProtocol(asyncio.Protocol): self.request.body.append(body) def on_message_complete(self): + # Entire request (headers and whole body) is received. + # We can cancel and remove the request timeout handler now. + if self._request_timeout_handler: + self._request_timeout_handler.cancel() + self._request_timeout_handler = None if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None)) @@ -227,6 +287,9 @@ class HttpProtocol(asyncio.Protocol): self.execute_request_handler() def execute_request_handler(self): + self._response_timeout_handler = self.loop.call_later( + self.response_timeout, self.response_timeout_callback) + self._last_request_time = current_time self._request_handler_task = self.loop.create_task( self.request_handler( self.request, @@ -240,12 +303,15 @@ class HttpProtocol(asyncio.Protocol): """ Writes response content synchronously to the transport. """ + if self._response_timeout_handler: + self._response_timeout_handler.cancel() + self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output( self.request.version, keep_alive, - self.request_timeout)) + self.keep_alive_timeout)) if self.has_log: netlog.info('', extra={ 'status': response.status, @@ -273,7 +339,10 @@ class HttpProtocol(asyncio.Protocol): if not keep_alive: self.transport.close() else: - self._last_request_time = current_time + self._keep_alive_timeout_handler = self.loop.call_later( + self.keep_alive_timeout, + self.keep_alive_timeout_callback) + self._last_response_time = current_time self.cleanup() async def stream_response(self, response): @@ -282,12 +351,14 @@ class HttpProtocol(asyncio.Protocol): the transport to the response so the response consumer can write to the response as needed. """ - + if self._response_timeout_handler: + self._response_timeout_handler.cancel() + self._response_timeout_handler = None try: keep_alive = self.keep_alive response.transport = self.transport await response.stream( - self.request.version, keep_alive, self.request_timeout) + self.request.version, keep_alive, self.keep_alive_timeout) if self.has_log: netlog.info('', extra={ 'status': response.status, @@ -315,10 +386,18 @@ class HttpProtocol(asyncio.Protocol): if not keep_alive: self.transport.close() else: - self._last_request_time = current_time + self._keep_alive_timeout_handler = self.loop.call_later( + self.keep_alive_timeout, + self.keep_alive_timeout_callback) + self._last_response_time = current_time self.cleanup() def write_error(self, exception): + # An error _is_ a response. + # Don't throw a response timeout, when a response _is_ given. + if self._response_timeout_handler: + self._response_timeout_handler.cancel() + self._response_timeout_handler = None response = None try: response = self.error_handler.response(self.request, exception) @@ -330,8 +409,9 @@ class HttpProtocol(asyncio.Protocol): self.request.ip if self.request else 'Unknown')) except Exception as e: self.bail_out( - "Writing error failed, connection closed {}".format(repr(e)), - from_error=True) + "Writing error failed, connection closed {}".format( + repr(e)), from_error=True + ) finally: if self.has_log: extra = dict() @@ -367,6 +447,9 @@ class HttpProtocol(asyncio.Protocol): log.error(message) def cleanup(self): + """This is called when KeepAlive feature is used, + it resets the connection in order for it to be able + to handle receiving another request on the same connection.""" self.parser = None self.request = None self.url = None @@ -421,12 +504,13 @@ def trigger_events(events, loop): def serve(host, port, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, - request_timeout=60, ssl=None, sock=None, request_max_size=None, - reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, + request_timeout=60, response_timeout=60, keep_alive_timeout=60, + ssl=None, sock=None, request_max_size=None, reuse_port=False, + loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_async=False, connections=None, - signal=Signal(), request_class=None, has_log=True, keep_alive=True, - is_request_stream=False, router=None, websocket_max_size=None, - websocket_max_queue=None, state=None, + signal=Signal(), request_class=None, has_log=True, + keep_alive=True, is_request_stream=False, router=None, + websocket_max_size=None, websocket_max_queue=None, state=None, graceful_shutdown_timeout=15.0): """Start asynchronous HTTP Server on an individual process. @@ -474,6 +558,8 @@ def serve(host, port, request_handler, error_handler, before_start=None, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, + response_timeout=response_timeout, + keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, has_log=has_log, diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py new file mode 100644 index 00000000..28030144 --- /dev/null +++ b/tests/test_keep_alive_timeout.py @@ -0,0 +1,142 @@ +from json import JSONDecodeError +from sanic import Sanic +from time import sleep as sync_sleep +import asyncio +from sanic.response import text +from sanic.config import Config +import aiohttp +from aiohttp import TCPConnector +from sanic.testing import SanicTestClient, HOST, PORT + + +class ReuseableTCPConnector(TCPConnector): + def __init__(self, *args, **kwargs): + super(ReuseableTCPConnector, self).__init__(*args, **kwargs) + self.conn = None + + @asyncio.coroutine + def connect(self, req): + if self.conn: + return self.conn + conn = yield from super(ReuseableTCPConnector, self).connect(req) + self.conn = conn + return conn + + def close(self): + return super(ReuseableTCPConnector, self).close() + + +class ReuseableSanicTestClient(SanicTestClient): + def __init__(self, app): + super(ReuseableSanicTestClient, self).__init__(app) + self._tcp_connector = None + self._session = None + + def _sanic_endpoint_test( + self, method='get', uri='/', gather_request=True, + debug=False, server_kwargs={}, + *request_args, **request_kwargs): + results = [None, None] + exceptions = [] + + if gather_request: + def _collect_request(request): + if results[0] is None: + results[0] = request + + self.app.request_middleware.appendleft(_collect_request) + + @self.app.listener('after_server_start') + async def _collect_response(sanic, loop): + try: + response = await self._local_request( + method, uri, *request_args, + **request_kwargs) + results[-1] = response + except Exception as e: + log.error( + 'Exception:\n{}'.format(traceback.format_exc())) + exceptions.append(e) + self.app.stop() + + server = self.app.create_server(host=HOST, debug=debug, port=PORT, **server_kwargs) + self.app.listeners['after_server_start'].pop() + + if exceptions: + raise ValueError( + "Exception during request: {}".format(exceptions)) + + if gather_request: + try: + request, response = results + return request, response + except: + raise ValueError( + "Request and response object expected, got ({})".format( + results)) + else: + try: + return results[-1] + except: + raise ValueError( + "Request object expected, got ({})".format(results)) + + async def _local_request(self, method, uri, cookies=None, *args, + **kwargs): + if uri.startswith(('http:', 'https:', 'ftp:', 'ftps://' '//')): + url = uri + else: + url = 'http://{host}:{port}{uri}'.format( + host=HOST, port=PORT, uri=uri) + if self._session: + session = self._session + else: + if self._tcp_connector: + conn = self._tcp_connector + else: + conn = ReuseableTCPConnector(verify_ssl=False) + self._tcp_connector = conn + session = aiohttp.ClientSession(cookies=cookies, + connector=conn) + self._session = session + + async with getattr(session, method.lower())( + url, *args, **kwargs) as response: + try: + response.text = await response.text() + except UnicodeDecodeError: + response.text = None + + try: + response.json = await response.json() + except (JSONDecodeError, + UnicodeDecodeError, + aiohttp.ClientResponseError): + response.json = None + + response.body = await response.read() + return response + + +Config.KEEP_ALIVE_TIMEOUT = 30 +Config.KEEP_ALIVE = True +keep_alive_timeout_app = Sanic('test_request_timeout') + + +@keep_alive_timeout_app.route('/1') +async def handler(request): + return text('OK') + + +def test_keep_alive_timeout(): + client = ReuseableSanicTestClient(keep_alive_timeout_app) + headers = { + 'Connection': 'keep-alive' + } + request, response = client.get('/1', headers=headers) + assert response.status == 200 + #sync_sleep(2) + request, response = client.get('/1') + assert response.status == 200 + + diff --git a/tests/test_request_timeout.py b/tests/test_request_timeout.py index 404aec12..e6c1f657 100644 --- a/tests/test_request_timeout.py +++ b/tests/test_request_timeout.py @@ -1,38 +1,97 @@ +from json import JSONDecodeError from sanic import Sanic import asyncio from sanic.response import text from sanic.exceptions import RequestTimeout from sanic.config import Config +import aiohttp +from aiohttp import TCPConnector +from sanic.testing import SanicTestClient, HOST, PORT + + +class DelayableTCPConnector(TCPConnector): + class DelayableHttpRequest(object): + def __new__(cls, req, delay): + cls = super(DelayableTCPConnector.DelayableHttpRequest, cls).\ + __new__(cls) + cls.req = req + cls.delay = delay + return cls + + def __getattr__(self, item): + return getattr(self.req, item) + + def send(self, *args, **kwargs): + if self.delay and self.delay > 0: + _ = yield from asyncio.sleep(self.delay) + self.req.send(*args, **kwargs) + + def __init__(self, *args, **kwargs): + _post_connect_delay = kwargs.pop('post_connect_delay', 0) + _pre_request_delay = kwargs.pop('pre_request_delay', 0) + super(DelayableTCPConnector, self).__init__(*args, **kwargs) + self._post_connect_delay = _post_connect_delay + self._pre_request_delay = _pre_request_delay + + @asyncio.coroutine + def connect(self, req): + req = DelayableTCPConnector.\ + DelayableHttpRequest(req, self._pre_request_delay) + conn = yield from super(DelayableTCPConnector, self).connect(req) + if self._post_connect_delay and self._post_connect_delay > 0: + _ = yield from asyncio.sleep(self._post_connect_delay) + return conn + + +class DelayableSanicTestClient(SanicTestClient): + def __init__(self, app, request_delay=1): + super(DelayableSanicTestClient, self).__init__(app) + self._request_delay = request_delay + + async def _local_request(self, method, uri, cookies=None, *args, + **kwargs): + if uri.startswith(('http:', 'https:', 'ftp:', 'ftps://' '//')): + url = uri + else: + url = 'http://{host}:{port}{uri}'.format( + host=HOST, port=PORT, uri=uri) + + conn = DelayableTCPConnector(pre_request_delay=self._request_delay, + verify_ssl=False) + async with aiohttp.ClientSession( + cookies=cookies, connector=conn) as session: + # Insert a delay after creating the connection + # But before sending the request. + + async with getattr(session, method.lower())( + url, *args, **kwargs) as response: + try: + response.text = await response.text() + except UnicodeDecodeError: + response.text = None + + try: + response.json = await response.json() + except (JSONDecodeError, + UnicodeDecodeError, + aiohttp.ClientResponseError): + response.json = None + + response.body = await response.read() + return response + Config.REQUEST_TIMEOUT = 1 -request_timeout_app = Sanic('test_request_timeout') request_timeout_default_app = Sanic('test_request_timeout_default') -@request_timeout_app.route('/1') -async def handler_1(request): - await asyncio.sleep(2) - return text('OK') - - -@request_timeout_app.exception(RequestTimeout) -def handler_exception(request, exception): - return text('Request Timeout from error_handler.', 408) - - -def test_server_error_request_timeout(): - request, response = request_timeout_app.test_client.get('/1') - assert response.status == 408 - assert response.text == 'Request Timeout from error_handler.' - - @request_timeout_default_app.route('/1') -async def handler_2(request): - await asyncio.sleep(2) +async def handler(request): return text('OK') def test_default_server_error_request_timeout(): - request, response = request_timeout_default_app.test_client.get('/1') + client = DelayableSanicTestClient(request_timeout_default_app, 2) + request, response = client.get('/1') assert response.status == 408 assert response.text == 'Error: Request Timeout' diff --git a/tests/test_response_timeout.py b/tests/test_response_timeout.py new file mode 100644 index 00000000..bf55a42e --- /dev/null +++ b/tests/test_response_timeout.py @@ -0,0 +1,38 @@ +from sanic import Sanic +import asyncio +from sanic.response import text +from sanic.exceptions import ServiceUnavailable +from sanic.config import Config + +Config.RESPONSE_TIMEOUT = 1 +response_timeout_app = Sanic('test_response_timeout') +response_timeout_default_app = Sanic('test_response_timeout_default') + + +@response_timeout_app.route('/1') +async def handler_1(request): + await asyncio.sleep(2) + return text('OK') + + +@response_timeout_app.exception(ServiceUnavailable) +def handler_exception(request, exception): + return text('Response Timeout from error_handler.', 503) + + +def test_server_error_response_timeout(): + request, response = response_timeout_app.test_client.get('/1') + assert response.status == 503 + assert response.text == 'Response Timeout from error_handler.' + + +@response_timeout_default_app.route('/1') +async def handler_2(request): + await asyncio.sleep(2) + return text('OK') + + +def test_default_server_error_response_timeout(): + request, response = response_timeout_default_app.test_client.get('/1') + assert response.status == 503 + assert response.text == 'Error: Response Timeout' From 1a74accd65cfe6a1c52d3697ab4552285fcd95c9 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Tue, 12 Sep 2017 13:09:42 +1000 Subject: [PATCH 02/15] finished the keepalive_timeout tests --- tests/test_keep_alive_timeout.py | 164 ++++++++++++++++++++++++++----- 1 file changed, 138 insertions(+), 26 deletions(-) diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 28030144..12e1629d 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -4,6 +4,7 @@ from time import sleep as sync_sleep import asyncio from sanic.response import text from sanic.config import Config +from sanic import server import aiohttp from aiohttp import TCPConnector from sanic.testing import SanicTestClient, HOST, PORT @@ -12,33 +13,40 @@ from sanic.testing import SanicTestClient, HOST, PORT class ReuseableTCPConnector(TCPConnector): def __init__(self, *args, **kwargs): super(ReuseableTCPConnector, self).__init__(*args, **kwargs) - self.conn = None + self.old_proto = None @asyncio.coroutine def connect(self, req): - if self.conn: - return self.conn - conn = yield from super(ReuseableTCPConnector, self).connect(req) - self.conn = conn - return conn - - def close(self): - return super(ReuseableTCPConnector, self).close() + new_conn = yield from super(ReuseableTCPConnector, self)\ + .connect(req) + if self.old_proto is not None: + if self.old_proto != new_conn.protocol: + raise RuntimeError( + "We got a new connection, wanted the same one!") + self.old_proto = new_conn.protocol + return new_conn class ReuseableSanicTestClient(SanicTestClient): - def __init__(self, app): + def __init__(self, app, loop=None): super(ReuseableSanicTestClient, self).__init__(app) + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop + self._server = None self._tcp_connector = None self._session = None + # Copied from SanicTestClient, but with some changes to reuse the + # same loop for the same app. def _sanic_endpoint_test( self, method='get', uri='/', gather_request=True, debug=False, server_kwargs={}, *request_args, **request_kwargs): + loop = self._loop results = [None, None] exceptions = [] - + do_kill_server = request_kwargs.pop('end_server', False) if gather_request: def _collect_request(request): if results[0] is None: @@ -47,26 +55,53 @@ class ReuseableSanicTestClient(SanicTestClient): self.app.request_middleware.appendleft(_collect_request) @self.app.listener('after_server_start') - async def _collect_response(sanic, loop): + async def _collect_response(loop): try: + if do_kill_server: + request_kwargs['end_session'] = True response = await self._local_request( method, uri, *request_args, **request_kwargs) results[-1] = response except Exception as e: - log.error( - 'Exception:\n{}'.format(traceback.format_exc())) + import traceback + traceback.print_tb(e.__traceback__) exceptions.append(e) - self.app.stop() + #Don't stop here! self.app.stop() - server = self.app.create_server(host=HOST, debug=debug, port=PORT, **server_kwargs) + if self._server is not None: + _server = self._server + else: + _server_co = self.app.create_server(host=HOST, debug=debug, + port=PORT, **server_kwargs) + + server.trigger_events( + self.app.listeners['before_server_start'], loop) + + try: + loop._stopping = False + http_server = loop.run_until_complete(_server_co) + except Exception as e: + raise e + self._server = _server = http_server + server.trigger_events( + self.app.listeners['after_server_start'], loop) self.app.listeners['after_server_start'].pop() + if do_kill_server: + try: + _server.close() + self._server = None + loop.run_until_complete(_server.wait_closed()) + self.app.stop() + except Exception as e: + exceptions.append(e) if exceptions: raise ValueError( "Exception during request: {}".format(exceptions)) if gather_request: + self.app.request_middleware.pop() try: request, response = results return request, response @@ -81,20 +116,29 @@ class ReuseableSanicTestClient(SanicTestClient): raise ValueError( "Request object expected, got ({})".format(results)) + # Copied from SanicTestClient, but with some changes to reuse the + # same TCPConnection and the sane ClientSession more than once. + # Note, you cannot use the same session if you are in a _different_ + # loop, so the changes above are required too. async def _local_request(self, method, uri, cookies=None, *args, **kwargs): + request_keepalive = kwargs.pop('request_keepalive', + Config.KEEP_ALIVE_TIMEOUT) if uri.startswith(('http:', 'https:', 'ftp:', 'ftps://' '//')): url = uri else: url = 'http://{host}:{port}{uri}'.format( host=HOST, port=PORT, uri=uri) + do_kill_session = kwargs.pop('end_session', False) if self._session: session = self._session else: if self._tcp_connector: conn = self._tcp_connector else: - conn = ReuseableTCPConnector(verify_ssl=False) + conn = ReuseableTCPConnector(verify_ssl=False, + keepalive_timeout= + request_keepalive) self._tcp_connector = conn session = aiohttp.ClientSession(cookies=cookies, connector=conn) @@ -115,28 +159,96 @@ class ReuseableSanicTestClient(SanicTestClient): response.json = None response.body = await response.read() - return response + if do_kill_session: + session.close() + self._session = None + return response -Config.KEEP_ALIVE_TIMEOUT = 30 +Config.KEEP_ALIVE_TIMEOUT = 2 Config.KEEP_ALIVE = True -keep_alive_timeout_app = Sanic('test_request_timeout') +keep_alive_timeout_app_reuse = Sanic('test_ka_timeout_reuse') +keep_alive_app_client_timeout = Sanic('test_ka_client_timeout') +keep_alive_app_server_timeout = Sanic('test_ka_server_timeout') -@keep_alive_timeout_app.route('/1') -async def handler(request): +@keep_alive_timeout_app_reuse.route('/1') +async def handler1(request): return text('OK') -def test_keep_alive_timeout(): - client = ReuseableSanicTestClient(keep_alive_timeout_app) +@keep_alive_app_client_timeout.route('/1') +async def handler2(request): + return text('OK') + + +@keep_alive_app_server_timeout.route('/1') +async def handler3(request): + return text('OK') + + +def test_keep_alive_timeout_reuse(): + """If the server keep-alive timeout and client keep-alive timeout are + both longer than the delay, the client _and_ server will successfully + reuse the existing connection.""" + loop = asyncio.get_event_loop() + client = ReuseableSanicTestClient(keep_alive_timeout_app_reuse, loop) headers = { 'Connection': 'keep-alive' } request, response = client.get('/1', headers=headers) assert response.status == 200 - #sync_sleep(2) - request, response = client.get('/1') + assert response.text == 'OK' + sync_sleep(1) + request, response = client.get('/1', end_server=True) assert response.status == 200 + assert response.text == 'OK' +def test_keep_alive_client_timeout(): + """If the server keep-alive timeout is longer than the client + keep-alive timeout, client will try to create a new connection here.""" + loop = asyncio.get_event_loop() + client = ReuseableSanicTestClient(keep_alive_app_client_timeout, + loop) + headers = { + 'Connection': 'keep-alive' + } + request, response = client.get('/1', headers=headers, + request_keepalive=1) + assert response.status == 200 + assert response.text == 'OK' + sync_sleep(3) + exception = None + try: + request, response = client.get('/1', end_server=True) + except ValueError as e: + exception = e + assert exception is not None + assert isinstance(exception, ValueError) + assert "got a new connection" in exception.args[0] + + +def test_keep_alive_server_timeout(): + """If the client keep-alive timeout is longer than the server + keep-alive timeout, the client will get a 'Connection reset' error.""" + loop = asyncio.get_event_loop() + client = ReuseableSanicTestClient(keep_alive_app_server_timeout, + loop) + headers = { + 'Connection': 'keep-alive' + } + request, response = client.get('/1', headers=headers, + request_keepalive=5) + assert response.status == 200 + assert response.text == 'OK' + sync_sleep(3) + exception = None + try: + request, response = client.get('/1', end_server=True) + except ValueError as e: + exception = e + assert exception is not None + assert isinstance(exception, ValueError) + assert "Connection reset" in exception.args[0] + From 173f94216a797041b54a8c4a84ca8e530dbdda4b Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Tue, 12 Sep 2017 13:40:43 +1000 Subject: [PATCH 03/15] Fixed the delays, and expected responses, in the keepalive_timeout tests --- tests/test_keep_alive_timeout.py | 50 ++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 12e1629d..09c51d00 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -1,7 +1,7 @@ from json import JSONDecodeError from sanic import Sanic -from time import sleep as sync_sleep import asyncio +from asyncio import sleep as aio_sleep from sanic.response import text from sanic.config import Config from sanic import server @@ -63,10 +63,8 @@ class ReuseableSanicTestClient(SanicTestClient): method, uri, *request_args, **request_kwargs) results[-1] = response - except Exception as e: - import traceback - traceback.print_tb(e.__traceback__) - exceptions.append(e) + except Exception as e2: + exceptions.append(e2) #Don't stop here! self.app.stop() if self._server is not None: @@ -81,8 +79,8 @@ class ReuseableSanicTestClient(SanicTestClient): try: loop._stopping = False http_server = loop.run_until_complete(_server_co) - except Exception as e: - raise e + except Exception as e1: + raise e1 self._server = _server = http_server server.trigger_events( self.app.listeners['after_server_start'], loop) @@ -94,8 +92,8 @@ class ReuseableSanicTestClient(SanicTestClient): self._server = None loop.run_until_complete(_server.wait_closed()) self.app.stop() - except Exception as e: - exceptions.append(e) + except Exception as e3: + exceptions.append(e3) if exceptions: raise ValueError( "Exception during request: {}".format(exceptions)) @@ -137,11 +135,13 @@ class ReuseableSanicTestClient(SanicTestClient): conn = self._tcp_connector else: conn = ReuseableTCPConnector(verify_ssl=False, + loop=self._loop, keepalive_timeout= request_keepalive) self._tcp_connector = conn session = aiohttp.ClientSession(cookies=cookies, - connector=conn) + connector=conn, + loop=self._loop) self._session = session async with getattr(session, method.lower())( @@ -191,7 +191,8 @@ def test_keep_alive_timeout_reuse(): """If the server keep-alive timeout and client keep-alive timeout are both longer than the delay, the client _and_ server will successfully reuse the existing connection.""" - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) client = ReuseableSanicTestClient(keep_alive_timeout_app_reuse, loop) headers = { 'Connection': 'keep-alive' @@ -199,7 +200,7 @@ def test_keep_alive_timeout_reuse(): request, response = client.get('/1', headers=headers) assert response.status == 200 assert response.text == 'OK' - sync_sleep(1) + loop.run_until_complete(aio_sleep(1)) request, response = client.get('/1', end_server=True) assert response.status == 200 assert response.text == 'OK' @@ -208,7 +209,8 @@ def test_keep_alive_timeout_reuse(): def test_keep_alive_client_timeout(): """If the server keep-alive timeout is longer than the client keep-alive timeout, client will try to create a new connection here.""" - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) client = ReuseableSanicTestClient(keep_alive_app_client_timeout, loop) headers = { @@ -218,10 +220,11 @@ def test_keep_alive_client_timeout(): request_keepalive=1) assert response.status == 200 assert response.text == 'OK' - sync_sleep(3) + loop.run_until_complete(aio_sleep(2)) exception = None try: - request, response = client.get('/1', end_server=True) + request, response = client.get('/1', end_server=True, + request_keepalive=1) except ValueError as e: exception = e assert exception is not None @@ -231,24 +234,29 @@ def test_keep_alive_client_timeout(): def test_keep_alive_server_timeout(): """If the client keep-alive timeout is longer than the server - keep-alive timeout, the client will get a 'Connection reset' error.""" - loop = asyncio.get_event_loop() + keep-alive timeout, the client will either a 'Connection reset' error + _or_ a new connection. Depending on how the event-loop handles the + broken server connection.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) client = ReuseableSanicTestClient(keep_alive_app_server_timeout, loop) headers = { 'Connection': 'keep-alive' } request, response = client.get('/1', headers=headers, - request_keepalive=5) + request_keepalive=60) assert response.status == 200 assert response.text == 'OK' - sync_sleep(3) + loop.run_until_complete(aio_sleep(3)) exception = None try: - request, response = client.get('/1', end_server=True) + request, response = client.get('/1', request_keepalive=60, + end_server=True) except ValueError as e: exception = e assert exception is not None assert isinstance(exception, ValueError) - assert "Connection reset" in exception.args[0] + assert "Connection reset" in exception.args[0] or \ + "got a new connection" in exception.args[0] From 8eb59ad4dc27261351ea9d1eb7f645ee4f0a9a40 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Wed, 13 Sep 2017 10:18:36 +1000 Subject: [PATCH 04/15] Fixed error where the RequestTimeout test wasn't actually testing the correct behaviour Fixed error where KeepAliveTimeout wasn't being triggered in the test suite, when using uvloop Fixed test cases when using other asyncio loops such as uvloop Fixed Flake8 linting errors --- sanic/config.py | 2 +- sanic/server.py | 1 - tests/test_keep_alive_timeout.py | 13 +++- tests/test_request_timeout.py | 102 +++++++++++++++++++++++++------ 4 files changed, 95 insertions(+), 23 deletions(-) diff --git a/sanic/config.py b/sanic/config.py index 560fa2ec..de91280f 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -129,7 +129,7 @@ class Config(dict): self.KEEP_ALIVE = keep_alive # Apache httpd server default keepalive timeout = 5 seconds # Nginx server default keepalive timeout = 75 seconds - # Nginx performance tuning guidelines uses keepalive timeout = 15 seconds + # Nginx performance tuning guidelines uses keepalive = 15 seconds # IE client hard keepalive limit = 60 seconds # Firefox client hard keepalive limit = 115 seconds diff --git a/sanic/server.py b/sanic/server.py index bcef8a91..eb9864cd 100644 --- a/sanic/server.py +++ b/sanic/server.py @@ -193,7 +193,6 @@ class HttpProtocol(asyncio.Protocol): log.info('KeepAlive Timeout. Closing connection.') self.transport.close() - # -------------------------------------------- # # Parsing # -------------------------------------------- # diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 09c51d00..15f6d705 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -20,10 +20,11 @@ class ReuseableTCPConnector(TCPConnector): new_conn = yield from super(ReuseableTCPConnector, self)\ .connect(req) if self.old_proto is not None: - if self.old_proto != new_conn.protocol: + if self.old_proto != new_conn._protocol: raise RuntimeError( "We got a new connection, wanted the same one!") - self.old_proto = new_conn.protocol + print(new_conn.__dict__) + self.old_proto = new_conn._protocol return new_conn @@ -64,6 +65,8 @@ class ReuseableSanicTestClient(SanicTestClient): **request_kwargs) results[-1] = response except Exception as e2: + import traceback + traceback.print_tb(e2.__traceback__) exceptions.append(e2) #Don't stop here! self.app.stop() @@ -80,6 +83,8 @@ class ReuseableSanicTestClient(SanicTestClient): loop._stopping = False http_server = loop.run_until_complete(_server_co) except Exception as e1: + import traceback + traceback.print_tb(e1.__traceback__) raise e1 self._server = _server = http_server server.trigger_events( @@ -93,7 +98,9 @@ class ReuseableSanicTestClient(SanicTestClient): loop.run_until_complete(_server.wait_closed()) self.app.stop() except Exception as e3: - exceptions.append(e3) + import traceback + traceback.print_tb(e3.__traceback__) + exceptions.append(e3) if exceptions: raise ValueError( "Exception during request: {}".format(exceptions)) diff --git a/tests/test_request_timeout.py b/tests/test_request_timeout.py index e6c1f657..a1d8a885 100644 --- a/tests/test_request_timeout.py +++ b/tests/test_request_timeout.py @@ -1,8 +1,8 @@ from json import JSONDecodeError + from sanic import Sanic import asyncio from sanic.response import text -from sanic.exceptions import RequestTimeout from sanic.config import Config import aiohttp from aiohttp import TCPConnector @@ -10,21 +10,68 @@ from sanic.testing import SanicTestClient, HOST, PORT class DelayableTCPConnector(TCPConnector): - class DelayableHttpRequest(object): + + class RequestContextManager(object): def __new__(cls, req, delay): - cls = super(DelayableTCPConnector.DelayableHttpRequest, cls).\ + cls = super(DelayableTCPConnector.RequestContextManager, cls).\ __new__(cls) cls.req = req + cls.send_task = None + cls.resp = None + cls.orig_send = getattr(req, 'send') + cls.orig_start = None cls.delay = delay + cls._acting_as = req return cls def __getattr__(self, item): - return getattr(self.req, item) + acting_as = self._acting_as + return getattr(acting_as, item) + + @asyncio.coroutine + def start(self, connection, read_until_eof=False): + if self.send_task is None: + raise RuntimeError("do a send() before you do a start()") + resp = yield from self.send_task + self.send_task = None + self.resp = resp + self._acting_as = self.resp + self.orig_start = getattr(resp, 'start') + + try: + ret = yield from self.orig_start(connection, + read_until_eof) + except Exception as e: + raise e + return ret + + def close(self): + if self.resp is not None: + self.resp.close() + if self.send_task is not None: + self.send_task.cancel() + + @asyncio.coroutine + def delayed_send(self, *args, **kwargs): + req = self.req + if self.delay and self.delay > 0: + #sync_sleep(self.delay) + _ = yield from asyncio.sleep(self.delay) + t = req.loop.time() + print("sending at {}".format(t), flush=True) + conn = next(iter(args)) # first arg is connection + try: + delayed_resp = self.orig_send(*args, **kwargs) + except Exception as e: + return aiohttp.ClientResponse(req.method, req.url) + return delayed_resp def send(self, *args, **kwargs): - if self.delay and self.delay > 0: - _ = yield from asyncio.sleep(self.delay) - self.req.send(*args, **kwargs) + gen = self.delayed_send(*args, **kwargs) + task = self.req.loop.create_task(gen) + self.send_task = task + self._acting_as = task + return self def __init__(self, *args, **kwargs): _post_connect_delay = kwargs.pop('post_connect_delay', 0) @@ -35,31 +82,37 @@ class DelayableTCPConnector(TCPConnector): @asyncio.coroutine def connect(self, req): - req = DelayableTCPConnector.\ - DelayableHttpRequest(req, self._pre_request_delay) + d_req = DelayableTCPConnector.\ + RequestContextManager(req, self._pre_request_delay) conn = yield from super(DelayableTCPConnector, self).connect(req) if self._post_connect_delay and self._post_connect_delay > 0: - _ = yield from asyncio.sleep(self._post_connect_delay) + _ = yield from asyncio.sleep(self._post_connect_delay, + loop=self._loop) + req.send = d_req.send + t = req.loop.time() + print("Connected at {}".format(t), flush=True) return conn class DelayableSanicTestClient(SanicTestClient): - def __init__(self, app, request_delay=1): + def __init__(self, app, loop, request_delay=1): super(DelayableSanicTestClient, self).__init__(app) self._request_delay = request_delay + self._loop = None async def _local_request(self, method, uri, cookies=None, *args, **kwargs): + if self._loop is None: + self._loop = asyncio.get_event_loop() if uri.startswith(('http:', 'https:', 'ftp:', 'ftps://' '//')): url = uri else: url = 'http://{host}:{port}{uri}'.format( host=HOST, port=PORT, uri=uri) - conn = DelayableTCPConnector(pre_request_delay=self._request_delay, - verify_ssl=False) - async with aiohttp.ClientSession( - cookies=cookies, connector=conn) as session: + verify_ssl=False, loop=self._loop) + async with aiohttp.ClientSession(cookies=cookies, connector=conn, + loop=self._loop) as session: # Insert a delay after creating the connection # But before sending the request. @@ -81,17 +134,30 @@ class DelayableSanicTestClient(SanicTestClient): return response -Config.REQUEST_TIMEOUT = 1 +Config.REQUEST_TIMEOUT = 2 request_timeout_default_app = Sanic('test_request_timeout_default') +request_no_timeout_app = Sanic('test_request_no_timeout') @request_timeout_default_app.route('/1') -async def handler(request): +async def handler1(request): + return text('OK') + + +@request_no_timeout_app.route('/1') +async def handler2(request): return text('OK') def test_default_server_error_request_timeout(): - client = DelayableSanicTestClient(request_timeout_default_app, 2) + client = DelayableSanicTestClient(request_timeout_default_app, None, 3) request, response = client.get('/1') assert response.status == 408 assert response.text == 'Error: Request Timeout' + + +def test_default_server_error_request_dont_timeout(): + client = DelayableSanicTestClient(request_no_timeout_app, None, 1) + request, response = client.get('/1') + assert response.status == 200 + assert response.text == 'OK' From 12dafd07b87e3412f48cfb3672c1804b3879464c Mon Sep 17 00:00:00 2001 From: huyuhan Date: Fri, 15 Sep 2017 18:34:56 +0800 Subject: [PATCH 05/15] add __repr__ for sanic request --- sanic/request.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sanic/request.py b/sanic/request.py index 27ff011e..4e8a2e07 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -68,6 +68,11 @@ class Request(dict): self._cookies = None self.stream = None + def __repr__(self): + if self.method is None or not self._parsed_url: + return '<%s>' % self.__class__.__name__ + return '<%s: %s %r>' % (self.__class__.__name__, self.method, self.path) + @property def json(self): if self.parsed_json is None: From 77f70a0792eef25841039dcd1e2266f2d78e79f2 Mon Sep 17 00:00:00 2001 From: huyuhan Date: Fri, 15 Sep 2017 20:56:44 +0800 Subject: [PATCH 06/15] add __repr__ for sanic request --- sanic/request.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index 4e8a2e07..fa80b47c 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -19,8 +19,9 @@ except ImportError: from sanic.exceptions import InvalidUsage from sanic.log import log - DEFAULT_HTTP_CONTENT_TYPE = "application/octet-stream" + + # HTTP/1.1: https://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7.2.1 # > If the media type remains unknown, the recipient SHOULD treat it # > as type "application/octet-stream" @@ -69,9 +70,9 @@ class Request(dict): self.stream = None def __repr__(self): - if self.method is None or not self._parsed_url: + if self.method is None or not self.path: return '<%s>' % self.__class__.__name__ - return '<%s: %s %r>' % (self.__class__.__name__, self.method, self.path) + return '<%s: %s %s>' % (self.__class__.__name__, self.method, self.path) @property def json(self): @@ -175,8 +176,8 @@ class Request(dict): remote_addrs = [ addr for addr in [ addr.strip() for addr in forwarded_for - ] if addr - ] + ] if addr + ] if len(remote_addrs) > 0: self._remote_addr = remote_addrs[0] else: From f6eb35f67d6637fdd4b13e4d38ba17d4d21f1fb3 Mon Sep 17 00:00:00 2001 From: huyuhan Date: Fri, 15 Sep 2017 21:05:25 +0800 Subject: [PATCH 07/15] add __repr__ for sanic request --- sanic/request.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index fa80b47c..ea5c071e 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -71,8 +71,10 @@ class Request(dict): def __repr__(self): if self.method is None or not self.path: - return '<%s>' % self.__class__.__name__ - return '<%s: %s %s>' % (self.__class__.__name__, self.method, self.path) + return '<{class_name}>'.format(class_name=self.__class__.__name__) + return '<{class_name}: {method} {path}>'.format(class_name=self.__class__.__name__, + method=self.method, + path=self.path) @property def json(self): From 074d36eeba5a793b0c87becfda81603fcc5cb824 Mon Sep 17 00:00:00 2001 From: huyuhan Date: Fri, 15 Sep 2017 21:15:05 +0800 Subject: [PATCH 08/15] add __repr__ for sanic request --- sanic/request.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index ea5c071e..26b14fd9 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -71,10 +71,10 @@ class Request(dict): def __repr__(self): if self.method is None or not self.path: - return '<{class_name}>'.format(class_name=self.__class__.__name__) - return '<{class_name}: {method} {path}>'.format(class_name=self.__class__.__name__, - method=self.method, - path=self.path) + return '<{0}>'.format(self.__class__.__name__) + return '<{0}: {1} {2}>'.format(self.__class__.__name__, + self.method, + self.path) @property def json(self): From 1d719252cbc6ce79943e76d317cecd57b1872fe4 Mon Sep 17 00:00:00 2001 From: Hugh McNamara Date: Tue, 19 Sep 2017 14:58:49 +0100 Subject: [PATCH 09/15] use dependency injection to allow alternative json parser or encoder --- sanic/request.py | 4 ++-- sanic/response.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index 27ff011e..aa778a8d 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -69,10 +69,10 @@ class Request(dict): self.stream = None @property - def json(self): + def json(self, loads=json_loads): if self.parsed_json is None: try: - self.parsed_json = json_loads(self.body) + self.parsed_json = loads(self.body) except Exception: if not self.body: return None diff --git a/sanic/response.py b/sanic/response.py index 3b0ef449..f661758b 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -237,7 +237,8 @@ class HTTPResponse(BaseHTTPResponse): def json(body, status=200, headers=None, - content_type="application/json", **kwargs): + content_type="application/json", dumps=json_dumps, + **kwargs): """ Returns response object with body in json format. @@ -246,7 +247,7 @@ def json(body, status=200, headers=None, :param headers: Custom Headers. :param kwargs: Remaining arguments that are passed to the json encoder. """ - return HTTPResponse(json_dumps(body, **kwargs), headers=headers, + return HTTPResponse(dumps(body, **kwargs), headers=headers, status=status, content_type=content_type) From a8f764c1612ab17eb99924ac2363c05110a1911d Mon Sep 17 00:00:00 2001 From: Hugh McNamara Date: Tue, 19 Sep 2017 18:12:53 +0100 Subject: [PATCH 10/15] make method instead of property for alternative json decoding of request --- sanic/request.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index aa778a8d..74200fe0 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -69,10 +69,10 @@ class Request(dict): self.stream = None @property - def json(self, loads=json_loads): + def json(self): if self.parsed_json is None: try: - self.parsed_json = loads(self.body) + self.parsed_json = json_loads(self.body) except Exception: if not self.body: return None @@ -80,6 +80,16 @@ class Request(dict): return self.parsed_json + def load_json(self, loads=json_loads): + try: + self.parsed_json = loads(self.body) + except Exception: + if not self.body: + return None + raise InvalidUsage("Failed when parsing body as json") + + return self.parsed_json + @property def token(self): """Attempt to return the auth header token. From 5cef1634edd029e914d0184627e98bcb63198173 Mon Sep 17 00:00:00 2001 From: Hugh McNamara Date: Fri, 22 Sep 2017 10:19:15 +0100 Subject: [PATCH 11/15] use json_loads function in json property of request --- sanic/request.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index 74200fe0..43ecc511 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -71,12 +71,7 @@ class Request(dict): @property def json(self): if self.parsed_json is None: - try: - self.parsed_json = json_loads(self.body) - except Exception: - if not self.body: - return None - raise InvalidUsage("Failed when parsing body as json") + self.load_json() return self.parsed_json From f96ab027677d525d1a690223d217c27cc05517e9 Mon Sep 17 00:00:00 2001 From: lixxu Date: Wed, 27 Sep 2017 09:59:49 +0800 Subject: [PATCH 12/15] set scheme to http if not provided --- sanic/app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sanic/app.py b/sanic/app.py index 3776c915..5d80b360 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -460,7 +460,10 @@ class Sanic: if external: if not scheme: - scheme = netloc[:8].split(':', 1)[0] + if ':' in netloc[:8]: + scheme = netloc[:8].split(':', 1)[0] + else: + scheme = 'http' if '://' in netloc[:8]: netloc = netloc.split('://', 1)[-1] From 91b2167ebac836b54205e7e9a031597bd3ea6ee6 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 27 Sep 2017 11:07:06 +0300 Subject: [PATCH 13/15] Update extensions.md Add - [JWT](https://github.com/ahopkins/sanic-jwt): Authentication extension for JSON Web Tokens (JWT) extension package. --- docs/sanic/extensions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sanic/extensions.md b/docs/sanic/extensions.md index 5643f4fc..ad9b8156 100644 --- a/docs/sanic/extensions.md +++ b/docs/sanic/extensions.md @@ -7,6 +7,7 @@ A list of Sanic extensions created by the community. - [CORS](https://github.com/ashleysommer/sanic-cors): A port of flask-cors. - [Compress](https://github.com/subyraman/sanic_compress): Allows you to easily gzip Sanic responses. A port of Flask-Compress. - [Jinja2](https://github.com/lixxu/sanic-jinja2): Support for Jinja2 template. +- [JWT](https://github.com/ahopkins/sanic-jwt): Authentication extension for JSON Web Tokens (JWT). - [OpenAPI/Swagger](https://github.com/channelcat/sanic-openapi): OpenAPI support, plus a Swagger UI. - [Pagination](https://github.com/lixxu/python-paginate): Simple pagination support. - [Motor](https://github.com/lixxu/sanic-motor): Simple motor wrapper. From 9aec5febb8fc33972f0ff13bed9c11e6865ec4ab Mon Sep 17 00:00:00 2001 From: Raphael Deem Date: Wed, 27 Sep 2017 01:24:43 -0700 Subject: [PATCH 14/15] support vhosts in static routes --- sanic/app.py | 4 ++-- sanic/static.py | 4 ++-- tests/test_static.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index 5d80b360..d553f09d 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -354,13 +354,13 @@ class Sanic: # Static Files def static(self, uri, file_or_directory, pattern=r'/?.+', use_modified_since=True, use_content_range=False, - stream_large_files=False, name='static'): + stream_large_files=False, name='static', host=None): """Register a root to serve files from. The input can either be a file or a directory. See """ static_register(self, uri, file_or_directory, pattern, use_modified_since, use_content_range, - stream_large_files, name) + stream_large_files, name, host) def blueprint(self, blueprint, **options): """Register a blueprint on the application. diff --git a/sanic/static.py b/sanic/static.py index a9683b27..1ebd7291 100644 --- a/sanic/static.py +++ b/sanic/static.py @@ -18,7 +18,7 @@ from sanic.response import file, file_stream, HTTPResponse def register(app, uri, file_or_directory, pattern, use_modified_since, use_content_range, - stream_large_files, name='static'): + stream_large_files, name='static', host=None): # TODO: Though sanic is not a file server, I feel like we should at least # make a good effort here. Modified-since is nice, but we could # also look into etags, expires, and caching @@ -122,4 +122,4 @@ def register(app, uri, file_or_directory, pattern, if not name.startswith('_static_'): name = '_static_{}'.format(name) - app.route(uri, methods=['GET', 'HEAD'], name=name)(_handler) + app.route(uri, methods=['GET', 'HEAD'], name=name, host=host)(_handler) diff --git a/tests/test_static.py b/tests/test_static.py index 091d63a4..6252b1c1 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -161,3 +161,20 @@ def test_static_content_range_error(file_name, static_file_directory): assert 'Content-Range' in response.headers assert response.headers['Content-Range'] == "bytes */%s" % ( len(get_file_content(static_file_directory, file_name)),) + + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) +def test_static_file(static_file_directory, file_name): + app = Sanic('test_static') + app.static( + '/testing.file', + get_file_path(static_file_directory, file_name), + host="www.example.com" + ) + + headers = {"Host": "www.example.com"} + request, response = app.test_client.get('/testing.file', headers=headers) + assert response.status == 200 + assert response.body == get_file_content(static_file_directory, file_name) + request, response = app.test_client.get('/testing.file') + assert response.status == 404 From 62871ec9b31345863f5c61e5b52c21c98cdabd32 Mon Sep 17 00:00:00 2001 From: lanf0n Date: Sat, 30 Sep 2017 01:16:26 +0800 Subject: [PATCH 15/15] add sphinx extension to add asyncio-specific markups --- docs/conf.py | 2 +- requirements-docs.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index e254c183..7dd7462c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ import sanic # -- General configuration ------------------------------------------------ -extensions = ['sphinx.ext.autodoc'] +extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.asyncio'] templates_path = ['_templates'] diff --git a/requirements-docs.txt b/requirements-docs.txt index efa74079..e12c1846 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,3 +1,4 @@ sphinx sphinx_rtd_theme recommonmark +sphinxcontrib-asyncio