From 52c2a8484e6aa5fa13aaade49e1f2597dd006e15 Mon Sep 17 00:00:00 2001 From: Yaser Amiri Date: Thu, 7 Dec 2017 16:30:54 +0330 Subject: [PATCH 01/20] Add auto reloader. --- sanic/app.py | 21 +++++-- sanic/reloader_helpers.py | 120 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 sanic/reloader_helpers.py diff --git a/sanic/app.py b/sanic/app.py index 05d99c08..b0f51dc1 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -1,3 +1,4 @@ +import os import logging import logging.config import re @@ -22,6 +23,7 @@ from sanic.static import register as static_register from sanic.testing import SanicTestClient from sanic.views import CompositionView from sanic.websocket import WebSocketProtocol, ConnectionClosed +import sanic.reloader_helpers as reloader_helpers class Sanic: @@ -604,7 +606,7 @@ class Sanic: def run(self, host=None, port=None, debug=False, ssl=None, sock=None, workers=1, protocol=None, backlog=100, stop_event=None, register_sys_signals=True, - access_log=True): + access_log=True, auto_reload=False): """Run the HTTP Server and listen until keyboard interrupt or term signal. On termination, drain connections before closing. @@ -638,12 +640,16 @@ class Sanic: host=host, port=port, debug=debug, ssl=ssl, sock=sock, workers=workers, protocol=protocol, backlog=backlog, register_sys_signals=register_sys_signals, - access_log=access_log) + access_log=access_log, auto_reload=auto_reload) try: self.is_running = True if workers == 1: - serve(**server_settings) + if os.name == 'posix' and auto_reload and \ + os.environ.get('MAIN_PROCESS_RUNNED') != 'true': + reloader_helpers.watchdog(2) + else: + serve(**server_settings) else: serve_multiple(server_settings, workers) except BaseException: @@ -733,7 +739,8 @@ class Sanic: def _helper(self, host=None, port=None, debug=False, ssl=None, sock=None, workers=1, loop=None, protocol=HttpProtocol, backlog=100, stop_event=None, - register_sys_signals=True, run_async=False, access_log=True): + register_sys_signals=True, run_async=False, access_log=True, + auto_reload=False): """Helper function used by `run` and `create_server`.""" if isinstance(ssl, dict): # try common aliaseses @@ -799,14 +806,16 @@ class Sanic: if self.configure_logging and debug: logger.setLevel(logging.DEBUG) - if self.config.LOGO is not None: + + if self.config.LOGO is not None and \ + os.environ.get('MAIN_PROCESS_RUNNED') != 'true': logger.debug(self.config.LOGO) if run_async: server_settings['run_async'] = True # Serve - if host and port: + if host and port and os.environ.get('MAIN_PROCESS_RUNNED') != 'true': proto = "http" if ssl is not None: proto = "https" diff --git a/sanic/reloader_helpers.py b/sanic/reloader_helpers.py new file mode 100644 index 00000000..24c6fca9 --- /dev/null +++ b/sanic/reloader_helpers.py @@ -0,0 +1,120 @@ +import os +import sys +import signal +import subprocess +from time import sleep +from multiprocessing import Process + + +def _iter_module_files(): + """This iterates over all relevant Python files. It goes through all + loaded files from modules, all files in folders of already loaded modules + as well as all files reachable through a package. + """ + # The list call is necessary on Python 3 in case the module + # dictionary modifies during iteration. + for module in list(sys.modules.values()): + if module is None: + continue + filename = getattr(module, '__file__', None) + if filename: + old = None + while not os.path.isfile(filename): + old = filename + filename = os.path.dirname(filename) + if filename == old: + break + else: + if filename[-4:] in ('.pyc', '.pyo'): + filename = filename[:-1] + yield filename + + +def _get_args_for_reloading(): + """Returns the executable.""" + rv = [sys.executable] + rv.extend(sys.argv) + return rv + + +def restart_with_reloader(): + """Create a new process and a subprocess in it + with the same arguments as this one. + """ + args = _get_args_for_reloading() + new_environ = os.environ.copy() + new_environ['MAIN_PROCESS_RUNNED'] = 'true' + cmd = ' '.join(args) + worker_process = Process( + target=subprocess.call, args=(cmd,), + kwargs=dict(shell=True, env=new_environ)) + worker_process.start() + return worker_process + + +def kill_process_children_unix(pid): + """Find and kill child process of a process (maximum two level). + + :param pid: PID of process (process ID) + :return: Nothing + """ + root_proc_path = "/proc/%s/task/%s/children" % (pid, pid) + if not os.path.isfile(root_proc_path): + return + with open(root_proc_path) as children_list_file: + children_list_pid = children_list_file.read().split() + + for child_pid in children_list_pid: + children_proc_path = "/proc/%s/task/%s/children" % \ + (child_pid, child_pid) + if not os.path.isfile(children_proc_path): + continue + with open(children_proc_path) as children_list_file_2: + children_list_pid_2 = children_list_file_2.read().split() + for _pid in children_list_pid_2: + os.kill(int(_pid), signal.SIGTERM) + + +def kill_program_completly(proc): + """Kill worker and it's child processes and exit. + + :param proc: worker process (process ID) + :return: Nothing + """ + kill_process_children_unix(proc.pid) + proc.terminate() + os._exit(0) + + +def watchdog(sleep_interval): + """Whatch project files, restart worker process if a change happened. + + :param sleep_interval: interval in second. + :return: Nothing + """ + + mtimes = {} + worker_process = restart_with_reloader() + signal.signal( + signal.SIGTERM, lambda *args: kill_program_completly(worker_process)) + signal.signal( + signal.SIGINT, lambda *args: kill_program_completly(worker_process)) + while True: + for filename in _iter_module_files(): + try: + mtime = os.stat(filename).st_mtime + except OSError: + continue + + old_time = mtimes.get(filename) + if old_time is None: + mtimes[filename] = mtime + continue + elif mtime > old_time: + kill_process_children_unix(worker_process.pid) + worker_process = restart_with_reloader() + + mtimes[filename] = mtime + break + + sleep(sleep_interval) From 3fe3c2c79f65eff91bae1534356c928f84b78ee5 Mon Sep 17 00:00:00 2001 From: Yaser Amiri Date: Thu, 7 Dec 2017 20:19:40 +0330 Subject: [PATCH 02/20] Add test for auto reloading. --- sanic/app.py | 4 +- sanic/reloader_helpers.py | 6 +-- tests/test_auto_reload.py | 85 +++++++++++++++++++++++++++++++++++++++ tox.ini | 1 + 4 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 tests/test_auto_reload.py diff --git a/sanic/app.py b/sanic/app.py index b0f51dc1..90a305f9 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -646,7 +646,7 @@ class Sanic: self.is_running = True if workers == 1: if os.name == 'posix' and auto_reload and \ - os.environ.get('MAIN_PROCESS_RUNNED') != 'true': + os.environ.get('MAIN_PROCESS_RUNNED') != 'true': reloader_helpers.watchdog(2) else: serve(**server_settings) @@ -808,7 +808,7 @@ class Sanic: logger.setLevel(logging.DEBUG) if self.config.LOGO is not None and \ - os.environ.get('MAIN_PROCESS_RUNNED') != 'true': + os.environ.get('MAIN_PROCESS_RUNNED') != 'true': logger.debug(self.config.LOGO) if run_async: diff --git a/sanic/reloader_helpers.py b/sanic/reloader_helpers.py index 24c6fca9..ced62921 100644 --- a/sanic/reloader_helpers.py +++ b/sanic/reloader_helpers.py @@ -58,10 +58,10 @@ def kill_process_children_unix(pid): :param pid: PID of process (process ID) :return: Nothing """ - root_proc_path = "/proc/%s/task/%s/children" % (pid, pid) - if not os.path.isfile(root_proc_path): + root_process_path = "/proc/%s/task/%s/children" % (pid, pid) + if not os.path.isfile(root_process_path): return - with open(root_proc_path) as children_list_file: + with open(root_process_path) as children_list_file: children_list_pid = children_list_file.read().split() for child_pid in children_list_pid: diff --git a/tests/test_auto_reload.py b/tests/test_auto_reload.py new file mode 100644 index 00000000..de193042 --- /dev/null +++ b/tests/test_auto_reload.py @@ -0,0 +1,85 @@ +import os +import sys +import subprocess +import signal +from threading import Thread +import requests +from time import sleep + +sanic_project_content_one = ''' +from sanic import Sanic +from sanic import response + +app = Sanic(__name__) + + +@app.route("/") +async def test(request): + return response.json({"test": 1}) + + +if __name__ == '__main__': + app.run(host="127.0.0.1", port=8000, auto_reload=True) +''' + +sanic_project_content_two = ''' +from sanic import Sanic +from sanic import response + +app = Sanic(__name__) + + +@app.route("/") +async def test(request): + return response.json({"test": 2}) + + +if __name__ == '__main__': + app.run(host="127.0.0.1", port=8000, auto_reload=True) +''' + +process_id = None +def execute_cmd(command): + process = subprocess.Popen(command, shell=True) + global process_id + process_id = process.pid + process.communicate() + + +class TestAutoReloading: + + def test_reloading_after_change_file(self,capsys): + if os.name != 'posix': + return + + with capsys.disabled(): pass + sanic_app_file_path = "simple_sanic_app.py" + with open (sanic_app_file_path, "w") as _file: + _file.write(sanic_project_content_one) + + cmd = ' '.join([sys.executable, sanic_app_file_path]) + thread = Thread(target=execute_cmd, args=(cmd,)) + thread.start() + + sleep(2) # wait for completing server start process + response = requests.get("http://127.0.0.1:8000/").json() + assert response == {"test": 1} + + with open (sanic_app_file_path, "w") as _file: + _file.write(sanic_project_content_two) + sleep(2) # wait for completing server start process + response = requests.get("http://127.0.0.1:8000/").json() + assert response == {"test": 2} + + thread.join(1) + os.remove(sanic_app_file_path) + + def teardown_method(self, method): + if process_id: + root_proc_path = "/proc/%s/task/%s/children" % (process_id, process_id) + if not os.path.isfile(root_proc_path): + return + with open(root_proc_path) as children_list_file: + children_list_pid = children_list_file.read().split() + for child_pid in children_list_pid: + os.kill(int(child_pid), signal.SIGTERM) diff --git a/tox.ini b/tox.ini index ff43a139..6bd2c72d 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ deps = chardet<=2.3.0 beautifulsoup4 gunicorn + requests commands = pytest tests --cov sanic --cov-report= {posargs} - coverage combine --append From 81494453b0d6aae1355075e9b7528d541162b156 Mon Sep 17 00:00:00 2001 From: Yaser Amiri Date: Tue, 26 Dec 2017 19:17:13 +0330 Subject: [PATCH 03/20] Remove dependency on requests library. Change auto reloader enviroment varible name to SANIC_SERVER_RUNNING Fix some typo mistakes, flake uncompatibilities and such problems. Raise NotImplementedError for operating systems except posix systems for auto reloading. --- sanic/app.py | 13 +++++-- sanic/reloader_helpers.py | 15 ++++---- tests/test_auto_reload.py | 65 +++++++++++++++++++++----------- tests/test_keep_alive_timeout.py | 3 +- tests/test_middleware.py | 4 +- tests/test_request_data.py | 3 +- tests/test_request_stream.py | 3 +- tox.ini | 1 - 8 files changed, 68 insertions(+), 39 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index 90a305f9..bb012d33 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -645,8 +645,13 @@ class Sanic: try: self.is_running = True if workers == 1: - if os.name == 'posix' and auto_reload and \ - os.environ.get('MAIN_PROCESS_RUNNED') != 'true': + if auto_reload and os.name != 'posix': + # This condition must be removed after implementing + # auto reloader for other operating systems. + raise NotImplementedError + + if auto_reload and \ + os.environ.get('SANIC_SERVER_RUNNING') != 'true': reloader_helpers.watchdog(2) else: serve(**server_settings) @@ -808,14 +813,14 @@ class Sanic: logger.setLevel(logging.DEBUG) if self.config.LOGO is not None and \ - os.environ.get('MAIN_PROCESS_RUNNED') != 'true': + os.environ.get('SANIC_SERVER_RUNNING') != 'true': logger.debug(self.config.LOGO) if run_async: server_settings['run_async'] = True # Serve - if host and port and os.environ.get('MAIN_PROCESS_RUNNED') != 'true': + if host and port and os.environ.get('SANIC_SERVER_RUNNING') != 'true': proto = "http" if ssl is not None: proto = "https" diff --git a/sanic/reloader_helpers.py b/sanic/reloader_helpers.py index ced62921..e1349089 100644 --- a/sanic/reloader_helpers.py +++ b/sanic/reloader_helpers.py @@ -7,7 +7,9 @@ from multiprocessing import Process def _iter_module_files(): - """This iterates over all relevant Python files. It goes through all + """This iterates over all relevant Python files. + + It goes through all loaded files from modules, all files in folders of already loaded modules as well as all files reachable through a package. """ @@ -38,12 +40,12 @@ def _get_args_for_reloading(): def restart_with_reloader(): - """Create a new process and a subprocess in it - with the same arguments as this one. + """Create a new process and a subprocess in it with the same arguments as + this one. """ args = _get_args_for_reloading() new_environ = os.environ.copy() - new_environ['MAIN_PROCESS_RUNNED'] = 'true' + new_environ['SANIC_SERVER_RUNNING'] = 'true' cmd = ' '.join(args) worker_process = Process( target=subprocess.call, args=(cmd,), @@ -58,7 +60,7 @@ def kill_process_children_unix(pid): :param pid: PID of process (process ID) :return: Nothing """ - root_process_path = "/proc/%s/task/%s/children" % (pid, pid) + root_process_path = "/proc/{pid}/task/{pid}/children".format(pid=pid) if not os.path.isfile(root_process_path): return with open(root_process_path) as children_list_file: @@ -87,12 +89,11 @@ def kill_program_completly(proc): def watchdog(sleep_interval): - """Whatch project files, restart worker process if a change happened. + """Watch project files, restart worker process if a change happened. :param sleep_interval: interval in second. :return: Nothing """ - mtimes = {} worker_process = restart_with_reloader() signal.signal( diff --git a/tests/test_auto_reload.py b/tests/test_auto_reload.py index de193042..104f823b 100644 --- a/tests/test_auto_reload.py +++ b/tests/test_auto_reload.py @@ -3,10 +3,13 @@ import sys import subprocess import signal from threading import Thread -import requests from time import sleep +from json.decoder import JSONDecodeError +import aiohttp +import asyncio +import async_timeout -sanic_project_content_one = ''' +sanic_project_content_one = ''' from sanic import Sanic from sanic import response @@ -22,7 +25,7 @@ if __name__ == '__main__': app.run(host="127.0.0.1", port=8000, auto_reload=True) ''' -sanic_project_content_two = ''' +sanic_project_content_two = ''' from sanic import Sanic from sanic import response @@ -39,6 +42,8 @@ if __name__ == '__main__': ''' process_id = None + + def execute_cmd(command): process = subprocess.Popen(command, shell=True) global process_id @@ -48,38 +53,56 @@ def execute_cmd(command): class TestAutoReloading: - def test_reloading_after_change_file(self,capsys): + def check_response(self, url, response): + """Send http request and tries to take it's response as json. + Returns a dictionary. + """ + async def req(url, excepted_response): + async with aiohttp.ClientSession() as session: + with async_timeout.timeout(10): + async with session.get(url) as response: + try: + result = await response.json() + except JSONDecodeError: + result = {} + return result == excepted_response + + loop = asyncio.get_event_loop() + return loop.run_until_complete(req(url, response)) + + def test_reloading_after_change_file(self, capsys): if os.name != 'posix': return - with capsys.disabled(): pass + with capsys.disabled(): + pass sanic_app_file_path = "simple_sanic_app.py" - with open (sanic_app_file_path, "w") as _file: + with open(sanic_app_file_path, "w") as _file: _file.write(sanic_project_content_one) cmd = ' '.join([sys.executable, sanic_app_file_path]) thread = Thread(target=execute_cmd, args=(cmd,)) thread.start() - sleep(2) # wait for completing server start process - response = requests.get("http://127.0.0.1:8000/").json() - assert response == {"test": 1} + sleep(2) # wait for completing server start process + assert self.check_response("http://127.0.0.1:8000/", {"test": 1}) - with open (sanic_app_file_path, "w") as _file: + with open(sanic_app_file_path, "w") as _file: _file.write(sanic_project_content_two) - sleep(2) # wait for completing server start process - response = requests.get("http://127.0.0.1:8000/").json() - assert response == {"test": 2} + + sleep(2) # wait for completing server start process + assert self.check_response("http://127.0.0.1:8000/", {"test": 2}) thread.join(1) os.remove(sanic_app_file_path) def teardown_method(self, method): - if process_id: - root_proc_path = "/proc/%s/task/%s/children" % (process_id, process_id) - if not os.path.isfile(root_proc_path): - return - with open(root_proc_path) as children_list_file: - children_list_pid = children_list_file.read().split() - for child_pid in children_list_pid: - os.kill(int(child_pid), signal.SIGTERM) + if process_id: + root_proc_path = \ + "/proc/{pid}/task/{pid}/children".format(pid=process_id) + if not os.path.isfile(root_proc_path): + return + with open(root_proc_path) as children_list_file: + children_list_pid = children_list_file.read().split() + for child_pid in children_list_pid: + os.kill(int(child_pid), signal.SIGTERM) diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 15f6d705..3cb98771 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -68,7 +68,7 @@ class ReuseableSanicTestClient(SanicTestClient): import traceback traceback.print_tb(e2.__traceback__) exceptions.append(e2) - #Don't stop here! self.app.stop() + # Don't stop here! self.app.stop() if self._server is not None: _server = self._server @@ -266,4 +266,3 @@ def test_keep_alive_server_timeout(): assert isinstance(exception, ValueError) assert "Connection reset" in exception.args[0] or \ "got a new connection" in exception.args[0] - diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 4d4d6901..a879834d 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -75,6 +75,7 @@ def test_middleware_response_exception(): assert response.text == 'OK' assert result['status_code'] == 404 + def test_middleware_override_request(): app = Sanic('test_middleware_override_request') @@ -109,7 +110,6 @@ def test_middleware_override_response(): assert response.text == 'OK' - def test_middleware_order(): app = Sanic('test_middleware_order') @@ -146,4 +146,4 @@ def test_middleware_order(): request, response = app.test_client.get('/') assert response.status == 200 - assert order == [1,2,3,4,5,6] + assert order == [1, 2, 3, 4, 5, 6] diff --git a/tests/test_request_data.py b/tests/test_request_data.py index f795ff1f..f9349427 100644 --- a/tests/test_request_data.py +++ b/tests/test_request_data.py @@ -20,7 +20,8 @@ def test_storage(): @app.route('/') def handler(request): - return json({'user': request.get('user'), 'sidekick': request.get('sidekick')}) + return json({'user': request.get('user'), + 'sidekick': request.get('sidekick')}) request, response = app.test_client.get('/') diff --git a/tests/test_request_stream.py b/tests/test_request_stream.py index 4ca4e44e..8941c53e 100644 --- a/tests/test_request_stream.py +++ b/tests/test_request_stream.py @@ -188,7 +188,8 @@ def test_request_stream_handle_exception(): # 405 request, response = app.test_client.get('/post/random_id', data=data) assert response.status == 405 - assert response.text == 'Error: Method GET not allowed for URL /post/random_id' + assert response.text == \ + 'Error: Method GET not allowed for URL /post/random_id' def test_request_stream_blueprint(): diff --git a/tox.ini b/tox.ini index 6bd2c72d..ff43a139 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,6 @@ deps = chardet<=2.3.0 beautifulsoup4 gunicorn - requests commands = pytest tests --cov sanic --cov-report= {posargs} - coverage combine --append From 9bdf7a9980073a3e9fef95bc0c6b934851822200 Mon Sep 17 00:00:00 2001 From: Yaser Amiri Date: Tue, 26 Dec 2017 23:35:54 +0330 Subject: [PATCH 04/20] Revert files those fixed for flake problems. --- tests/test_keep_alive_timeout.py | 3 ++- tests/test_middleware.py | 4 ++-- tests/test_request_data.py | 3 +-- tests/test_request_stream.py | 3 +-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_keep_alive_timeout.py b/tests/test_keep_alive_timeout.py index 3cb98771..15f6d705 100644 --- a/tests/test_keep_alive_timeout.py +++ b/tests/test_keep_alive_timeout.py @@ -68,7 +68,7 @@ class ReuseableSanicTestClient(SanicTestClient): import traceback traceback.print_tb(e2.__traceback__) exceptions.append(e2) - # Don't stop here! self.app.stop() + #Don't stop here! self.app.stop() if self._server is not None: _server = self._server @@ -266,3 +266,4 @@ def test_keep_alive_server_timeout(): assert isinstance(exception, ValueError) assert "Connection reset" in exception.args[0] or \ "got a new connection" in exception.args[0] + diff --git a/tests/test_middleware.py b/tests/test_middleware.py index a879834d..4d4d6901 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -75,7 +75,6 @@ def test_middleware_response_exception(): assert response.text == 'OK' assert result['status_code'] == 404 - def test_middleware_override_request(): app = Sanic('test_middleware_override_request') @@ -110,6 +109,7 @@ def test_middleware_override_response(): assert response.text == 'OK' + def test_middleware_order(): app = Sanic('test_middleware_order') @@ -146,4 +146,4 @@ def test_middleware_order(): request, response = app.test_client.get('/') assert response.status == 200 - assert order == [1, 2, 3, 4, 5, 6] + assert order == [1,2,3,4,5,6] diff --git a/tests/test_request_data.py b/tests/test_request_data.py index f9349427..f795ff1f 100644 --- a/tests/test_request_data.py +++ b/tests/test_request_data.py @@ -20,8 +20,7 @@ def test_storage(): @app.route('/') def handler(request): - return json({'user': request.get('user'), - 'sidekick': request.get('sidekick')}) + return json({'user': request.get('user'), 'sidekick': request.get('sidekick')}) request, response = app.test_client.get('/') diff --git a/tests/test_request_stream.py b/tests/test_request_stream.py index 8941c53e..4ca4e44e 100644 --- a/tests/test_request_stream.py +++ b/tests/test_request_stream.py @@ -188,8 +188,7 @@ def test_request_stream_handle_exception(): # 405 request, response = app.test_client.get('/post/random_id', data=data) assert response.status == 405 - assert response.text == \ - 'Error: Method GET not allowed for URL /post/random_id' + assert response.text == 'Error: Method GET not allowed for URL /post/random_id' def test_request_stream_blueprint(): From ba1dbacd352845b107c227b1470506a2bc57760c Mon Sep 17 00:00:00 2001 From: Yaser Amiri Date: Sat, 20 Jan 2018 12:49:16 +0330 Subject: [PATCH 05/20] Change parsing cookies mechanism. (like Django instade of http.cookies.SimpleCookie) --- sanic/request.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index d4f6dc6f..7a35c4b1 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -1,8 +1,9 @@ import sys import json +import socket from cgi import parse_header from collections import namedtuple -from http.cookies import SimpleCookie +from http import cookies from httptools import parse_url from urllib.parse import parse_qs, urlunparse @@ -157,13 +158,17 @@ class Request(dict): def cookies(self): if self._cookies is None: cookie = self.headers.get('Cookie') + self._cookies = {} if cookie is not None: - cookies = SimpleCookie() - cookies.load(cookie) - self._cookies = {name: cookie.value - for name, cookie in cookies.items()} - else: - self._cookies = {} + for chunk in cookie.split(';'): + if '=' in chunk: + key, val = chunk.split('=', 1) + else: + key, val = '', chunk + key, val = key.strip(), val.strip() + if key or val: + self._cookies[key] = cookies._unquote(val) + return self._cookies @property @@ -181,13 +186,22 @@ class Request(dict): @property def socket(self): if not hasattr(self, '_socket'): - self._get_socket() + self._get_address() return self._socket def _get_address(self): - self._socket = (self.transport.get_extra_info('peername') or - (None, None)) - self._ip, self._port = self._socket + sock = self.transport.get_extra_info('socket') + + if sock.family == socket.AF_INET: + self._socket = (self.transport.get_extra_info('peername') or + (None, None)) + self._ip, self._port = self._socket + elif sock.family == socket.AF_INET6: + self._socket = (self.transport.get_extra_info('peername') or + (None, None, None, None)) + self._ip, self._port, *_ = self._socket + else: + self._ip, self._port = (None, None) @property def remote_addr(self): From f8b1122467b32a083b714627eb24ed6bc8a708f4 Mon Sep 17 00:00:00 2001 From: Yaser Amiri Date: Sun, 21 Jan 2018 09:10:15 +0330 Subject: [PATCH 06/20] Revert "Change parsing cookies mechanism. (like Django instade of http.cookies.SimpleCookie)" This reverts commit ba1dbacd352845b107c227b1470506a2bc57760c. --- sanic/request.py | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index 7a35c4b1..d4f6dc6f 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -1,9 +1,8 @@ import sys import json -import socket from cgi import parse_header from collections import namedtuple -from http import cookies +from http.cookies import SimpleCookie from httptools import parse_url from urllib.parse import parse_qs, urlunparse @@ -158,17 +157,13 @@ class Request(dict): def cookies(self): if self._cookies is None: cookie = self.headers.get('Cookie') - self._cookies = {} if cookie is not None: - for chunk in cookie.split(';'): - if '=' in chunk: - key, val = chunk.split('=', 1) - else: - key, val = '', chunk - key, val = key.strip(), val.strip() - if key or val: - self._cookies[key] = cookies._unquote(val) - + cookies = SimpleCookie() + cookies.load(cookie) + self._cookies = {name: cookie.value + for name, cookie in cookies.items()} + else: + self._cookies = {} return self._cookies @property @@ -186,22 +181,13 @@ class Request(dict): @property def socket(self): if not hasattr(self, '_socket'): - self._get_address() + self._get_socket() return self._socket def _get_address(self): - sock = self.transport.get_extra_info('socket') - - if sock.family == socket.AF_INET: - self._socket = (self.transport.get_extra_info('peername') or - (None, None)) - self._ip, self._port = self._socket - elif sock.family == socket.AF_INET6: - self._socket = (self.transport.get_extra_info('peername') or - (None, None, None, None)) - self._ip, self._port, *_ = self._socket - else: - self._ip, self._port = (None, None) + self._socket = (self.transport.get_extra_info('peername') or + (None, None)) + self._ip, self._port = self._socket @property def remote_addr(self): From 82cb182fe74cb56a21be8b27844d958c90274167 Mon Sep 17 00:00:00 2001 From: Alec Buckenheimer Date: Tue, 6 Feb 2018 09:57:16 -0500 Subject: [PATCH 07/20] added pip requirement to only install ujson and uvloop with cpython on non windows machines --- requirements-dev.txt | 4 ++-- requirements.txt | 4 ++-- setup.py | 18 ++++++------------ 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cf00aad4..ecadfc68 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,6 +7,6 @@ httptools flake8 pytest==3.3.2 tox -ujson -uvloop +ujson; sys_platform != "windows" and implementation_name == "cpython" +uvloop; sys_platform != "windows" and implementation_name == "cpython" gunicorn diff --git a/requirements.txt b/requirements.txt index e370b52f..fa18eabf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aiofiles httptools -ujson -uvloop +ujson; sys_platform != "windows" and implementation_name == "cpython" +uvloop; sys_platform != "windows" and implementation_name == "cpython" websockets diff --git a/setup.py b/setup.py index 45aa7b6f..bb8fbe4f 100644 --- a/setup.py +++ b/setup.py @@ -51,8 +51,9 @@ setup_kwargs = { ], } -ujson = 'ujson>=1.35' -uvloop = 'uvloop>=0.5.3' +env_dependency = '; sys_platform != "windows" and implementation_name == "cpython"' +ujson = 'ujson>=1.35' + env_dependency +uvloop = 'uvloop>=0.5.3' + env_dependency requirements = [ 'httptools>=0.0.9', @@ -66,16 +67,9 @@ if strtobool(os.environ.get("SANIC_NO_UJSON", "no")): requirements.remove(ujson) # 'nt' means windows OS -if strtobool(os.environ.get("SANIC_NO_UVLOOP", "no")) or os.name == 'nt': +if strtobool(os.environ.get("SANIC_NO_UVLOOP", "no")): print("Installing without uvLoop") requirements.remove(uvloop) -try: - setup_kwargs['install_requires'] = requirements - setup(**setup_kwargs) -except DistutilsPlatformError as exception: - requirements.remove(ujson) - requirements.remove(uvloop) - print("Installing without uJSON or uvLoop") - setup_kwargs['install_requires'] = requirements - setup(**setup_kwargs) +setup_kwargs['install_requires'] = requirements +setup(**setup_kwargs) From 5ef567405fdf7ef476fb4683fd8e455d4f762db2 Mon Sep 17 00:00:00 2001 From: Alec Buckenheimer Date: Tue, 6 Feb 2018 19:56:25 -0500 Subject: [PATCH 08/20] fixed platform from windows to win32 --- requirements-dev.txt | 4 ++-- requirements.txt | 4 ++-- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ecadfc68..d46aee12 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,6 +7,6 @@ httptools flake8 pytest==3.3.2 tox -ujson; sys_platform != "windows" and implementation_name == "cpython" -uvloop; sys_platform != "windows" and implementation_name == "cpython" +ujson; sys_platform != "win32" and implementation_name == "cpython" +uvloop; sys_platform != "win32" and implementation_name == "cpython" gunicorn diff --git a/requirements.txt b/requirements.txt index fa18eabf..faeec926 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aiofiles httptools -ujson; sys_platform != "windows" and implementation_name == "cpython" -uvloop; sys_platform != "windows" and implementation_name == "cpython" +ujson; sys_platform != "win32" and implementation_name == "cpython" +uvloop; sys_platform != "win32" and implementation_name == "cpython" websockets diff --git a/setup.py b/setup.py index bb8fbe4f..3f7812a2 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ setup_kwargs = { ], } -env_dependency = '; sys_platform != "windows" and implementation_name == "cpython"' +env_dependency = '; sys_platform != "win32" and implementation_name == "cpython"' ujson = 'ujson>=1.35' + env_dependency uvloop = 'uvloop>=0.5.3' + env_dependency From 60774c5a49abc7397aca36d51d30a211b586cba1 Mon Sep 17 00:00:00 2001 From: Timothy Ebiuwhe Date: Fri, 9 Feb 2018 22:27:20 +0100 Subject: [PATCH 09/20] Fixed bug that occurs on calling @app.route or any of it's variants causes a route to be added twice. One without the slash, the other with the Setting strict_slashes to false when a route does not end with slashes slash. This is ok if the Router._add method runs linearly, but problematic when it runs recursively. Unfortunately recursion is triggered when the host param to the Router._add function is a list of hosts. --- sanic/router.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sanic/router.py b/sanic/router.py index 052ff1bf..8b3fd3dc 100644 --- a/sanic/router.py +++ b/sanic/router.py @@ -128,6 +128,13 @@ class Router: if strict_slashes: return + if not isinstance(host, str) and host is not None: + # we have gotten back to the top of the recursion tree where the + # host was originally a list. By now, we've processed the strict + # slashes logic on the leaf nodes (the individual host strings in + # the list of host) + return + # Add versions with and without trailing / slashed_methods = self.routes_all.get(uri + '/', frozenset({})) unslashed_methods = self.routes_all.get(uri[:-1], frozenset({})) From 220b40f7f443f1050cbf26c6bf7f2a58730470ff Mon Sep 17 00:00:00 2001 From: Timothy Ebiuwhe Date: Fri, 9 Feb 2018 22:33:34 +0100 Subject: [PATCH 10/20] Added regression tests for issue #1120 --- tests/test_routes.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_routes.py b/tests/test_routes.py index 5f3b3376..ed35ea48 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -174,6 +174,40 @@ def test_route_optional_slash(): request, response = app.test_client.get('/get/') assert response.text == 'OK' +def test_route_strict_slashes_set_to_false_and_host_is_a_list(): + #Part of regression test for issue #1120 + app = Sanic('test_route_strict_slashes_set_to_false_and_host_is_a_list') + + site1 = 'localhost:{}'.format(app.test_client.port) + + #before fix, this raises a RouteExists error + @app.get('/get', host=[site1, 'site2.com'], strict_slashes=False) + def handler(request): + return text('OK') + + request, response = app.test_client.get('http://' + site1 + '/get') + assert response.text == 'OK' + + @app.post('/post', host=[site1, 'site2.com'], strict_slashes=False) + def handler(request): + return text('OK') + + request, response = app.test_client.post('http://' + site1 +'/post') + assert response.text == 'OK' + + @app.put('/put', host=[site1, 'site2.com'], strict_slashes=False) + def handler(request): + return text('OK') + + request, response = app.test_client.put('http://' + site1 +'/put') + assert response.text == 'OK' + + @app.delete('/delete', host=[site1, 'site2.com'], strict_slashes=False) + def handler(request): + return text('OK') + + request, response = app.test_client.delete('http://' + site1 +'/delete') + assert response.text == 'OK' def test_shorthand_routes_post(): app = Sanic('test_shorhand_routes_post') From 571b5b544d2c9533f63beaee95550b4567fe218a Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Fri, 9 Feb 2018 14:01:17 -0800 Subject: [PATCH 11/20] added app.register_listener method w/test --- sanic/app.py | 15 +++++++++++++++ tests/test_server_events.py | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/sanic/app.py b/sanic/app.py index 6e8377f5..42206b36 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -117,6 +117,21 @@ class Sanic: return decorator + def register_listener(self, listener, event): + """ + Register the listener for a given event. + + Args: + listener: callable i.e. setup_db(app, loop) + + event: when to register listener i.e. 'before_server_start' + + Returns: listener + + """ + + return self.listener(event)(listener) + # Decorator def route(self, uri, methods=frozenset({'GET'}), host=None, strict_slashes=None, stream=False, version=None, name=None): diff --git a/tests/test_server_events.py b/tests/test_server_events.py index ab0a1fb1..9c0d99eb 100644 --- a/tests/test_server_events.py +++ b/tests/test_server_events.py @@ -49,6 +49,23 @@ def test_single_listener(listener_name): assert random_name_app.name + listener_name == output.pop() +@pytest.mark.parametrize('listener_name', AVAILABLE_LISTENERS) +def test_register_listener(listener_name): + """ + Test that listeners on their own work with + app.register_listener method + """ + random_name_app = Sanic(''.join( + [choice(ascii_letters) for _ in range(choice(range(5, 10)))])) + output = list() + # Register listener + listener = create_listener(listener_name, output) + random_name_app.register_listener(listener, + event=listener_name) + start_stop_app(random_name_app) + assert random_name_app.name + listener_name == output.pop() + + def test_all_listeners(): random_name_app = Sanic(''.join( [choice(ascii_letters) for _ in range(choice(range(5, 10)))])) From 0fe0796870fcb62551a979dc7bcf7b4c20e3e1c2 Mon Sep 17 00:00:00 2001 From: Yun Xu Date: Fri, 9 Feb 2018 20:44:02 -0800 Subject: [PATCH 12/20] expose websocket protocol arguments --- sanic/app.py | 2 ++ sanic/config.py | 2 ++ sanic/server.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/sanic/app.py b/sanic/app.py index 6e8377f5..1513e467 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -807,6 +807,8 @@ class Sanic: 'access_log': access_log, 'websocket_max_size': self.config.WEBSOCKET_MAX_SIZE, 'websocket_max_queue': self.config.WEBSOCKET_MAX_QUEUE, + 'websocket_read_limit': self.config.WEBSOCKET_READ_LIMIT, + 'websocket_write_limit': self.config.WEBSOCKET_WRITE_LIMIT, 'graceful_shutdown_timeout': self.config.GRACEFUL_SHUTDOWN_TIMEOUT } diff --git a/sanic/config.py b/sanic/config.py index 922a9874..8e1f383c 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -36,6 +36,8 @@ class Config(dict): self.KEEP_ALIVE_TIMEOUT = 5 # 5 seconds self.WEBSOCKET_MAX_SIZE = 2 ** 20 # 1 megabytes self.WEBSOCKET_MAX_QUEUE = 32 + self.WEBSOCKET_READ_LIMIT = 2 ** 16 + self.WEBSOCKET_WRITE_LIMIT = 2 ** 16 self.GRACEFUL_SHUTDOWN_TIMEOUT = 15.0 # 15 sec if load_env: diff --git a/sanic/server.py b/sanic/server.py index 10a9040a..62193db5 100644 --- a/sanic/server.py +++ b/sanic/server.py @@ -514,6 +514,7 @@ def serve(host, port, request_handler, error_handler, before_start=None, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, + websocket_read_limit=2 ** 16, websocket_write_limit=2 ** 16, state=None, graceful_shutdown_timeout=15.0): """Start asynchronous HTTP Server on an individual process. @@ -574,6 +575,8 @@ def serve(host, port, request_handler, error_handler, before_start=None, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, + websocket_read_limit=websocket_read_limit, + websocket_write_limit=websocket_write_limit, state=state, debug=debug, ) From 745a1d6e94936b593fb5d93070a6e3d7c6eca800 Mon Sep 17 00:00:00 2001 From: Yun Xu Date: Fri, 9 Feb 2018 21:03:21 -0800 Subject: [PATCH 13/20] document websocket args --- sanic/server.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sanic/server.py b/sanic/server.py index 62193db5..15ae4708 100644 --- a/sanic/server.py +++ b/sanic/server.py @@ -544,6 +544,16 @@ def serve(host, port, request_handler, error_handler, before_start=None, :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log + :param websocket_max_size: enforces the maximum size for + incoming messages in bytes. + :param websocket_max_queue: sets the maximum length of the queue + that holds incoming messages. + :param websocket_read_limit: sets the high-water limit of the buffer for + incoming bytes, the low-water limit is half + the high-water limit. + :param websocket_write_limit: sets the high-water limit of the buffer for + outgoing bytes, the low-water limit is a + quarter of the high-water limit. :param is_request_stream: disable/enable Request.stream :param router: Router object :return: Nothing From 090df6f224b281a83e38003de7b8b7ccc486a486 Mon Sep 17 00:00:00 2001 From: Yun Xu Date: Fri, 9 Feb 2018 21:26:39 -0800 Subject: [PATCH 14/20] add websocket section in doc --- docs/sanic/websocket.rst | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/sanic/websocket.rst diff --git a/docs/sanic/websocket.rst b/docs/sanic/websocket.rst new file mode 100644 index 00000000..899d4ffb --- /dev/null +++ b/docs/sanic/websocket.rst @@ -0,0 +1,50 @@ +WebSocket +========= + +Sanic supports websockets, to setup a WebSocket: + +.. code:: python + + from sanic import Sanic + from sanic.response import json + from sanic.websocket import WebSocketProtocol + + app = Sanic() + + @app.websocket('/feed') + async def feed(request, ws): + while True: + data = 'hello!' + print('Sending: ' + data) + await ws.send(data) + data = await ws.recv() + print('Received: ' + data) + + if __name__ == "__main__": + app.run(host="0.0.0.0", port=8000, protocol=WebSocketProtocol) + + +Alternatively, the ``app.add_websocket_route`` method can be used instead of the +decorator: + +.. code:: python + async def feed(request, ws): + pass + + app.add_websocket_route(feed, '/feed') + + +Handlers for a WebSocket route are passed the request as first argument, and a +WebSocket protocol object as second argument. The protocol object has ``send`` +and ``recv`` methods to send and receive data respectively. + + +You could setup your own WebSocket configuration through ``app.config``, like + +.. code:: python + app.config.WEBSOCKET_MAX_SIZE = 2 ** 20 + app.config.WEBSOCKET_MAX_QUEUE = 32 + app.config.WEBSOCKET_READ_LIMIT = 2 ** 16 + app.config.WEBSOCKET_WRITE_LIMIT = 2 ** 16 + +Find more in ``Configuration`` section. From 2b70346db4f021763ce35896fc2039a0264d39ff Mon Sep 17 00:00:00 2001 From: Yun Xu Date: Fri, 9 Feb 2018 21:32:09 -0800 Subject: [PATCH 15/20] fix doc --- docs/sanic/websocket.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sanic/websocket.rst b/docs/sanic/websocket.rst index 899d4ffb..8b813bf8 100644 --- a/docs/sanic/websocket.rst +++ b/docs/sanic/websocket.rst @@ -28,6 +28,7 @@ Alternatively, the ``app.add_websocket_route`` method can be used instead of the decorator: .. code:: python + async def feed(request, ws): pass From dfc2166d8bdff08c4124fea42728152d3b77df51 Mon Sep 17 00:00:00 2001 From: Yun Xu Date: Sat, 10 Feb 2018 12:21:23 -0800 Subject: [PATCH 16/20] add websocket.rst to index.rst --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 9f4fa00c..aaa296ee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,7 @@ Guides sanic/exceptions sanic/middleware sanic/blueprints + sanic/websocket sanic/config sanic/cookies sanic/decorators From 1d75f6c2be5aa2aa6ab94d682696beeb8cfe9229 Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Tue, 13 Feb 2018 10:15:16 -0800 Subject: [PATCH 17/20] changed docstring spacing --- sanic/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index 42206b36..80344207 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -123,11 +123,9 @@ class Sanic: Args: listener: callable i.e. setup_db(app, loop) - event: when to register listener i.e. 'before_server_start' Returns: listener - """ return self.listener(event)(listener) From e911e2e1dfea7bc3e24675a13c8f691a04ac2bb9 Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Tue, 13 Feb 2018 23:58:03 -0800 Subject: [PATCH 18/20] updated doc --- docs/sanic/middleware.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/sanic/middleware.md b/docs/sanic/middleware.md index 228c9d47..eafe3c6f 100644 --- a/docs/sanic/middleware.md +++ b/docs/sanic/middleware.md @@ -100,6 +100,20 @@ async def close_db(app, loop): await app.db.close() ``` +It's also possible to register a listener using the `register_listener` method. +This may be useful if you define your listeners in another module besides +the one you instantiate your app in. + +```python +app = Sanic() + +async def setup_db(app, loop): + app.db = await db_setup() + +app.register_listener(setup_db, 'before_server_start') + +``` + If you want to schedule a background task to run after the loop has started, Sanic provides the `add_task` method to easily do so. From e735fe54c3779c3b481a5213d745f792f3983e3f Mon Sep 17 00:00:00 2001 From: panxb Date: Thu, 15 Feb 2018 00:11:37 +0800 Subject: [PATCH 19/20] raw requires a bytes-like object raw requires a bytes-like object, or an object that implements __bytes__, not 'str' --- docs/sanic/response.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sanic/response.md b/docs/sanic/response.md index 9c3c95f7..f322f8c8 100644 --- a/docs/sanic/response.md +++ b/docs/sanic/response.md @@ -91,7 +91,7 @@ from sanic import response @app.route('/raw') def handle_request(request): - return response.raw('raw data') + return response.raw(b'raw data') ``` ## Modify headers or status From 9c75ad3de1d08b2dc9f50c74e8a1b89f7ac628b7 Mon Sep 17 00:00:00 2001 From: Julien00859 Date: Wed, 21 Feb 2018 00:50:27 +0100 Subject: [PATCH 20/20] close #1136 --- sanic/handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanic/handlers.py b/sanic/handlers.py index 9afcfb94..81dd38d7 100644 --- a/sanic/handlers.py +++ b/sanic/handlers.py @@ -79,9 +79,9 @@ class ErrorHandler: response = None try: if handler: - response = handler(request=request, exception=exception) + response = handler(request, exception) if response is None: - response = self.default(request=request, exception=exception) + response = self.default(request, exception) except Exception: self.log(format_exc()) if self.debug: