Merge pull request #18 from channelcat/master

Merge upstream master branch

docs/index.rst

@@ -14,6 +14,7 @@ Guides
    sanic/exceptions
    sanic/middleware
    sanic/blueprints
+   sanic/websocket
    sanic/config
    sanic/cookies
    sanic/decorators

docs/sanic/middleware.md

@@ -100,6 +100,20 @@ async def close_db(app, loop):
     await app.db.close()
 ```
 
+It's also possible to register a listener using the `register_listener` method. 
+This may be useful if you define your listeners in another module besides
+the one you instantiate your app in.
+
+```python
+app = Sanic()
+    
+async def setup_db(app, loop):
+    app.db = await db_setup()
+    
+app.register_listener(setup_db, 'before_server_start')
+
+```
+
 If you want to schedule a background task to run after the loop has started,
 Sanic provides the `add_task` method to easily do so.
 

docs/sanic/response.md

@@ -91,7 +91,7 @@ from sanic import response
 
 @app.route('/raw')
 def handle_request(request):
-    return response.raw('raw data')
+    return response.raw(b'raw data')
 ```
 
 ## Modify headers or status

docs/sanic/websocket.rst

@@ -0,0 +1,51 @@
+WebSocket
+=========
+
+Sanic supports websockets, to setup a WebSocket:
+
+.. code:: python
+
+    from sanic import Sanic
+    from sanic.response import json
+    from sanic.websocket import WebSocketProtocol
+
+    app = Sanic()
+
+    @app.websocket('/feed')
+    async def feed(request, ws):
+        while True:
+            data = 'hello!'
+            print('Sending: ' + data)
+            await ws.send(data)
+            data = await ws.recv()
+            print('Received: ' + data)
+
+    if __name__ == "__main__":
+        app.run(host="0.0.0.0", port=8000, protocol=WebSocketProtocol)
+
+
+Alternatively, the ``app.add_websocket_route`` method can be used instead of the
+decorator:
+
+.. code:: python
+
+    async def feed(request, ws):
+        pass
+
+    app.add_websocket_route(feed, '/feed')
+
+
+Handlers for a WebSocket route are passed the request as first argument, and a
+WebSocket protocol object as second argument. The protocol object has ``send``
+and ``recv`` methods to send and receive data respectively.
+
+
+You could setup your own WebSocket configuration through ``app.config``, like
+
+.. code:: python
+    app.config.WEBSOCKET_MAX_SIZE = 2 ** 20
+    app.config.WEBSOCKET_MAX_QUEUE = 32
+    app.config.WEBSOCKET_READ_LIMIT = 2 ** 16
+    app.config.WEBSOCKET_WRITE_LIMIT = 2 ** 16
+
+Find more in ``Configuration`` section.

requirements-dev.txt

@@ -7,6 +7,6 @@ httptools
 flake8
 pytest==3.3.2
 tox
-ujson
-uvloop
+ujson; sys_platform != "win32" and implementation_name == "cpython"
+uvloop; sys_platform != "win32" and implementation_name == "cpython"
 gunicorn

requirements.txt

@@ -1,5 +1,5 @@
 aiofiles
 httptools
-ujson
-uvloop
+ujson; sys_platform != "win32" and implementation_name == "cpython"
+uvloop; sys_platform != "win32" and implementation_name == "cpython"
 websockets

sanic/app.py

@@ -1,3 +1,4 @@
+import os
 import logging
 import logging.config
 import re
@@ -22,6 +23,7 @@ from sanic.static import register as static_register
 from sanic.testing import SanicTestClient
 from sanic.views import CompositionView
 from sanic.websocket import WebSocketProtocol, ConnectionClosed
+import sanic.reloader_helpers as reloader_helpers
 
 
 class Sanic:
@@ -117,6 +119,19 @@ class Sanic:
 
         return decorator
 
+    def register_listener(self, listener, event):
+        """
+        Register the listener for a given event.
+
+        Args:
+            listener: callable i.e. setup_db(app, loop)
+            event: when to register listener i.e. 'before_server_start'
+
+        Returns: listener
+        """
+
+        return self.listener(event)(listener)
+
     # Decorator
     def route(self, uri, methods=frozenset({'GET'}), host=None,
               strict_slashes=None, stream=False, version=None, name=None):
@@ -634,7 +649,7 @@ class Sanic:
     def run(self, host=None, port=None, debug=False, ssl=None,
             sock=None, workers=1, protocol=None,
             backlog=100, stop_event=None, register_sys_signals=True,
-            access_log=True):
+            access_log=True, auto_reload=False):
         """Run the HTTP Server and listen until keyboard interrupt or term
         signal. On termination, drain connections before closing.
 
@@ -668,12 +683,21 @@ class Sanic:
             host=host, port=port, debug=debug, ssl=ssl, sock=sock,
             workers=workers, protocol=protocol, backlog=backlog,
             register_sys_signals=register_sys_signals,
-            access_log=access_log)
+            access_log=access_log, auto_reload=auto_reload)
 
         try:
             self.is_running = True
             if workers == 1:
-                serve(**server_settings)
+                if auto_reload and os.name != 'posix':
+                    # This condition must be removed after implementing
+                    # auto reloader for other operating systems.
+                    raise NotImplementedError
+
+                if auto_reload and \
+                        os.environ.get('SANIC_SERVER_RUNNING') != 'true':
+                    reloader_helpers.watchdog(2)
+                else:
+                    serve(**server_settings)
             else:
                 serve_multiple(server_settings, workers)
         except BaseException:
@@ -763,7 +787,8 @@ class Sanic:
     def _helper(self, host=None, port=None, debug=False,
                 ssl=None, sock=None, workers=1, loop=None,
                 protocol=HttpProtocol, backlog=100, stop_event=None,
-                register_sys_signals=True, run_async=False, access_log=True):
+                register_sys_signals=True, run_async=False, access_log=True,
+                auto_reload=False):
         """Helper function used by `run` and `create_server`."""
         if isinstance(ssl, dict):
             # try common aliaseses
@@ -807,6 +832,8 @@ class Sanic:
             'access_log': access_log,
             'websocket_max_size': self.config.WEBSOCKET_MAX_SIZE,
             'websocket_max_queue': self.config.WEBSOCKET_MAX_QUEUE,
+            'websocket_read_limit': self.config.WEBSOCKET_READ_LIMIT,
+            'websocket_write_limit': self.config.WEBSOCKET_WRITE_LIMIT,
             'graceful_shutdown_timeout': self.config.GRACEFUL_SHUTDOWN_TIMEOUT
         }
 
@@ -829,14 +856,16 @@ class Sanic:
 
         if self.configure_logging and debug:
             logger.setLevel(logging.DEBUG)
-        if self.config.LOGO is not None:
+
+        if self.config.LOGO is not None and \
+                os.environ.get('SANIC_SERVER_RUNNING') != 'true':
             logger.debug(self.config.LOGO)
 
         if run_async:
             server_settings['run_async'] = True
 
         # Serve
-        if host and port:
+        if host and port and os.environ.get('SANIC_SERVER_RUNNING') != 'true':
             proto = "http"
             if ssl is not None:
                 proto = "https"

sanic/config.py

@@ -36,6 +36,8 @@ class Config(dict):
         self.KEEP_ALIVE_TIMEOUT = 5  # 5 seconds
         self.WEBSOCKET_MAX_SIZE = 2 ** 20  # 1 megabytes
         self.WEBSOCKET_MAX_QUEUE = 32
+        self.WEBSOCKET_READ_LIMIT = 2 ** 16
+        self.WEBSOCKET_WRITE_LIMIT = 2 ** 16
         self.GRACEFUL_SHUTDOWN_TIMEOUT = 15.0  # 15 sec
 
         if load_env:

sanic/handlers.py

@@ -79,9 +79,9 @@ class ErrorHandler:
         response = None
         try:
             if handler:
-                response = handler(request=request, exception=exception)
+                response = handler(request, exception)
             if response is None:
-                response = self.default(request=request, exception=exception)
+                response = self.default(request, exception)
         except Exception:
             self.log(format_exc())
             if self.debug:

sanic/reloader_helpers.py

@@ -0,0 +1,121 @@
+import os
+import sys
+import signal
+import subprocess
+from time import sleep
+from multiprocessing import Process
+
+
+def _iter_module_files():
+    """This iterates over all relevant Python files.
+
+    It goes through all
+    loaded files from modules, all files in folders of already loaded modules
+    as well as all files reachable through a package.
+    """
+    # The list call is necessary on Python 3 in case the module
+    # dictionary modifies during iteration.
+    for module in list(sys.modules.values()):
+        if module is None:
+            continue
+        filename = getattr(module, '__file__', None)
+        if filename:
+            old = None
+            while not os.path.isfile(filename):
+                old = filename
+                filename = os.path.dirname(filename)
+                if filename == old:
+                    break
+            else:
+                if filename[-4:] in ('.pyc', '.pyo'):
+                    filename = filename[:-1]
+                yield filename
+
+
+def _get_args_for_reloading():
+    """Returns the executable."""
+    rv = [sys.executable]
+    rv.extend(sys.argv)
+    return rv
+
+
+def restart_with_reloader():
+    """Create a new process and a subprocess in it with the same arguments as
+    this one.
+    """
+    args = _get_args_for_reloading()
+    new_environ = os.environ.copy()
+    new_environ['SANIC_SERVER_RUNNING'] = 'true'
+    cmd = ' '.join(args)
+    worker_process = Process(
+                            target=subprocess.call, args=(cmd,),
+                            kwargs=dict(shell=True, env=new_environ))
+    worker_process.start()
+    return worker_process
+
+
+def kill_process_children_unix(pid):
+    """Find and kill child process of a process (maximum two level).
+
+    :param pid: PID of process (process ID)
+    :return: Nothing
+    """
+    root_process_path = "/proc/{pid}/task/{pid}/children".format(pid=pid)
+    if not os.path.isfile(root_process_path):
+        return
+    with open(root_process_path) as children_list_file:
+        children_list_pid = children_list_file.read().split()
+
+    for child_pid in children_list_pid:
+        children_proc_path = "/proc/%s/task/%s/children" % \
+            (child_pid, child_pid)
+        if not os.path.isfile(children_proc_path):
+            continue
+        with open(children_proc_path) as children_list_file_2:
+            children_list_pid_2 = children_list_file_2.read().split()
+        for _pid in children_list_pid_2:
+            os.kill(int(_pid), signal.SIGTERM)
+
+
+def kill_program_completly(proc):
+    """Kill worker and it's child processes and exit.
+
+    :param proc: worker process (process ID)
+    :return: Nothing
+    """
+    kill_process_children_unix(proc.pid)
+    proc.terminate()
+    os._exit(0)
+
+
+def watchdog(sleep_interval):
+    """Watch project files, restart worker process if a change happened.
+
+    :param sleep_interval: interval in second.
+    :return: Nothing
+    """
+    mtimes = {}
+    worker_process = restart_with_reloader()
+    signal.signal(
+        signal.SIGTERM, lambda *args: kill_program_completly(worker_process))
+    signal.signal(
+        signal.SIGINT, lambda *args: kill_program_completly(worker_process))
+    while True:
+        for filename in _iter_module_files():
+            try:
+                mtime = os.stat(filename).st_mtime
+            except OSError:
+                continue
+
+            old_time = mtimes.get(filename)
+            if old_time is None:
+                mtimes[filename] = mtime
+                continue
+            elif mtime > old_time:
+                kill_process_children_unix(worker_process.pid)
+                worker_process = restart_with_reloader()
+
+                mtimes[filename] = mtime
+                break
+
+        sleep(sleep_interval)

sanic/router.py

@@ -128,6 +128,13 @@ class Router:
         if strict_slashes:
             return
 
+        if not isinstance(host, str) and host is not None:
+            # we have gotten back to the top of the recursion tree where the
+            # host was originally a list. By now, we've processed the strict
+            # slashes logic on the leaf nodes (the individual host strings in
+            # the list of host)
+            return
+
         # Add versions with and without trailing /
         slashed_methods = self.routes_all.get(uri + '/', frozenset({}))
         unslashed_methods = self.routes_all.get(uri[:-1], frozenset({}))

sanic/server.py

@@ -514,6 +514,7 @@ def serve(host, port, request_handler, error_handler, before_start=None,
           connections=None, signal=Signal(), request_class=None,
           access_log=True, keep_alive=True, is_request_stream=False,
           router=None, websocket_max_size=None, websocket_max_queue=None,
+          websocket_read_limit=2 ** 16, websocket_write_limit=2 ** 16,
           state=None, graceful_shutdown_timeout=15.0):
     """Start asynchronous HTTP Server on an individual process.
 
@@ -543,6 +544,16 @@ def serve(host, port, request_handler, error_handler, before_start=None,
     :param protocol: subclass of asyncio protocol class
     :param request_class: Request class to use
     :param access_log: disable/enable access log
+    :param websocket_max_size: enforces the maximum size for
+                               incoming messages in bytes.
+    :param websocket_max_queue: sets the maximum length of the queue
+                                that holds incoming messages.
+    :param websocket_read_limit: sets the high-water limit of the buffer for
+                                 incoming bytes, the low-water limit is half
+                                 the high-water limit.
+    :param websocket_write_limit: sets the high-water limit of the buffer for
+                                  outgoing bytes, the low-water limit is a
+                                  quarter of the high-water limit.
     :param is_request_stream: disable/enable Request.stream
     :param router: Router object
     :return: Nothing
@@ -574,6 +585,8 @@ def serve(host, port, request_handler, error_handler, before_start=None,
         router=router,
         websocket_max_size=websocket_max_size,
         websocket_max_queue=websocket_max_queue,
+        websocket_read_limit=websocket_read_limit,
+        websocket_write_limit=websocket_write_limit,
         state=state,
         debug=debug,
     )

setup.py

@@ -51,8 +51,9 @@ setup_kwargs = {
     ],
 }
 
-ujson = 'ujson>=1.35'
-uvloop = 'uvloop>=0.5.3'
+env_dependency = '; sys_platform != "win32" and implementation_name == "cpython"'
+ujson = 'ujson>=1.35' + env_dependency
+uvloop = 'uvloop>=0.5.3' + env_dependency
 
 requirements = [
     'httptools>=0.0.9',
@@ -66,16 +67,9 @@ if strtobool(os.environ.get("SANIC_NO_UJSON", "no")):
     requirements.remove(ujson)
 
 # 'nt' means windows OS
-if strtobool(os.environ.get("SANIC_NO_UVLOOP", "no")) or os.name == 'nt':
+if strtobool(os.environ.get("SANIC_NO_UVLOOP", "no")):
     print("Installing without uvLoop")
     requirements.remove(uvloop)
 
-try:
-    setup_kwargs['install_requires'] = requirements
-    setup(**setup_kwargs)
-except DistutilsPlatformError as exception:
-    requirements.remove(ujson)
-    requirements.remove(uvloop)
-    print("Installing without uJSON or uvLoop")
-    setup_kwargs['install_requires'] = requirements
-    setup(**setup_kwargs)
+setup_kwargs['install_requires'] = requirements
+setup(**setup_kwargs)

tests/test_auto_reload.py

@@ -0,0 +1,108 @@
+import os
+import sys
+import subprocess
+import signal
+from threading import Thread
+from time import sleep
+from json.decoder import JSONDecodeError
+import aiohttp
+import asyncio
+import async_timeout
+
+sanic_project_content_one = '''
+from sanic import Sanic
+from sanic import response
+
+app = Sanic(__name__)
+
+
+@app.route("/")
+async def test(request):
+    return response.json({"test": 1})
+
+
+if __name__ == '__main__':
+    app.run(host="127.0.0.1", port=8000, auto_reload=True)
+'''
+
+sanic_project_content_two = '''
+from sanic import Sanic
+from sanic import response
+
+app = Sanic(__name__)
+
+
+@app.route("/")
+async def test(request):
+    return response.json({"test": 2})
+
+
+if __name__ == '__main__':
+    app.run(host="127.0.0.1", port=8000, auto_reload=True)
+'''
+
+process_id = None
+
+
+def execute_cmd(command):
+    process = subprocess.Popen(command, shell=True)
+    global process_id
+    process_id = process.pid
+    process.communicate()
+
+
+class TestAutoReloading:
+
+    def check_response(self, url, response):
+        """Send http request and tries to take it's response as json.
+        Returns a dictionary.
+        """
+        async def req(url, excepted_response):
+            async with aiohttp.ClientSession() as session:
+                with async_timeout.timeout(10):
+                    async with session.get(url) as response:
+                        try:
+                            result = await response.json()
+                        except JSONDecodeError:
+                            result = {}
+            return result == excepted_response
+
+        loop = asyncio.get_event_loop()
+        return loop.run_until_complete(req(url, response))
+
+    def test_reloading_after_change_file(self, capsys):
+        if os.name != 'posix':
+            return
+
+        with capsys.disabled():
+            pass
+        sanic_app_file_path = "simple_sanic_app.py"
+        with open(sanic_app_file_path, "w") as _file:
+            _file.write(sanic_project_content_one)
+
+        cmd = ' '.join([sys.executable, sanic_app_file_path])
+        thread = Thread(target=execute_cmd, args=(cmd,))
+        thread.start()
+
+        sleep(2)  # wait for completing server start process
+        assert self.check_response("http://127.0.0.1:8000/", {"test": 1})
+
+        with open(sanic_app_file_path, "w") as _file:
+            _file.write(sanic_project_content_two)
+
+        sleep(2)  # wait for completing server start process
+        assert self.check_response("http://127.0.0.1:8000/", {"test": 2})
+
+        thread.join(1)
+        os.remove(sanic_app_file_path)
+
+    def teardown_method(self, method):
+        if process_id:
+            root_proc_path = \
+                "/proc/{pid}/task/{pid}/children".format(pid=process_id)
+            if not os.path.isfile(root_proc_path):
+                return
+            with open(root_proc_path) as children_list_file:
+                children_list_pid = children_list_file.read().split()
+            for child_pid in children_list_pid:
+                os.kill(int(child_pid), signal.SIGTERM)

tests/test_routes.py

@@ -174,6 +174,40 @@ def test_route_optional_slash():
     request, response = app.test_client.get('/get/')
     assert response.text == 'OK'
 
+def test_route_strict_slashes_set_to_false_and_host_is_a_list():
+    #Part of regression test for issue #1120
+    app = Sanic('test_route_strict_slashes_set_to_false_and_host_is_a_list')
+
+    site1 = 'localhost:{}'.format(app.test_client.port)
+
+    #before fix, this raises a RouteExists error
+    @app.get('/get', host=[site1, 'site2.com'], strict_slashes=False)
+    def handler(request):
+        return text('OK')
+
+    request, response = app.test_client.get('http://' + site1 + '/get')
+    assert response.text == 'OK'
+
+    @app.post('/post', host=[site1, 'site2.com'], strict_slashes=False)
+    def handler(request):
+        return text('OK')
+
+    request, response = app.test_client.post('http://' + site1 +'/post')
+    assert response.text == 'OK'
+
+    @app.put('/put', host=[site1, 'site2.com'], strict_slashes=False)
+    def handler(request):
+        return text('OK')
+
+    request, response = app.test_client.put('http://' + site1 +'/put')
+    assert response.text == 'OK'
+
+    @app.delete('/delete', host=[site1, 'site2.com'], strict_slashes=False)
+    def handler(request):
+        return text('OK')
+
+    request, response = app.test_client.delete('http://' + site1 +'/delete')
+    assert response.text == 'OK'
 
 def test_shorthand_routes_post():
     app = Sanic('test_shorhand_routes_post')

tests/test_server_events.py

@@ -49,6 +49,23 @@ def test_single_listener(listener_name):
     assert random_name_app.name + listener_name == output.pop()
 
 
+@pytest.mark.parametrize('listener_name', AVAILABLE_LISTENERS)
+def test_register_listener(listener_name):
+    """
+    Test that listeners on their own work with
+    app.register_listener method
+    """
+    random_name_app = Sanic(''.join(
+        [choice(ascii_letters) for _ in range(choice(range(5, 10)))]))
+    output = list()
+    # Register listener
+    listener = create_listener(listener_name, output)
+    random_name_app.register_listener(listener,
+                                      event=listener_name)
+    start_stop_app(random_name_app)
+    assert random_name_app.name + listener_name == output.pop()
+
+
 def test_all_listeners():
     random_name_app = Sanic(''.join(
         [choice(ascii_letters) for _ in range(choice(range(5, 10)))]))