Add reloading on addtional directories (#2167)

sanic/__main__.py

@@ -1,7 +1,7 @@
 import os
 import sys
 
-from argparse import ArgumentParser, RawDescriptionHelpFormatter
+from argparse import ArgumentParser, RawTextHelpFormatter
 from importlib import import_module
 from typing import Any, Dict, Optional
 
@@ -17,7 +17,7 @@ class SanicArgumentParser(ArgumentParser):
     def add_bool_arguments(self, *args, **kwargs):
         group = self.add_mutually_exclusive_group()
         group.add_argument(*args, action="store_true", **kwargs)
-        kwargs["help"] = "no " + kwargs["help"]
+        kwargs["help"] = f"no {kwargs['help']}\n "
         group.add_argument(
             "--no-" + args[0][2:], *args[1:], action="store_false", **kwargs
         )
@@ -27,7 +27,15 @@ def main():
     parser = SanicArgumentParser(
         prog="sanic",
         description=BASE_LOGO,
-        formatter_class=RawDescriptionHelpFormatter,
+        formatter_class=lambda prog: RawTextHelpFormatter(
+            prog, max_help_position=33
+        ),
+    )
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"Sanic {__version__}; Routing {__routing_version__}",
     )
     parser.add_argument(
         "-H",
@@ -51,13 +59,24 @@ def main():
         dest="unix",
         type=str,
         default="",
-        help="location of unix socket",
+        help="location of unix socket\n ",
     )
     parser.add_argument(
         "--cert", dest="cert", type=str, help="location of certificate for SSL"
     )
     parser.add_argument(
-        "--key", dest="key", type=str, help="location of keyfile for SSL."
+        "--key", dest="key", type=str, help="location of keyfile for SSL\n "
+    )
+    parser.add_bool_arguments(
+        "--access-logs", dest="access_log", help="display access logs"
+    )
+    parser.add_argument(
+        "--factory",
+        action="store_true",
+        help=(
+            "Treat app as an application factory, "
+            "i.e. a () -> <Sanic app> callable\n "
+        ),
     )
     parser.add_argument(
         "-w",
@@ -65,32 +84,23 @@ def main():
         dest="workers",
         type=int,
         default=1,
-        help="number of worker processes [default 1]",
+        help="number of worker processes [default 1]\n ",
     )
     parser.add_argument("-d", "--debug", dest="debug", action="store_true")
     parser.add_argument(
         "-r",
+        "--reload",
         "--auto-reload",
         dest="auto_reload",
         action="store_true",
         help="Watch source directory for file changes and reload on changes",
     )
     parser.add_argument(
-        "--factory",
+        "-R",
-        action="store_true",
+        "--reload-dir",
-        help=(
+        dest="path",
-            "Treat app as an application factory, "
+        action="append",
-            "i.e. a () -> <Sanic app> callable."
+        help="Extra directories to watch and reload on changes\n ",
-        ),
-    )
-    parser.add_argument(
-        "-v",
-        "--version",
-        action="version",
-        version=f"Sanic {__version__}; Routing {__routing_version__}",
-    )
-    parser.add_bool_arguments(
-        "--access-logs", dest="access_log", help="display access logs"
     )
     parser.add_argument(
         "module", help="path to your Sanic app. Example: path.to.server:app"
@@ -140,6 +150,17 @@ def main():
         }
         if args.auto_reload:
             kwargs["auto_reload"] = True
+
+        if args.path:
+            if args.auto_reload or args.debug:
+                kwargs["reload_dir"] = args.path
+            else:
+                error_logger.warning(
+                    "Ignoring '--reload-dir' since auto reloading was not "
+                    "enabled. If you would like to watch directories for "
+                    "changes, consider using --debug or --auto-reload."
+                )
+
         app.run(**kwargs)
     except ImportError as e:
         if module_name.startswith(e.name):

sanic/app.py

@@ -14,6 +14,7 @@ from asyncio.futures import Future
 from collections import defaultdict, deque
 from functools import partial
 from inspect import isawaitable
+from pathlib import Path
 from socket import socket
 from ssl import Purpose, SSLContext, create_default_context
 from traceback import format_exc
@@ -105,6 +106,7 @@ class Sanic(BaseSanic):
         "name",
         "named_request_middleware",
         "named_response_middleware",
+        "reload_dirs",
         "request_class",
         "request_middleware",
         "response_middleware",
@@ -168,6 +170,7 @@ class Sanic(BaseSanic):
         self.listeners: Dict[str, List[ListenerType]] = defaultdict(list)
         self.named_request_middleware: Dict[str, Deque[MiddlewareType]] = {}
         self.named_response_middleware: Dict[str, Deque[MiddlewareType]] = {}
+        self.reload_dirs: Set[Path] = set()
         self.request_class = request_class
         self.request_middleware: Deque[MiddlewareType] = deque()
         self.response_middleware: Deque[MiddlewareType] = deque()
@@ -389,7 +392,7 @@ class Sanic(BaseSanic):
             if self.config.EVENT_AUTOREGISTER:
                 self.signal_router.reset()
                 self.add_signal(None, event)
-                signal = self.signal_router.name_index.get(event)
+                signal = self.signal_router.name_index[event]
                 self.signal_router.finalize()
             else:
                 raise NotFound("Could not find signal %s" % event)
@@ -846,6 +849,7 @@ class Sanic(BaseSanic):
         access_log: Optional[bool] = None,
         unix: Optional[str] = None,
         loop: None = None,
+        reload_dir: Optional[Union[List[str], str]] = None,
     ) -> None:
         """
         Run the HTTP Server and listen until keyboard interrupt or term
@@ -880,6 +884,18 @@ class Sanic(BaseSanic):
         :type unix: str
         :return: Nothing
         """
+        if reload_dir:
+            if isinstance(reload_dir, str):
+                reload_dir = [reload_dir]
+
+            for directory in reload_dir:
+                direc = Path(directory)
+                if not direc.is_dir():
+                    logger.warning(
+                        f"Directory {directory} could not be located"
+                    )
+                self.reload_dirs.add(Path(directory))
+
         if loop is not None:
             raise TypeError(
                 "loop is not a valid argument. To use an existing loop, "

sanic/reloader_helpers.py

@@ -1,3 +1,4 @@
+import itertools
 import os
 import signal
 import subprocess
@@ -59,6 +60,20 @@ def restart_with_reloader():
     )
 
 
+def _check_file(filename, mtimes):
+    need_reload = False
+
+    mtime = os.stat(filename).st_mtime
+    old_time = mtimes.get(filename)
+    if old_time is None:
+        mtimes[filename] = mtime
+    elif mtime > old_time:
+        mtimes[filename] = mtime
+        need_reload = True
+
+    return need_reload
+
+
 def watchdog(sleep_interval, app):
     """Watch project files, restart worker process if a change happened.
 
@@ -85,17 +100,16 @@ def watchdog(sleep_interval, app):
         while True:
             need_reload = False
 
-            for filename in _iter_module_files():
+            for filename in itertools.chain(
+                _iter_module_files(),
+                *(d.glob("**/*") for d in app.reload_dirs),
+            ):
                 try:
-                    mtime = os.stat(filename).st_mtime
+                    check = _check_file(filename, mtimes)
                 except OSError:
                     continue
 
-                old_time = mtimes.get(filename)
+                if check:
-                if old_time is None:
-                    mtimes[filename] = mtime
-                elif mtime > old_time:
-                    mtimes[filename] = mtime
                     need_reload = True
 
             if need_reload:

tests/test_cli.py

@@ -33,7 +33,7 @@ def capture(command):
         "fake.server:app",
         "fake.server:create_app()",
         "fake.server.create_app()",
-    )
+    ),
 )
 def test_server_run(appname):
     command = ["sanic", appname]

tests/test_reloader.py

@@ -23,6 +23,8 @@ try:
 except ImportError:
     flags = 0
 
+TIMER_DELAY = 2
+
 
 def terminate(proc):
     if flags:
@@ -56,6 +58,40 @@ def write_app(filename, **runargs):
     return text
 
 
+def write_json_config_app(filename, jsonfile, **runargs):
+    with open(filename, "w") as f:
+        f.write(
+            dedent(
+                f"""\
+            import os
+            from sanic import Sanic
+            import json
+
+            app = Sanic(__name__)
+            with open("{jsonfile}", "r") as f:
+                config = json.load(f)
+            app.config.update_config(config)
+
+            app.route("/")(lambda x: x)
+
+            @app.listener("after_server_start")
+            def complete(*args):
+                print("complete", os.getpid(), app.config.FOO)
+
+            if __name__ == "__main__":
+                app.run(**{runargs!r})
+            """
+            )
+        )
+
+
+def write_file(filename):
+    text = secrets.token_urlsafe()
+    with open(filename, "w") as f:
+        f.write(f"""{{"FOO": "{text}"}}""")
+    return text
+
+
 def scanner(proc):
     for line in proc.stdout:
         line = line.decode().strip()
@@ -90,9 +126,10 @@ async def test_reloader_live(runargs, mode):
     with TemporaryDirectory() as tmpdir:
         filename = os.path.join(tmpdir, "reloader.py")
         text = write_app(filename, **runargs)
-        proc = Popen(argv[mode], cwd=tmpdir, stdout=PIPE, creationflags=flags)
+        command = argv[mode]
+        proc = Popen(command, cwd=tmpdir, stdout=PIPE, creationflags=flags)
         try:
-            timeout = Timer(5, terminate, [proc])
+            timeout = Timer(TIMER_DELAY, terminate, [proc])
             timeout.start()
             # Python apparently keeps using the old source sometimes if
             # we don't sleep before rewrite (pycache timestamp problem?)
@@ -107,3 +144,40 @@ async def test_reloader_live(runargs, mode):
             terminate(proc)
             with suppress(TimeoutExpired):
                 proc.wait(timeout=3)
+
+
+@pytest.mark.parametrize(
+    "runargs, mode",
+    [
+        (dict(port=42102, auto_reload=True), "script"),
+        (dict(port=42103, debug=True), "module"),
+        ({}, "sanic"),
+    ],
+)
+async def test_reloader_live_with_dir(runargs, mode):
+    with TemporaryDirectory() as tmpdir:
+        filename = os.path.join(tmpdir, "reloader.py")
+        config_file = os.path.join(tmpdir, "config.json")
+        runargs["reload_dir"] = tmpdir
+        write_json_config_app(filename, config_file, **runargs)
+        text = write_file(config_file)
+        command = argv[mode]
+        if mode == "sanic":
+            command += ["--reload-dir", tmpdir]
+        proc = Popen(command, cwd=tmpdir, stdout=PIPE, creationflags=flags)
+        try:
+            timeout = Timer(TIMER_DELAY, terminate, [proc])
+            timeout.start()
+            # Python apparently keeps using the old source sometimes if
+            # we don't sleep before rewrite (pycache timestamp problem?)
+            sleep(1)
+            line = scanner(proc)
+            assert text in next(line)
+            # Edit source code and try again
+            text = write_file(config_file)
+            assert text in next(line)
+        finally:
+            timeout.cancel()
+            terminate(proc)
+            with suppress(TimeoutExpired):
+                proc.wait(timeout=3)