From 5bb9aa0c2c8768b5cd7eafb3bd4dfb7cc999fff8 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Fri, 18 Jun 2021 11:39:09 +0300 Subject: [PATCH] Add reloading on addtional directories (#2167) --- sanic/__main__.py | 63 ++++++++++++++++++++----------- sanic/app.py | 18 ++++++++- sanic/reloader_helpers.py | 28 ++++++++++---- tests/test_cli.py | 2 +- tests/test_reloader.py | 78 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 157 insertions(+), 32 deletions(-) diff --git a/sanic/__main__.py b/sanic/__main__.py index cf392510..63cd7688 100644 --- a/sanic/__main__.py +++ b/sanic/__main__.py @@ -1,7 +1,7 @@ import os import sys -from argparse import ArgumentParser, RawDescriptionHelpFormatter +from argparse import ArgumentParser, RawTextHelpFormatter from importlib import import_module from typing import Any, Dict, Optional @@ -17,7 +17,7 @@ class SanicArgumentParser(ArgumentParser): def add_bool_arguments(self, *args, **kwargs): group = self.add_mutually_exclusive_group() group.add_argument(*args, action="store_true", **kwargs) - kwargs["help"] = "no " + kwargs["help"] + kwargs["help"] = f"no {kwargs['help']}\n " group.add_argument( "--no-" + args[0][2:], *args[1:], action="store_false", **kwargs ) @@ -27,7 +27,15 @@ def main(): parser = SanicArgumentParser( prog="sanic", description=BASE_LOGO, - formatter_class=RawDescriptionHelpFormatter, + formatter_class=lambda prog: RawTextHelpFormatter( + prog, max_help_position=33 + ), + ) + parser.add_argument( + "-v", + "--version", + action="version", + version=f"Sanic {__version__}; Routing {__routing_version__}", ) parser.add_argument( "-H", @@ -51,13 +59,24 @@ def main(): dest="unix", type=str, default="", - help="location of unix socket", + help="location of unix socket\n ", ) parser.add_argument( "--cert", dest="cert", type=str, help="location of certificate for SSL" ) parser.add_argument( - "--key", dest="key", type=str, help="location of keyfile for SSL." + "--key", dest="key", type=str, help="location of keyfile for SSL\n " + ) + parser.add_bool_arguments( + "--access-logs", dest="access_log", help="display access logs" + ) + parser.add_argument( + "--factory", + action="store_true", + help=( + "Treat app as an application factory, " + "i.e. a () -> callable\n " + ), ) parser.add_argument( "-w", @@ -65,32 +84,23 @@ def main(): dest="workers", type=int, default=1, - help="number of worker processes [default 1]", + help="number of worker processes [default 1]\n ", ) parser.add_argument("-d", "--debug", dest="debug", action="store_true") parser.add_argument( "-r", + "--reload", "--auto-reload", dest="auto_reload", action="store_true", help="Watch source directory for file changes and reload on changes", ) parser.add_argument( - "--factory", - action="store_true", - help=( - "Treat app as an application factory, " - "i.e. a () -> callable." - ), - ) - parser.add_argument( - "-v", - "--version", - action="version", - version=f"Sanic {__version__}; Routing {__routing_version__}", - ) - parser.add_bool_arguments( - "--access-logs", dest="access_log", help="display access logs" + "-R", + "--reload-dir", + dest="path", + action="append", + help="Extra directories to watch and reload on changes\n ", ) parser.add_argument( "module", help="path to your Sanic app. Example: path.to.server:app" @@ -140,6 +150,17 @@ def main(): } if args.auto_reload: kwargs["auto_reload"] = True + + if args.path: + if args.auto_reload or args.debug: + kwargs["reload_dir"] = args.path + else: + error_logger.warning( + "Ignoring '--reload-dir' since auto reloading was not " + "enabled. If you would like to watch directories for " + "changes, consider using --debug or --auto-reload." + ) + app.run(**kwargs) except ImportError as e: if module_name.startswith(e.name): diff --git a/sanic/app.py b/sanic/app.py index 8d33d735..1d08f02e 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -14,6 +14,7 @@ from asyncio.futures import Future from collections import defaultdict, deque from functools import partial from inspect import isawaitable +from pathlib import Path from socket import socket from ssl import Purpose, SSLContext, create_default_context from traceback import format_exc @@ -105,6 +106,7 @@ class Sanic(BaseSanic): "name", "named_request_middleware", "named_response_middleware", + "reload_dirs", "request_class", "request_middleware", "response_middleware", @@ -168,6 +170,7 @@ class Sanic(BaseSanic): self.listeners: Dict[str, List[ListenerType]] = defaultdict(list) self.named_request_middleware: Dict[str, Deque[MiddlewareType]] = {} self.named_response_middleware: Dict[str, Deque[MiddlewareType]] = {} + self.reload_dirs: Set[Path] = set() self.request_class = request_class self.request_middleware: Deque[MiddlewareType] = deque() self.response_middleware: Deque[MiddlewareType] = deque() @@ -389,7 +392,7 @@ class Sanic(BaseSanic): if self.config.EVENT_AUTOREGISTER: self.signal_router.reset() self.add_signal(None, event) - signal = self.signal_router.name_index.get(event) + signal = self.signal_router.name_index[event] self.signal_router.finalize() else: raise NotFound("Could not find signal %s" % event) @@ -846,6 +849,7 @@ class Sanic(BaseSanic): access_log: Optional[bool] = None, unix: Optional[str] = None, loop: None = None, + reload_dir: Optional[Union[List[str], str]] = None, ) -> None: """ Run the HTTP Server and listen until keyboard interrupt or term @@ -880,6 +884,18 @@ class Sanic(BaseSanic): :type unix: str :return: Nothing """ + if reload_dir: + if isinstance(reload_dir, str): + reload_dir = [reload_dir] + + for directory in reload_dir: + direc = Path(directory) + if not direc.is_dir(): + logger.warning( + f"Directory {directory} could not be located" + ) + self.reload_dirs.add(Path(directory)) + if loop is not None: raise TypeError( "loop is not a valid argument. To use an existing loop, " diff --git a/sanic/reloader_helpers.py b/sanic/reloader_helpers.py index c09697f2..4551472a 100644 --- a/sanic/reloader_helpers.py +++ b/sanic/reloader_helpers.py @@ -1,3 +1,4 @@ +import itertools import os import signal import subprocess @@ -59,6 +60,20 @@ def restart_with_reloader(): ) +def _check_file(filename, mtimes): + need_reload = False + + mtime = os.stat(filename).st_mtime + old_time = mtimes.get(filename) + if old_time is None: + mtimes[filename] = mtime + elif mtime > old_time: + mtimes[filename] = mtime + need_reload = True + + return need_reload + + def watchdog(sleep_interval, app): """Watch project files, restart worker process if a change happened. @@ -85,17 +100,16 @@ def watchdog(sleep_interval, app): while True: need_reload = False - for filename in _iter_module_files(): + for filename in itertools.chain( + _iter_module_files(), + *(d.glob("**/*") for d in app.reload_dirs), + ): try: - mtime = os.stat(filename).st_mtime + check = _check_file(filename, mtimes) except OSError: continue - old_time = mtimes.get(filename) - if old_time is None: - mtimes[filename] = mtime - elif mtime > old_time: - mtimes[filename] = mtime + if check: need_reload = True if need_reload: diff --git a/tests/test_cli.py b/tests/test_cli.py index df2d0e61..5f69dd95 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,7 +33,7 @@ def capture(command): "fake.server:app", "fake.server:create_app()", "fake.server.create_app()", - ) + ), ) def test_server_run(appname): command = ["sanic", appname] diff --git a/tests/test_reloader.py b/tests/test_reloader.py index a31a3d7d..1cc84f97 100644 --- a/tests/test_reloader.py +++ b/tests/test_reloader.py @@ -23,6 +23,8 @@ try: except ImportError: flags = 0 +TIMER_DELAY = 2 + def terminate(proc): if flags: @@ -56,6 +58,40 @@ def write_app(filename, **runargs): return text +def write_json_config_app(filename, jsonfile, **runargs): + with open(filename, "w") as f: + f.write( + dedent( + f"""\ + import os + from sanic import Sanic + import json + + app = Sanic(__name__) + with open("{jsonfile}", "r") as f: + config = json.load(f) + app.config.update_config(config) + + app.route("/")(lambda x: x) + + @app.listener("after_server_start") + def complete(*args): + print("complete", os.getpid(), app.config.FOO) + + if __name__ == "__main__": + app.run(**{runargs!r}) + """ + ) + ) + + +def write_file(filename): + text = secrets.token_urlsafe() + with open(filename, "w") as f: + f.write(f"""{{"FOO": "{text}"}}""") + return text + + def scanner(proc): for line in proc.stdout: line = line.decode().strip() @@ -90,9 +126,10 @@ async def test_reloader_live(runargs, mode): with TemporaryDirectory() as tmpdir: filename = os.path.join(tmpdir, "reloader.py") text = write_app(filename, **runargs) - proc = Popen(argv[mode], cwd=tmpdir, stdout=PIPE, creationflags=flags) + command = argv[mode] + proc = Popen(command, cwd=tmpdir, stdout=PIPE, creationflags=flags) try: - timeout = Timer(5, terminate, [proc]) + timeout = Timer(TIMER_DELAY, terminate, [proc]) timeout.start() # Python apparently keeps using the old source sometimes if # we don't sleep before rewrite (pycache timestamp problem?) @@ -107,3 +144,40 @@ async def test_reloader_live(runargs, mode): terminate(proc) with suppress(TimeoutExpired): proc.wait(timeout=3) + + +@pytest.mark.parametrize( + "runargs, mode", + [ + (dict(port=42102, auto_reload=True), "script"), + (dict(port=42103, debug=True), "module"), + ({}, "sanic"), + ], +) +async def test_reloader_live_with_dir(runargs, mode): + with TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, "reloader.py") + config_file = os.path.join(tmpdir, "config.json") + runargs["reload_dir"] = tmpdir + write_json_config_app(filename, config_file, **runargs) + text = write_file(config_file) + command = argv[mode] + if mode == "sanic": + command += ["--reload-dir", tmpdir] + proc = Popen(command, cwd=tmpdir, stdout=PIPE, creationflags=flags) + try: + timeout = Timer(TIMER_DELAY, terminate, [proc]) + timeout.start() + # Python apparently keeps using the old source sometimes if + # we don't sleep before rewrite (pycache timestamp problem?) + sleep(1) + line = scanner(proc) + assert text in next(line) + # Edit source code and try again + text = write_file(config_file) + assert text in next(line) + finally: + timeout.cancel() + terminate(proc) + with suppress(TimeoutExpired): + proc.wait(timeout=3)