Implement restart ordering (#2632)

tests/worker/test_inspector.py

@@ -88,6 +88,12 @@ def test_run_inspector_reload(publisher, http_client):
     publisher.send.assert_called_once_with("__ALL_PROCESSES__:")
 
 
+def test_run_inspector_reload_zero_downtime(publisher, http_client):
+    _, response = http_client.post("/reload", json={"zero_downtime": True})
+    assert response.status == 200
+    publisher.send.assert_called_once_with("__ALL_PROCESSES__::STARTUP_FIRST")
+
+
 def test_run_inspector_shutdown(publisher, http_client):
     _, response = http_client.post("/shutdown")
     assert response.status == 200

tests/worker/test_manager.py

@@ -6,6 +6,7 @@ import pytest
 
 from sanic.compat import OS_IS_WINDOWS
 from sanic.exceptions import ServerKilled
+from sanic.worker.constants import RestartOrder
 from sanic.worker.manager import WorkerManager
 
 
@@ -102,11 +103,17 @@ def test_restart_all():
     )
 
 
-def test_monitor_all():
+@pytest.mark.parametrize("zero_downtime", (False, True))
+def test_monitor_all(zero_downtime):
     p1 = Mock()
     p2 = Mock()
     sub = Mock()
-    sub.recv.side_effect = ["__ALL_PROCESSES__:", ""]
+    incoming = (
+        "__ALL_PROCESSES__::STARTUP_FIRST"
+        if zero_downtime
+        else "__ALL_PROCESSES__:"
+    )
+    sub.recv.side_effect = [incoming, ""]
     context = Mock()
     context.Process.side_effect = [p1, p2]
     manager = WorkerManager(2, fake_serve, {}, context, (Mock(), sub), {})
@@ -114,16 +121,29 @@ def test_monitor_all():
     manager.wait_for_ack = Mock()  # type: ignore
     manager.monitor()
 
+    restart_order = (
+        RestartOrder.STARTUP_FIRST
+        if zero_downtime
+        else RestartOrder.SHUTDOWN_FIRST
+    )
     manager.restart.assert_called_once_with(
-        process_names=None, reloaded_files=""
+        process_names=None,
+        reloaded_files="",
+        restart_order=restart_order,
     )
 
 
-def test_monitor_all_with_files():
+@pytest.mark.parametrize("zero_downtime", (False, True))
+def test_monitor_all_with_files(zero_downtime):
     p1 = Mock()
     p2 = Mock()
     sub = Mock()
-    sub.recv.side_effect = ["__ALL_PROCESSES__:foo,bar", ""]
+    incoming = (
+        "__ALL_PROCESSES__:foo,bar:STARTUP_FIRST"
+        if zero_downtime
+        else "__ALL_PROCESSES__:foo,bar"
+    )
+    sub.recv.side_effect = [incoming, ""]
     context = Mock()
     context.Process.side_effect = [p1, p2]
     manager = WorkerManager(2, fake_serve, {}, context, (Mock(), sub), {})
@@ -131,17 +151,30 @@ def test_monitor_all_with_files():
     manager.wait_for_ack = Mock()  # type: ignore
     manager.monitor()
 
+    restart_order = (
+        RestartOrder.STARTUP_FIRST
+        if zero_downtime
+        else RestartOrder.SHUTDOWN_FIRST
+    )
     manager.restart.assert_called_once_with(
-        process_names=None, reloaded_files="foo,bar"
+        process_names=None,
+        reloaded_files="foo,bar",
+        restart_order=restart_order,
     )
 
 
-def test_monitor_one_process():
+@pytest.mark.parametrize("zero_downtime", (False, True))
+def test_monitor_one_process(zero_downtime):
     p1 = Mock()
     p1.name = "Testing"
     p2 = Mock()
     sub = Mock()
-    sub.recv.side_effect = [f"{p1.name}:foo,bar", ""]
+    incoming = (
+        f"{p1.name}:foo,bar:STARTUP_FIRST"
+        if zero_downtime
+        else f"{p1.name}:foo,bar"
+    )
+    sub.recv.side_effect = [incoming, ""]
     context = Mock()
     context.Process.side_effect = [p1, p2]
     manager = WorkerManager(2, fake_serve, {}, context, (Mock(), sub), {})
@@ -149,8 +182,15 @@ def test_monitor_one_process():
     manager.wait_for_ack = Mock()  # type: ignore
     manager.monitor()
 
+    restart_order = (
+        RestartOrder.STARTUP_FIRST
+        if zero_downtime
+        else RestartOrder.SHUTDOWN_FIRST
+    )
     manager.restart.assert_called_once_with(
-        process_names=[p1.name], reloaded_files="foo,bar"
+        process_names=[p1.name],
+        reloaded_files="foo,bar",
+        restart_order=restart_order,
     )
 
 

tests/worker/test_multiplexer.py

@@ -98,17 +98,17 @@ def test_ack(worker_state: Dict[str, Any], m: WorkerMultiplexer):
 
 def test_restart_self(monitor_publisher: Mock, m: WorkerMultiplexer):
     m.restart()
-    monitor_publisher.send.assert_called_once_with("Test")
+    monitor_publisher.send.assert_called_once_with("Test:")
 
 
 def test_restart_foo(monitor_publisher: Mock, m: WorkerMultiplexer):
     m.restart("foo")
-    monitor_publisher.send.assert_called_once_with("foo")
+    monitor_publisher.send.assert_called_once_with("foo:")
 
 
 def test_reload_alias(monitor_publisher: Mock, m: WorkerMultiplexer):
     m.reload()
-    monitor_publisher.send.assert_called_once_with("Test")
+    monitor_publisher.send.assert_called_once_with("Test:")
 
 
 def test_terminate(monitor_publisher: Mock, m: WorkerMultiplexer):
@@ -135,10 +135,20 @@ def test_properties(
 @pytest.mark.parametrize(
     "params,expected",
     (
-        ({}, "Test"),
-        ({"name": "foo"}, "foo"),
+        ({}, "Test:"),
+        ({"name": "foo"}, "foo:"),
         ({"all_workers": True}, "__ALL_PROCESSES__:"),
+        ({"zero_downtime": True}, "Test::STARTUP_FIRST"),
         ({"name": "foo", "all_workers": True}, ValueError),
+        ({"name": "foo", "zero_downtime": True}, "foo::STARTUP_FIRST"),
+        (
+            {"all_workers": True, "zero_downtime": True},
+            "__ALL_PROCESSES__::STARTUP_FIRST",
+        ),
+        (
+            {"name": "foo", "all_workers": True, "zero_downtime": True},
+            ValueError,
+        ),
     ),
 )
 def test_restart_params(

tests/worker/test_reloader.py

@@ -1,13 +1,19 @@
+import re
 import signal
+import threading
 
 from asyncio import Event
+from logging import DEBUG
 from pathlib import Path
+from time import sleep
 from unittest.mock import Mock
 
 import pytest
 
 from sanic.app import Sanic
+from sanic.worker.constants import ProcessState, RestartOrder
 from sanic.worker.loader import AppLoader
+from sanic.worker.process import WorkerProcess
 from sanic.worker.reloader import Reloader
 
 
@@ -67,6 +73,88 @@ def test_iter_files():
     assert len_total_files == len_python_files + len_static_files
 
 
+@pytest.mark.parametrize(
+    "order,expected",
+    (
+        (
+            RestartOrder.SHUTDOWN_FIRST,
+            [
+                "Restarting a process",
+                "Begin restart termination",
+                "Starting a process",
+            ],
+        ),
+        (
+            RestartOrder.STARTUP_FIRST,
+            [
+                "Restarting a process",
+                "Starting a process",
+                "Begin restart termination",
+                "Waiting for process to be acked",
+                "Process acked. Terminating",
+            ],
+        ),
+    ),
+)
+def test_default_reload_shutdown_order(monkeypatch, caplog, order, expected):
+    current_process = Mock()
+    worker_process = WorkerProcess(
+        lambda **_: current_process,
+        "Test",
+        lambda **_: ...,
+        {},
+        {},
+    )
+
+    def start(self):
+        worker_process.set_state(ProcessState.ACKED)
+        self._target()
+
+    orig = threading.Thread.start
+    monkeypatch.setattr(threading.Thread, "start", start)
+
+    with caplog.at_level(DEBUG):
+        worker_process.restart(restart_order=order)
+
+    ansi = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+
+    def clean(msg: str):
+        msg, _ = ansi.sub("", msg).split(":", 1)
+        return msg
+
+    debug = [clean(record[2]) for record in caplog.record_tuples]
+    assert debug == expected
+    current_process.start.assert_called_once()
+    current_process.terminate.assert_called_once()
+    monkeypatch.setattr(threading.Thread, "start", orig)
+
+
+def test_reload_delayed(monkeypatch):
+    WorkerProcess.THRESHOLD = 1
+
+    current_process = Mock()
+    worker_process = WorkerProcess(
+        lambda **_: current_process,
+        "Test",
+        lambda **_: ...,
+        {},
+        {},
+    )
+
+    def start(self):
+        sleep(0.2)
+        self._target()
+
+    orig = threading.Thread.start
+    monkeypatch.setattr(threading.Thread, "start", start)
+
+    message = "Worker Test failed to come ack within 0.1 seconds"
+    with pytest.raises(TimeoutError, match=message):
+        worker_process.restart(restart_order=RestartOrder.STARTUP_FIRST)
+
+    monkeypatch.setattr(threading.Thread, "start", orig)
+
+
 def test_reloader_triggers_start_stop_listeners(
     app: Sanic, app_loader: AppLoader
 ):

tests/worker/test_worker_serve.py

@@ -118,7 +118,7 @@ def test_serve_with_inspector(
     if config:
         Inspector.assert_called_once()
         WorkerManager.manage.assert_called_once_with(
-            "Inspector", inspector, {}, transient=True
+            "Inspector", inspector, {}, transient=False
         )
     else:
         Inspector.assert_not_called()