Allow 8192 header max to be breached (#2155)

* Allow 8192 header max to be breached

* Add REQUEST_MAX_HEADER_SIZE as config value

* remove queue size

sanic/app.py

@@ -183,7 +183,6 @@ class Sanic(BaseSanic):
 
         if register is not None:
             self.config.REGISTER = register
-
         if self.config.REGISTER:
             self.__class__.register_app(self)
 

sanic/config.py

@@ -4,6 +4,8 @@ from pathlib import Path
 from typing import Any, Dict, Optional, Union
 from warnings import warn
 
+from sanic.http import Http
+
 from .utils import load_module_from_file_location, str_to_bool
 
 
@@ -28,6 +30,7 @@ DEFAULT_CONFIG = {
     "REAL_IP_HEADER": None,
     "REGISTER": True,
     "REQUEST_BUFFER_SIZE": 65536,  # 64 KiB
+    "REQUEST_MAX_HEADER_SIZE": 8192,  # 8 KiB, but cannot exceed 16384
     "REQUEST_ID_HEADER": "X-Request-ID",
     "REQUEST_MAX_SIZE": 100000000,  # 100 megabytes
     "REQUEST_TIMEOUT": 60,  # 60 seconds
@@ -42,12 +45,36 @@ DEFAULT_CONFIG = {
 
 
 class Config(dict):
+    ACCESS_LOG: bool
+    EVENT_AUTOREGISTER: bool
+    FALLBACK_ERROR_FORMAT: str
+    FORWARDED_FOR_HEADER: str
+    FORWARDED_SECRET: Optional[str]
+    GRACEFUL_SHUTDOWN_TIMEOUT: float
+    KEEP_ALIVE_TIMEOUT: int
+    KEEP_ALIVE: bool
+    PROXIES_COUNT: Optional[int]
+    REAL_IP_HEADER: Optional[str]
+    REGISTER: bool
+    REQUEST_BUFFER_SIZE: int
+    REQUEST_MAX_HEADER_SIZE: int
+    REQUEST_ID_HEADER: str
+    REQUEST_MAX_SIZE: int
+    REQUEST_TIMEOUT: int
+    RESPONSE_TIMEOUT: int
+    WEBSOCKET_MAX_QUEUE: int
+    WEBSOCKET_MAX_SIZE: int
+    WEBSOCKET_PING_INTERVAL: int
+    WEBSOCKET_PING_TIMEOUT: int
+    WEBSOCKET_READ_LIMIT: int
+    WEBSOCKET_WRITE_LIMIT: int
+
     def __init__(
         self,
         defaults: Dict[str, Union[str, bool, int, float, None]] = None,
         load_env: Optional[Union[bool, str]] = True,
         env_prefix: Optional[str] = SANIC_PREFIX,
-        keep_alive: Optional[int] = None,
+        keep_alive: Optional[bool] = None,
     ):
         defaults = defaults or {}
         super().__init__({**DEFAULT_CONFIG, **defaults})
@@ -72,6 +99,8 @@ class Config(dict):
         else:
             self.load_environment_vars(SANIC_PREFIX)
 
+        self._configure_header_size()
+
     def __getattr__(self, attr):
         try:
             return self[attr]
@@ -80,6 +109,19 @@ class Config(dict):
 
     def __setattr__(self, attr, value):
         self[attr] = value
+        if attr in (
+            "REQUEST_MAX_HEADER_SIZE",
+            "REQUEST_BUFFER_SIZE",
+            "REQUEST_MAX_SIZE",
+        ):
+            self._configure_header_size()
+
+    def _configure_header_size(self):
+        Http.set_header_max_size(
+            self.REQUEST_MAX_HEADER_SIZE,
+            self.REQUEST_BUFFER_SIZE - 4096,
+            self.REQUEST_MAX_SIZE,
+        )
 
     def load_environment_vars(self, prefix=SANIC_PREFIX):
         """

sanic/http.py

@@ -64,6 +64,9 @@ class Http:
     :raises RuntimeError:
     """
 
+    HEADER_CEILING = 16_384
+    HEADER_MAX_SIZE = 0
+
     __slots__ = [
         "_send",
         "_receive_more",
@@ -169,7 +172,6 @@ class Http:
         """
         Receive and parse request header into self.request.
         """
-        HEADER_MAX_SIZE = min(8192, self.request_max_size)
         # Receive until full header is in buffer
         buf = self.recv_buffer
         pos = 0
@@ -180,12 +182,12 @@ class Http:
                 break
 
             pos = max(0, len(buf) - 3)
-            if pos >= HEADER_MAX_SIZE:
+            if pos >= self.HEADER_MAX_SIZE:
                 break
 
             await self._receive_more()
 
-        if pos >= HEADER_MAX_SIZE:
+        if pos >= self.HEADER_MAX_SIZE:
             raise PayloadTooLarge("Request header exceeds the size limit")
 
         # Parse header content
@@ -541,3 +543,10 @@ class Http:
     @property
     def send(self):
         return self.response_func
+
+    @classmethod
+    def set_header_max_size(cls, *sizes: int):
+        cls.HEADER_MAX_SIZE = min(
+            *sizes,
+            cls.HEADER_CEILING,
+        )

tests/test_headers.py

@@ -7,6 +7,13 @@ from sanic.exceptions import PayloadTooLarge
 from sanic.http import Http
 
 
+@pytest.fixture
+def raised_ceiling():
+    Http.HEADER_CEILING = 32_768
+    yield
+    Http.HEADER_CEILING = 16_384
+
+
 @pytest.mark.parametrize(
     "input, expected",
     [
@@ -76,15 +83,75 @@ async def test_header_size_exceeded():
         recv_buffer += b"123"
 
     protocol = Mock()
+    Http.set_header_max_size(1)
     http = Http(protocol)
     http._receive_more = _receive_more
-    http.request_max_size = 1
     http.recv_buffer = recv_buffer
 
     with pytest.raises(PayloadTooLarge):
         await http.http1_request_header()
 
 
+@pytest.mark.asyncio
+async def test_header_size_increased_okay():
+    recv_buffer = bytearray()
+
+    async def _receive_more():
+        nonlocal recv_buffer
+        recv_buffer += b"123"
+
+    protocol = Mock()
+    Http.set_header_max_size(12_288)
+    http = Http(protocol)
+    http._receive_more = _receive_more
+    http.recv_buffer = recv_buffer
+
+    with pytest.raises(PayloadTooLarge):
+        await http.http1_request_header()
+
+    assert len(recv_buffer) == 12_291
+
+
+@pytest.mark.asyncio
+async def test_header_size_exceeded_maxed_out():
+    recv_buffer = bytearray()
+
+    async def _receive_more():
+        nonlocal recv_buffer
+        recv_buffer += b"123"
+
+    protocol = Mock()
+    Http.set_header_max_size(18_432)
+    http = Http(protocol)
+    http._receive_more = _receive_more
+    http.recv_buffer = recv_buffer
+
+    with pytest.raises(PayloadTooLarge):
+        await http.http1_request_header()
+
+    assert len(recv_buffer) == 16_389
+
+
+@pytest.mark.asyncio
+async def test_header_size_exceeded_raised_ceiling(raised_ceiling):
+    recv_buffer = bytearray()
+
+    async def _receive_more():
+        nonlocal recv_buffer
+        recv_buffer += b"123"
+
+    protocol = Mock()
+    http = Http(protocol)
+    Http.set_header_max_size(65_536)
+    http._receive_more = _receive_more
+    http.recv_buffer = recv_buffer
+
+    with pytest.raises(PayloadTooLarge):
+        await http.http1_request_header()
+
+    assert len(recv_buffer) == 32_772
+
+
 def test_raw_headers(app):
     app.route("/")(lambda _: text(""))
     request, _ = app.test_client.get(