From 141be0028dbac535c0c422fdffde159e3997addd Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Fri, 4 Jun 2021 13:56:29 +0300 Subject: [PATCH] Allow 8192 header max to be breached (#2155) * Allow 8192 header max to be breached * Add REQUEST_MAX_HEADER_SIZE as config value * remove queue size --- sanic/app.py | 1 - sanic/config.py | 44 ++++++++++++++++++++++++++- sanic/http.py | 15 ++++++++-- tests/test_headers.py | 69 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 123 insertions(+), 6 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index fb8d36f5..8d33d735 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -183,7 +183,6 @@ class Sanic(BaseSanic): if register is not None: self.config.REGISTER = register - if self.config.REGISTER: self.__class__.register_app(self) diff --git a/sanic/config.py b/sanic/config.py index cb722f74..27699f80 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import Any, Dict, Optional, Union from warnings import warn +from sanic.http import Http + from .utils import load_module_from_file_location, str_to_bool @@ -28,6 +30,7 @@ DEFAULT_CONFIG = { "REAL_IP_HEADER": None, "REGISTER": True, "REQUEST_BUFFER_SIZE": 65536, # 64 KiB + "REQUEST_MAX_HEADER_SIZE": 8192, # 8 KiB, but cannot exceed 16384 "REQUEST_ID_HEADER": "X-Request-ID", "REQUEST_MAX_SIZE": 100000000, # 100 megabytes "REQUEST_TIMEOUT": 60, # 60 seconds @@ -42,12 +45,36 @@ DEFAULT_CONFIG = { class Config(dict): + ACCESS_LOG: bool + EVENT_AUTOREGISTER: bool + FALLBACK_ERROR_FORMAT: str + FORWARDED_FOR_HEADER: str + FORWARDED_SECRET: Optional[str] + GRACEFUL_SHUTDOWN_TIMEOUT: float + KEEP_ALIVE_TIMEOUT: int + KEEP_ALIVE: bool + PROXIES_COUNT: Optional[int] + REAL_IP_HEADER: Optional[str] + REGISTER: bool + REQUEST_BUFFER_SIZE: int + REQUEST_MAX_HEADER_SIZE: int + REQUEST_ID_HEADER: str + REQUEST_MAX_SIZE: int + REQUEST_TIMEOUT: int + RESPONSE_TIMEOUT: int + WEBSOCKET_MAX_QUEUE: int + WEBSOCKET_MAX_SIZE: int + WEBSOCKET_PING_INTERVAL: int + WEBSOCKET_PING_TIMEOUT: int + WEBSOCKET_READ_LIMIT: int + WEBSOCKET_WRITE_LIMIT: int + def __init__( self, defaults: Dict[str, Union[str, bool, int, float, None]] = None, load_env: Optional[Union[bool, str]] = True, env_prefix: Optional[str] = SANIC_PREFIX, - keep_alive: Optional[int] = None, + keep_alive: Optional[bool] = None, ): defaults = defaults or {} super().__init__({**DEFAULT_CONFIG, **defaults}) @@ -72,6 +99,8 @@ class Config(dict): else: self.load_environment_vars(SANIC_PREFIX) + self._configure_header_size() + def __getattr__(self, attr): try: return self[attr] @@ -80,6 +109,19 @@ class Config(dict): def __setattr__(self, attr, value): self[attr] = value + if attr in ( + "REQUEST_MAX_HEADER_SIZE", + "REQUEST_BUFFER_SIZE", + "REQUEST_MAX_SIZE", + ): + self._configure_header_size() + + def _configure_header_size(self): + Http.set_header_max_size( + self.REQUEST_MAX_HEADER_SIZE, + self.REQUEST_BUFFER_SIZE - 4096, + self.REQUEST_MAX_SIZE, + ) def load_environment_vars(self, prefix=SANIC_PREFIX): """ diff --git a/sanic/http.py b/sanic/http.py index 24e45d4b..80301aff 100644 --- a/sanic/http.py +++ b/sanic/http.py @@ -64,6 +64,9 @@ class Http: :raises RuntimeError: """ + HEADER_CEILING = 16_384 + HEADER_MAX_SIZE = 0 + __slots__ = [ "_send", "_receive_more", @@ -169,7 +172,6 @@ class Http: """ Receive and parse request header into self.request. """ - HEADER_MAX_SIZE = min(8192, self.request_max_size) # Receive until full header is in buffer buf = self.recv_buffer pos = 0 @@ -180,12 +182,12 @@ class Http: break pos = max(0, len(buf) - 3) - if pos >= HEADER_MAX_SIZE: + if pos >= self.HEADER_MAX_SIZE: break await self._receive_more() - if pos >= HEADER_MAX_SIZE: + if pos >= self.HEADER_MAX_SIZE: raise PayloadTooLarge("Request header exceeds the size limit") # Parse header content @@ -541,3 +543,10 @@ class Http: @property def send(self): return self.response_func + + @classmethod + def set_header_max_size(cls, *sizes: int): + cls.HEADER_MAX_SIZE = min( + *sizes, + cls.HEADER_CEILING, + ) diff --git a/tests/test_headers.py b/tests/test_headers.py index 4580a073..546a9ef7 100644 --- a/tests/test_headers.py +++ b/tests/test_headers.py @@ -7,6 +7,13 @@ from sanic.exceptions import PayloadTooLarge from sanic.http import Http +@pytest.fixture +def raised_ceiling(): + Http.HEADER_CEILING = 32_768 + yield + Http.HEADER_CEILING = 16_384 + + @pytest.mark.parametrize( "input, expected", [ @@ -76,15 +83,75 @@ async def test_header_size_exceeded(): recv_buffer += b"123" protocol = Mock() + Http.set_header_max_size(1) http = Http(protocol) http._receive_more = _receive_more - http.request_max_size = 1 http.recv_buffer = recv_buffer with pytest.raises(PayloadTooLarge): await http.http1_request_header() +@pytest.mark.asyncio +async def test_header_size_increased_okay(): + recv_buffer = bytearray() + + async def _receive_more(): + nonlocal recv_buffer + recv_buffer += b"123" + + protocol = Mock() + Http.set_header_max_size(12_288) + http = Http(protocol) + http._receive_more = _receive_more + http.recv_buffer = recv_buffer + + with pytest.raises(PayloadTooLarge): + await http.http1_request_header() + + assert len(recv_buffer) == 12_291 + + +@pytest.mark.asyncio +async def test_header_size_exceeded_maxed_out(): + recv_buffer = bytearray() + + async def _receive_more(): + nonlocal recv_buffer + recv_buffer += b"123" + + protocol = Mock() + Http.set_header_max_size(18_432) + http = Http(protocol) + http._receive_more = _receive_more + http.recv_buffer = recv_buffer + + with pytest.raises(PayloadTooLarge): + await http.http1_request_header() + + assert len(recv_buffer) == 16_389 + + +@pytest.mark.asyncio +async def test_header_size_exceeded_raised_ceiling(raised_ceiling): + recv_buffer = bytearray() + + async def _receive_more(): + nonlocal recv_buffer + recv_buffer += b"123" + + protocol = Mock() + http = Http(protocol) + Http.set_header_max_size(65_536) + http._receive_more = _receive_more + http.recv_buffer = recv_buffer + + with pytest.raises(PayloadTooLarge): + await http.http1_request_header() + + assert len(recv_buffer) == 32_772 + + def test_raw_headers(app): app.route("/")(lambda _: text("")) request, _ = app.test_client.get(