Allow 8192 header max to be breached (#2155)

* Allow 8192 header max to be breached

* Add REQUEST_MAX_HEADER_SIZE as config value

* remove queue size
This commit is contained in:
Adam Hopkins 2021-06-04 13:56:29 +03:00 committed by GitHub
parent a140c47195
commit 141be0028d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 123 additions and 6 deletions

View File

@ -183,7 +183,6 @@ class Sanic(BaseSanic):
if register is not None: if register is not None:
self.config.REGISTER = register self.config.REGISTER = register
if self.config.REGISTER: if self.config.REGISTER:
self.__class__.register_app(self) self.__class__.register_app(self)

View File

@ -4,6 +4,8 @@ from pathlib import Path
from typing import Any, Dict, Optional, Union from typing import Any, Dict, Optional, Union
from warnings import warn from warnings import warn
from sanic.http import Http
from .utils import load_module_from_file_location, str_to_bool from .utils import load_module_from_file_location, str_to_bool
@ -28,6 +30,7 @@ DEFAULT_CONFIG = {
"REAL_IP_HEADER": None, "REAL_IP_HEADER": None,
"REGISTER": True, "REGISTER": True,
"REQUEST_BUFFER_SIZE": 65536, # 64 KiB "REQUEST_BUFFER_SIZE": 65536, # 64 KiB
"REQUEST_MAX_HEADER_SIZE": 8192, # 8 KiB, but cannot exceed 16384
"REQUEST_ID_HEADER": "X-Request-ID", "REQUEST_ID_HEADER": "X-Request-ID",
"REQUEST_MAX_SIZE": 100000000, # 100 megabytes "REQUEST_MAX_SIZE": 100000000, # 100 megabytes
"REQUEST_TIMEOUT": 60, # 60 seconds "REQUEST_TIMEOUT": 60, # 60 seconds
@ -42,12 +45,36 @@ DEFAULT_CONFIG = {
class Config(dict): class Config(dict):
ACCESS_LOG: bool
EVENT_AUTOREGISTER: bool
FALLBACK_ERROR_FORMAT: str
FORWARDED_FOR_HEADER: str
FORWARDED_SECRET: Optional[str]
GRACEFUL_SHUTDOWN_TIMEOUT: float
KEEP_ALIVE_TIMEOUT: int
KEEP_ALIVE: bool
PROXIES_COUNT: Optional[int]
REAL_IP_HEADER: Optional[str]
REGISTER: bool
REQUEST_BUFFER_SIZE: int
REQUEST_MAX_HEADER_SIZE: int
REQUEST_ID_HEADER: str
REQUEST_MAX_SIZE: int
REQUEST_TIMEOUT: int
RESPONSE_TIMEOUT: int
WEBSOCKET_MAX_QUEUE: int
WEBSOCKET_MAX_SIZE: int
WEBSOCKET_PING_INTERVAL: int
WEBSOCKET_PING_TIMEOUT: int
WEBSOCKET_READ_LIMIT: int
WEBSOCKET_WRITE_LIMIT: int
def __init__( def __init__(
self, self,
defaults: Dict[str, Union[str, bool, int, float, None]] = None, defaults: Dict[str, Union[str, bool, int, float, None]] = None,
load_env: Optional[Union[bool, str]] = True, load_env: Optional[Union[bool, str]] = True,
env_prefix: Optional[str] = SANIC_PREFIX, env_prefix: Optional[str] = SANIC_PREFIX,
keep_alive: Optional[int] = None, keep_alive: Optional[bool] = None,
): ):
defaults = defaults or {} defaults = defaults or {}
super().__init__({**DEFAULT_CONFIG, **defaults}) super().__init__({**DEFAULT_CONFIG, **defaults})
@ -72,6 +99,8 @@ class Config(dict):
else: else:
self.load_environment_vars(SANIC_PREFIX) self.load_environment_vars(SANIC_PREFIX)
self._configure_header_size()
def __getattr__(self, attr): def __getattr__(self, attr):
try: try:
return self[attr] return self[attr]
@ -80,6 +109,19 @@ class Config(dict):
def __setattr__(self, attr, value): def __setattr__(self, attr, value):
self[attr] = value self[attr] = value
if attr in (
"REQUEST_MAX_HEADER_SIZE",
"REQUEST_BUFFER_SIZE",
"REQUEST_MAX_SIZE",
):
self._configure_header_size()
def _configure_header_size(self):
Http.set_header_max_size(
self.REQUEST_MAX_HEADER_SIZE,
self.REQUEST_BUFFER_SIZE - 4096,
self.REQUEST_MAX_SIZE,
)
def load_environment_vars(self, prefix=SANIC_PREFIX): def load_environment_vars(self, prefix=SANIC_PREFIX):
""" """

View File

@ -64,6 +64,9 @@ class Http:
:raises RuntimeError: :raises RuntimeError:
""" """
HEADER_CEILING = 16_384
HEADER_MAX_SIZE = 0
__slots__ = [ __slots__ = [
"_send", "_send",
"_receive_more", "_receive_more",
@ -169,7 +172,6 @@ class Http:
""" """
Receive and parse request header into self.request. Receive and parse request header into self.request.
""" """
HEADER_MAX_SIZE = min(8192, self.request_max_size)
# Receive until full header is in buffer # Receive until full header is in buffer
buf = self.recv_buffer buf = self.recv_buffer
pos = 0 pos = 0
@ -180,12 +182,12 @@ class Http:
break break
pos = max(0, len(buf) - 3) pos = max(0, len(buf) - 3)
if pos >= HEADER_MAX_SIZE: if pos >= self.HEADER_MAX_SIZE:
break break
await self._receive_more() await self._receive_more()
if pos >= HEADER_MAX_SIZE: if pos >= self.HEADER_MAX_SIZE:
raise PayloadTooLarge("Request header exceeds the size limit") raise PayloadTooLarge("Request header exceeds the size limit")
# Parse header content # Parse header content
@ -541,3 +543,10 @@ class Http:
@property @property
def send(self): def send(self):
return self.response_func return self.response_func
@classmethod
def set_header_max_size(cls, *sizes: int):
cls.HEADER_MAX_SIZE = min(
*sizes,
cls.HEADER_CEILING,
)

View File

@ -7,6 +7,13 @@ from sanic.exceptions import PayloadTooLarge
from sanic.http import Http from sanic.http import Http
@pytest.fixture
def raised_ceiling():
Http.HEADER_CEILING = 32_768
yield
Http.HEADER_CEILING = 16_384
@pytest.mark.parametrize( @pytest.mark.parametrize(
"input, expected", "input, expected",
[ [
@ -76,15 +83,75 @@ async def test_header_size_exceeded():
recv_buffer += b"123" recv_buffer += b"123"
protocol = Mock() protocol = Mock()
Http.set_header_max_size(1)
http = Http(protocol) http = Http(protocol)
http._receive_more = _receive_more http._receive_more = _receive_more
http.request_max_size = 1
http.recv_buffer = recv_buffer http.recv_buffer = recv_buffer
with pytest.raises(PayloadTooLarge): with pytest.raises(PayloadTooLarge):
await http.http1_request_header() await http.http1_request_header()
@pytest.mark.asyncio
async def test_header_size_increased_okay():
recv_buffer = bytearray()
async def _receive_more():
nonlocal recv_buffer
recv_buffer += b"123"
protocol = Mock()
Http.set_header_max_size(12_288)
http = Http(protocol)
http._receive_more = _receive_more
http.recv_buffer = recv_buffer
with pytest.raises(PayloadTooLarge):
await http.http1_request_header()
assert len(recv_buffer) == 12_291
@pytest.mark.asyncio
async def test_header_size_exceeded_maxed_out():
recv_buffer = bytearray()
async def _receive_more():
nonlocal recv_buffer
recv_buffer += b"123"
protocol = Mock()
Http.set_header_max_size(18_432)
http = Http(protocol)
http._receive_more = _receive_more
http.recv_buffer = recv_buffer
with pytest.raises(PayloadTooLarge):
await http.http1_request_header()
assert len(recv_buffer) == 16_389
@pytest.mark.asyncio
async def test_header_size_exceeded_raised_ceiling(raised_ceiling):
recv_buffer = bytearray()
async def _receive_more():
nonlocal recv_buffer
recv_buffer += b"123"
protocol = Mock()
http = Http(protocol)
Http.set_header_max_size(65_536)
http._receive_more = _receive_more
http.recv_buffer = recv_buffer
with pytest.raises(PayloadTooLarge):
await http.http1_request_header()
assert len(recv_buffer) == 32_772
def test_raw_headers(app): def test_raw_headers(app):
app.route("/")(lambda _: text("")) app.route("/")(lambda _: text(""))
request, _ = app.test_client.get( request, _ = app.test_client.get(