From 9f07109616abac730376a9d32aef533dc45e7bce Mon Sep 17 00:00:00 2001 From: andreymal Date: Sat, 20 Apr 2019 22:26:30 +0300 Subject: [PATCH 1/5] Allow to disable Transfer-Encoding: chunked --- sanic/response.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/sanic/response.py b/sanic/response.py index 43e24877..bc05e2ea 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -59,16 +59,23 @@ class StreamingHTTPResponse(BaseHTTPResponse): "status", "content_type", "headers", + "chunked", "_cookies", ) def __init__( - self, streaming_fn, status=200, headers=None, content_type="text/plain" + self, + streaming_fn, + status=200, + headers=None, + content_type="text/plain", + chunked=None, ): self.content_type = content_type self.streaming_fn = streaming_fn self.status = status self.headers = CIMultiDict(headers or {}) + self.chunked = chunked self._cookies = None async def write(self, data): @@ -79,7 +86,10 @@ class StreamingHTTPResponse(BaseHTTPResponse): if type(data) != bytes: data = self._encode_body(data) - self.protocol.push_data(b"%x\r\n%b\r\n" % (len(data), data)) + if self.chunked: + self.protocol.push_data(b"%x\r\n%b\r\n" % (len(data), data)) + else: + self.protocol.push_data(data) await self.protocol.drain() async def stream( @@ -88,6 +98,8 @@ class StreamingHTTPResponse(BaseHTTPResponse): """Streams headers, runs the `streaming_fn` callback that writes content to the response body, then finalizes the response body. """ + if self.chunked is None: + self.chunked = version != "1.0" headers = self.get_headers( version, keep_alive=keep_alive, @@ -96,7 +108,8 @@ class StreamingHTTPResponse(BaseHTTPResponse): self.protocol.push_data(headers) await self.protocol.drain() await self.streaming_fn(self) - self.protocol.push_data(b"0\r\n\r\n") + if self.chunked: + self.protocol.push_data(b"0\r\n\r\n") # no need to await drain here after this write, because it is the # very last thing we write and nothing needs to wait for it. @@ -109,8 +122,13 @@ class StreamingHTTPResponse(BaseHTTPResponse): if keep_alive and keep_alive_timeout is not None: timeout_header = b"Keep-Alive: %d\r\n" % keep_alive_timeout - self.headers["Transfer-Encoding"] = "chunked" - self.headers.pop("Content-Length", None) + chunked = self.chunked + if chunked is None: + chunked = version != "1.0" + + if chunked: + self.headers["Transfer-Encoding"] = "chunked" + self.headers.pop("Content-Length", None) self.headers["Content-Type"] = self.headers.get( "Content-Type", self.content_type ) @@ -327,6 +345,7 @@ async def file_stream( mime_type=None, headers=None, filename=None, + chunked=None, _range=None, ): """Return a streaming response object with file data. @@ -336,6 +355,7 @@ async def file_stream( :param mime_type: Specific mime_type. :param headers: Custom Headers. :param filename: Override filename. + :param chunked: Enable or disable chunked transfer-encoding (default: auto) :param _range: """ headers = headers or {} @@ -383,6 +403,7 @@ async def file_stream( status=status, headers=headers, content_type=mime_type, + chunked=chunked, ) @@ -391,6 +412,7 @@ def stream( status=200, headers=None, content_type="text/plain; charset=utf-8", + chunked=None, ): """Accepts an coroutine `streaming_fn` which can be used to write chunks to a streaming response. Returns a `StreamingHTTPResponse`. @@ -409,9 +431,14 @@ def stream( writes content to that response. :param mime_type: Specific mime_type. :param headers: Custom Headers. + :param chunked: Enable or disable chunked transfer-encoding (default: auto) """ return StreamingHTTPResponse( - streaming_fn, headers=headers, content_type=content_type, status=status + streaming_fn, + headers=headers, + content_type=content_type, + status=status, + chunked=chunked, ) From 03855d316ba9b454f3cb2c23cb5f2f5f3f4221f8 Mon Sep 17 00:00:00 2001 From: andreymal Date: Sat, 20 Apr 2019 22:27:10 +0300 Subject: [PATCH 2/5] Update tests for StreamingHTTPResponse --- tests/test_response.py | 99 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 93 insertions(+), 6 deletions(-) diff --git a/tests/test_response.py b/tests/test_response.py index 256a37c6..9edfe1e0 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -192,22 +192,59 @@ def test_no_content(json_app): def streaming_app(app): @app.route("/") async def test(request): - return stream(sample_streaming_fn, content_type="text/csv") + return stream( + sample_streaming_fn, + headers={"Content-Length": "7"}, + content_type="text/csv", + ) return app -def test_streaming_adds_correct_headers(streaming_app): +@pytest.fixture +def non_chunked_streaming_app(app): + @app.route("/") + async def test(request): + return stream( + sample_streaming_fn, + headers={"Content-Length": "7"}, + content_type="text/csv", + chunked=False, + ) + + return app + + +def test_chunked_streaming_adds_correct_headers(streaming_app): request, response = streaming_app.test_client.get("/") assert response.headers["Transfer-Encoding"] == "chunked" assert response.headers["Content-Type"] == "text/csv" + # Content-Length is not allowed by HTTP/1.1 specification + # when "Transfer-Encoding: chunked" is used + assert "Content-Length" not in response.headers -def test_streaming_returns_correct_content(streaming_app): +def test_chunked_streaming_returns_correct_content(streaming_app): request, response = streaming_app.test_client.get("/") assert response.text == "foo,bar" +def test_non_chunked_streaming_adds_correct_headers( + non_chunked_streaming_app +): + request, response = non_chunked_streaming_app.test_client.get("/") + assert "Transfer-Encoding" not in response.headers + assert response.headers["Content-Type"] == "text/csv" + assert response.headers["Content-Length"] == "7" + + +def test_non_chunked_streaming_returns_correct_content( + non_chunked_streaming_app +): + request, response = non_chunked_streaming_app.test_client.get("/") + assert response.text == "foo,bar" + + @pytest.mark.parametrize("status", [200, 201, 400, 401]) def test_stream_response_status_returns_correct_headers(status): response = StreamingHTTPResponse(sample_streaming_fn, status=status) @@ -227,13 +264,27 @@ def test_stream_response_keep_alive_returns_correct_headers( assert b"Keep-Alive: %s\r\n" % str(keep_alive_timeout).encode() in headers -def test_stream_response_includes_chunked_header(): +def test_stream_response_includes_chunked_header_http11(): response = StreamingHTTPResponse(sample_streaming_fn) - headers = response.get_headers() + headers = response.get_headers(version="1.1") assert b"Transfer-Encoding: chunked\r\n" in headers -def test_stream_response_writes_correct_content_to_transport(streaming_app): +def test_stream_response_does_not_include_chunked_header_http10(): + response = StreamingHTTPResponse(sample_streaming_fn) + headers = response.get_headers(version="1.0") + assert b"Transfer-Encoding: chunked\r\n" not in headers + + +def test_stream_response_does_not_include_chunked_header_if_disabled(): + response = StreamingHTTPResponse(sample_streaming_fn, chunked=False) + headers = response.get_headers(version="1.1") + assert b"Transfer-Encoding: chunked\r\n" not in headers + + +def test_stream_response_writes_correct_content_to_transport_when_chunked( + streaming_app +): response = StreamingHTTPResponse(sample_streaming_fn) response.protocol = MagicMock(HttpProtocol) response.protocol.transport = MagicMock(asyncio.Transport) @@ -262,6 +313,42 @@ def test_stream_response_writes_correct_content_to_transport(streaming_app): b"0\r\n\r\n" ) + assert len(response.protocol.transport.write.call_args_list) == 4 + + app.stop() + + streaming_app.run(host=HOST, port=PORT) + + +def test_stream_response_writes_correct_content_to_transport_when_not_chunked( + streaming_app, +): + response = StreamingHTTPResponse(sample_streaming_fn) + response.protocol = MagicMock(HttpProtocol) + response.protocol.transport = MagicMock(asyncio.Transport) + + async def mock_drain(): + pass + + def mock_push_data(data): + response.protocol.transport.write(data) + + response.protocol.push_data = mock_push_data + response.protocol.drain = mock_drain + + @streaming_app.listener("after_server_start") + async def run_stream(app, loop): + await response.stream(version="1.0") + assert response.protocol.transport.write.call_args_list[1][0][0] == ( + b"foo," + ) + + assert response.protocol.transport.write.call_args_list[2][0][0] == ( + b"bar" + ) + + assert len(response.protocol.transport.write.call_args_list) == 3 + app.stop() streaming_app.run(host=HOST, port=PORT) From 6be12ba7739b061ab9ec1c5bf80d15cdde03a438 Mon Sep 17 00:00:00 2001 From: andreymal Date: Sat, 20 Apr 2019 23:37:45 +0300 Subject: [PATCH 3/5] Upadte documentation for streaming response --- docs/sanic/response.md | 2 ++ docs/sanic/streaming.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/sanic/response.md b/docs/sanic/response.md index 47901c05..11034727 100644 --- a/docs/sanic/response.md +++ b/docs/sanic/response.md @@ -60,6 +60,8 @@ async def index(request): return response.stream(streaming_fn, content_type='text/plain') ``` +See [Streaming](streaming.md) for more information. + ## File Streaming For large files, a combination of File and Streaming above ```python diff --git a/docs/sanic/streaming.md b/docs/sanic/streaming.md index 769d269d..53d23ea3 100644 --- a/docs/sanic/streaming.md +++ b/docs/sanic/streaming.md @@ -117,3 +117,5 @@ async def index(request): return stream(stream_from_db) ``` + +If a client supports HTTP/1.1, Sanic will use [chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding); you can explicitly enable or disable it using `chunked` option of the `stream` function. From 9615e37ef9e2f6a13a9925974b0b701c9b03ef29 Mon Sep 17 00:00:00 2001 From: andreymal Date: Sat, 20 Apr 2019 23:50:19 +0300 Subject: [PATCH 4/5] Add file streaming section to the streaming documentation page --- docs/sanic/streaming.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/sanic/streaming.md b/docs/sanic/streaming.md index 53d23ea3..29e28bf7 100644 --- a/docs/sanic/streaming.md +++ b/docs/sanic/streaming.md @@ -119,3 +119,25 @@ async def index(request): ``` If a client supports HTTP/1.1, Sanic will use [chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding); you can explicitly enable or disable it using `chunked` option of the `stream` function. + +## File Streaming + +Sanic provides `sanic.response.file_stream` function that is useful when you want to send a large file. It returns a `StreamingHTTPResponse` object and will use chunked transfer encoding by default; for this reason Sanic doesn't add `Content-Length` HTTP header in the response. If you want to use this header, you can disable chunked transfer encoding and add it manually: + +```python +from aiofiles import os as async_os +from sanic.response import file_stream + +@app.route("/") +async def index(request): + file_path = "/srv/www/whatever.png" + + file_stat = await async_os.stat(file_path) + headers = {"Content-Length": str(file_stat.st_size)} + + return await file_stream( + file_path, + headers=headers, + chunked=False, + ) +``` From 7d6e60ab7dbe2ab92a451cf75bb8ca0a4eb69efb Mon Sep 17 00:00:00 2001 From: andreymal Date: Mon, 22 Apr 2019 10:52:38 +0300 Subject: [PATCH 5/5] Never use chunked transfer encoding for HTTP/1.0 --- sanic/response.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/sanic/response.py b/sanic/response.py index bc05e2ea..be178eff 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -69,7 +69,7 @@ class StreamingHTTPResponse(BaseHTTPResponse): status=200, headers=None, content_type="text/plain", - chunked=None, + chunked=True, ): self.content_type = content_type self.streaming_fn = streaming_fn @@ -98,8 +98,8 @@ class StreamingHTTPResponse(BaseHTTPResponse): """Streams headers, runs the `streaming_fn` callback that writes content to the response body, then finalizes the response body. """ - if self.chunked is None: - self.chunked = version != "1.0" + if version != "1.1": + self.chunked = False headers = self.get_headers( version, keep_alive=keep_alive, @@ -122,11 +122,7 @@ class StreamingHTTPResponse(BaseHTTPResponse): if keep_alive and keep_alive_timeout is not None: timeout_header = b"Keep-Alive: %d\r\n" % keep_alive_timeout - chunked = self.chunked - if chunked is None: - chunked = version != "1.0" - - if chunked: + if self.chunked and version == "1.1": self.headers["Transfer-Encoding"] = "chunked" self.headers.pop("Content-Length", None) self.headers["Content-Type"] = self.headers.get( @@ -345,7 +341,7 @@ async def file_stream( mime_type=None, headers=None, filename=None, - chunked=None, + chunked=True, _range=None, ): """Return a streaming response object with file data. @@ -355,7 +351,7 @@ async def file_stream( :param mime_type: Specific mime_type. :param headers: Custom Headers. :param filename: Override filename. - :param chunked: Enable or disable chunked transfer-encoding (default: auto) + :param chunked: Enable or disable chunked transfer-encoding :param _range: """ headers = headers or {} @@ -412,7 +408,7 @@ def stream( status=200, headers=None, content_type="text/plain; charset=utf-8", - chunked=None, + chunked=True, ): """Accepts an coroutine `streaming_fn` which can be used to write chunks to a streaming response. Returns a `StreamingHTTPResponse`. @@ -431,7 +427,7 @@ def stream( writes content to that response. :param mime_type: Specific mime_type. :param headers: Custom Headers. - :param chunked: Enable or disable chunked transfer-encoding (default: auto) + :param chunked: Enable or disable chunked transfer-encoding """ return StreamingHTTPResponse( streaming_fn,