From e155fe403d8ed86762acff88ae2848f7a0c136cb Mon Sep 17 00:00:00 2001 From: ashleysommer Date: Thu, 18 May 2017 18:04:28 +1000 Subject: [PATCH 1/5] Add file_stream response handler For streaming large static files Like `file()` but breaks the file into chunks and sends it with a `StreamingHTTPResponse` Chunk size is configurable, but defaults to 4k, this seemed to be the sweet spot in my testing. Also supports ContentRange same as `file()` does. --- sanic/response.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/sanic/response.py b/sanic/response.py index 0fc3c575..34a53387 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -310,6 +310,55 @@ async def file(location, mime_type=None, headers=None, _range=None): body_bytes=out_stream) +async def file_stream(location, chunk_size=4096, mime_type=None, headers=None, + _range=None): + """Return a streaming response object with file data. + + :param location: Location of file on system. + :param chunk_size: The size of each chunk in the stream (in bytes) + :param mime_type: Specific mime_type. + :param headers: Custom Headers. + :param _range: + """ + filename = path.split(location)[-1] + + _file = await open_async(location, mode='rb') + + async def _streaming_fn(response): + nonlocal _file, chunk_size + try: + if _range: + chunk_size = min((_range.size, chunk_size)) + await _file.seek(_range.start) + to_send = _range.size + while to_send > 0: + content = await _file.read(chunk_size) + if len(content) < 1: + break + to_send -= len(content) + response.write(content) + else: + while True: + content = await _file.read(chunk_size) + if len(content) < 1: + break + response.write(content) + except Exception as e: + print(e) + finally: + await _file.close() + return # Returning from this fn closes the stream + + mime_type = mime_type or guess_type(filename)[0] or 'text/plain' + if _range: + headers['Content-Range'] = 'bytes %s-%s/%s' % ( + _range.start, _range.end, _range.total) + return StreamingHTTPResponse(streaming_fn=_streaming_fn, + status=200, + headers=headers, + content_type=mime_type) + + def stream( streaming_fn, status=200, headers=None, content_type="text/plain; charset=utf-8"): From 181977ad4ed8dac3443f5906a44883e035971f2c Mon Sep 17 00:00:00 2001 From: ashleysommer Date: Thu, 18 May 2017 18:12:26 +1000 Subject: [PATCH 2/5] Added brief documentation with an example for file_stream Added test to ensure `file_stream()` works in the test suite. --- docs/sanic/response.md | 10 ++++++++++ examples/try_everything.py | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/docs/sanic/response.md b/docs/sanic/response.md index 12718ca1..9c3c95f7 100644 --- a/docs/sanic/response.md +++ b/docs/sanic/response.md @@ -60,6 +60,16 @@ async def index(request): return response.stream(streaming_fn, content_type='text/plain') ``` +## File Streaming +For large files, a combination of File and Streaming above +```python +from sanic import response + +@app.route('/big_file.png') +async def handle_request(request): + return await response.file_stream('/srv/www/whatever.png') +``` + ## Redirect ```python diff --git a/examples/try_everything.py b/examples/try_everything.py index d46b832e..f380d925 100644 --- a/examples/try_everything.py +++ b/examples/try_everything.py @@ -37,6 +37,10 @@ async def test_await(request): async def test_file(request): return await response.file(os.path.abspath("setup.py")) +@app.route("/file_stream") +async def test_file_stream(request): + return await response.file_stream(os.path.abspath("setup.py"), + chunk_size=1024) # ----------------------------------------------- # # Exceptions From ff2ae11ac860f0365eb2cc0b011d0d75a4a40380 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Fri, 19 May 2017 13:00:01 +1000 Subject: [PATCH 3/5] Remove exception print(e) statement. --- sanic/response.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sanic/response.py b/sanic/response.py index 34a53387..9a304bb7 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -343,8 +343,6 @@ async def file_stream(location, chunk_size=4096, mime_type=None, headers=None, if len(content) < 1: break response.write(content) - except Exception as e: - print(e) finally: await _file.close() return # Returning from this fn closes the stream From 181edb723541c22d35db6ef06656adb8ade8b979 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Fri, 19 May 2017 13:01:21 +1000 Subject: [PATCH 4/5] Test `file()` and `file_stream()` response helpers. Added test for `file()` response helper and `file_stream()` response helper. --- tests/test_file_response.py | 70 +++++++++++++++++++++++++++ tests/test_file_stream_response.py | 77 ++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 tests/test_file_response.py create mode 100644 tests/test_file_stream_response.py diff --git a/tests/test_file_response.py b/tests/test_file_response.py new file mode 100644 index 00000000..7574215b --- /dev/null +++ b/tests/test_file_response.py @@ -0,0 +1,70 @@ +import inspect +import os +from mimetypes import guess_type +from urllib.parse import unquote +from aiofiles import os as async_os +import pytest + +from sanic import Sanic +from sanic.response import file, HTTPResponse + + +@pytest.fixture(scope='module') +def static_file_directory(): + """The static directory to serve""" + current_file = inspect.getfile(inspect.currentframe()) + current_directory = os.path.dirname(os.path.abspath(current_file)) + static_directory = os.path.join(current_directory, 'static') + return static_directory + + +def get_file_path(static_file_directory, file_name): + return os.path.join(static_file_directory, file_name) + + +def get_file_content(static_file_directory, file_name): + """The content of the static file to check""" + with open(get_file_path(static_file_directory, file_name), 'rb') as file: + return file.read() + + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) +def test_file_helper(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET']) + def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + return file(file_path, mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.get('/files/{}'.format(file_name)) + assert response.status == 200 + assert response.body == get_file_content(static_file_directory, file_name) + + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt']) +def test_file_helper_head_request(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET', 'HEAD']) + async def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + stats = await async_os.stat(file_path) + headers = dict() + headers['Accept-Ranges'] = 'bytes' + headers['Content-Length'] = str(stats.st_size) + if request.method == "HEAD": + return HTTPResponse( + headers=headers, + content_type=guess_type(file_path)[0] or 'text/plain') + else: + return file(file_path, headers=headers, + mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.head('/files/{}'.format(file_name)) + assert response.status == 200 + assert 'Accept-Ranges' in response.headers + assert 'Content-Length' in response.headers + assert int(response.headers[ + 'Content-Length']) == len( + get_file_content(static_file_directory, file_name)) diff --git a/tests/test_file_stream_response.py b/tests/test_file_stream_response.py new file mode 100644 index 00000000..2a79a683 --- /dev/null +++ b/tests/test_file_stream_response.py @@ -0,0 +1,77 @@ +import inspect +import os +from mimetypes import guess_type +from urllib.parse import unquote +from aiofiles import os as async_os +import pytest + +from sanic import Sanic +from sanic.response import file_stream, HTTPResponse + + +@pytest.fixture(scope='module') +def static_file_directory(): + """The static directory to serve""" + current_file = inspect.getfile(inspect.currentframe()) + current_directory = os.path.dirname(os.path.abspath(current_file)) + static_directory = os.path.join(current_directory, 'static') + return static_directory + + +def get_file_path(static_file_directory, file_name): + return os.path.join(static_file_directory, file_name) + + +def get_file_content(static_file_directory, file_name): + """The content of the static file to check""" + with open(get_file_path(static_file_directory, file_name), 'rb') as file: + return file.read() + + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) +def test_file_stream_helper(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET']) + def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + return file_stream(file_path, chunk_size=32, + mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.get('/files/{}'.format(file_name)) + assert response.status == 200 + assert response.body == get_file_content(static_file_directory, file_name) + + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt']) +def test_file_helper_head_request(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET', 'HEAD']) + async def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + headers = dict() + headers['Accept-Ranges'] = 'bytes' + if request.method == "HEAD": + # Return a normal HTTPResponse, not a + # StreamingHTTPResponse for a HEAD request + stats = await async_os.stat(file_path) + headers['Content-Length'] = str(stats.st_size) + return HTTPResponse( + headers=headers, + content_type=guess_type(file_path)[0] or 'text/plain') + else: + return file_stream(file_path, chunk_size=32, headers=headers, + mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.head('/files/{}'.format(file_name)) + assert response.status == 200 + # A HEAD request should never be streamed/chunked. + if 'Transfer-Encoding' in response.headers: + assert response.headers['Transfer-Encoding'] != "chunked" + assert 'Accept-Ranges' in response.headers + # A HEAD request should get the Content-Length too + assert 'Content-Length' in response.headers + assert int(response.headers[ + 'Content-Length']) == len( + get_file_content(static_file_directory, file_name)) From 5c5656f98187b7fbbb0476af62ff4b0954bda83f Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Sat, 20 May 2017 09:41:36 +1000 Subject: [PATCH 5/5] Moved file_stream tests to test_responses.py --- tests/test_file_response.py | 70 ------------------ tests/test_file_stream_response.py | 77 ------------------- tests/test_response.py | 115 ++++++++++++++++++++++++++++- 3 files changed, 113 insertions(+), 149 deletions(-) delete mode 100644 tests/test_file_response.py delete mode 100644 tests/test_file_stream_response.py diff --git a/tests/test_file_response.py b/tests/test_file_response.py deleted file mode 100644 index 7574215b..00000000 --- a/tests/test_file_response.py +++ /dev/null @@ -1,70 +0,0 @@ -import inspect -import os -from mimetypes import guess_type -from urllib.parse import unquote -from aiofiles import os as async_os -import pytest - -from sanic import Sanic -from sanic.response import file, HTTPResponse - - -@pytest.fixture(scope='module') -def static_file_directory(): - """The static directory to serve""" - current_file = inspect.getfile(inspect.currentframe()) - current_directory = os.path.dirname(os.path.abspath(current_file)) - static_directory = os.path.join(current_directory, 'static') - return static_directory - - -def get_file_path(static_file_directory, file_name): - return os.path.join(static_file_directory, file_name) - - -def get_file_content(static_file_directory, file_name): - """The content of the static file to check""" - with open(get_file_path(static_file_directory, file_name), 'rb') as file: - return file.read() - - -@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) -def test_file_helper(file_name, static_file_directory): - app = Sanic('test_file_helper') - @app.route('/files/', methods=['GET']) - def file_route(request, filename): - file_path = os.path.join(static_file_directory, filename) - file_path = os.path.abspath(unquote(file_path)) - return file(file_path, mime_type=guess_type(file_path)[0] or 'text/plain') - - request, response = app.test_client.get('/files/{}'.format(file_name)) - assert response.status == 200 - assert response.body == get_file_content(static_file_directory, file_name) - - -@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt']) -def test_file_helper_head_request(file_name, static_file_directory): - app = Sanic('test_file_helper') - @app.route('/files/', methods=['GET', 'HEAD']) - async def file_route(request, filename): - file_path = os.path.join(static_file_directory, filename) - file_path = os.path.abspath(unquote(file_path)) - stats = await async_os.stat(file_path) - headers = dict() - headers['Accept-Ranges'] = 'bytes' - headers['Content-Length'] = str(stats.st_size) - if request.method == "HEAD": - return HTTPResponse( - headers=headers, - content_type=guess_type(file_path)[0] or 'text/plain') - else: - return file(file_path, headers=headers, - mime_type=guess_type(file_path)[0] or 'text/plain') - - request, response = app.test_client.head('/files/{}'.format(file_name)) - assert response.status == 200 - assert 'Accept-Ranges' in response.headers - assert 'Content-Length' in response.headers - assert int(response.headers[ - 'Content-Length']) == len( - get_file_content(static_file_directory, file_name)) diff --git a/tests/test_file_stream_response.py b/tests/test_file_stream_response.py deleted file mode 100644 index 2a79a683..00000000 --- a/tests/test_file_stream_response.py +++ /dev/null @@ -1,77 +0,0 @@ -import inspect -import os -from mimetypes import guess_type -from urllib.parse import unquote -from aiofiles import os as async_os -import pytest - -from sanic import Sanic -from sanic.response import file_stream, HTTPResponse - - -@pytest.fixture(scope='module') -def static_file_directory(): - """The static directory to serve""" - current_file = inspect.getfile(inspect.currentframe()) - current_directory = os.path.dirname(os.path.abspath(current_file)) - static_directory = os.path.join(current_directory, 'static') - return static_directory - - -def get_file_path(static_file_directory, file_name): - return os.path.join(static_file_directory, file_name) - - -def get_file_content(static_file_directory, file_name): - """The content of the static file to check""" - with open(get_file_path(static_file_directory, file_name), 'rb') as file: - return file.read() - - -@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) -def test_file_stream_helper(file_name, static_file_directory): - app = Sanic('test_file_helper') - @app.route('/files/', methods=['GET']) - def file_route(request, filename): - file_path = os.path.join(static_file_directory, filename) - file_path = os.path.abspath(unquote(file_path)) - return file_stream(file_path, chunk_size=32, - mime_type=guess_type(file_path)[0] or 'text/plain') - - request, response = app.test_client.get('/files/{}'.format(file_name)) - assert response.status == 200 - assert response.body == get_file_content(static_file_directory, file_name) - - -@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt']) -def test_file_helper_head_request(file_name, static_file_directory): - app = Sanic('test_file_helper') - @app.route('/files/', methods=['GET', 'HEAD']) - async def file_route(request, filename): - file_path = os.path.join(static_file_directory, filename) - file_path = os.path.abspath(unquote(file_path)) - headers = dict() - headers['Accept-Ranges'] = 'bytes' - if request.method == "HEAD": - # Return a normal HTTPResponse, not a - # StreamingHTTPResponse for a HEAD request - stats = await async_os.stat(file_path) - headers['Content-Length'] = str(stats.st_size) - return HTTPResponse( - headers=headers, - content_type=guess_type(file_path)[0] or 'text/plain') - else: - return file_stream(file_path, chunk_size=32, headers=headers, - mime_type=guess_type(file_path)[0] or 'text/plain') - - request, response = app.test_client.head('/files/{}'.format(file_name)) - assert response.status == 200 - # A HEAD request should never be streamed/chunked. - if 'Transfer-Encoding' in response.headers: - assert response.headers['Transfer-Encoding'] != "chunked" - assert 'Accept-Ranges' in response.headers - # A HEAD request should get the Content-Length too - assert 'Content-Length' in response.headers - assert int(response.headers[ - 'Content-Length']) == len( - get_file_content(static_file_directory, file_name)) diff --git a/tests/test_response.py b/tests/test_response.py index ff5fd42b..ba87f68d 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -1,13 +1,20 @@ import asyncio +import inspect +import os +from aiofiles import os as async_os +from mimetypes import guess_type +from urllib.parse import unquote + import pytest from random import choice from sanic import Sanic -from sanic.response import HTTPResponse, stream, StreamingHTTPResponse +from sanic.response import HTTPResponse, stream, StreamingHTTPResponse, file, file_stream from sanic.testing import HOST, PORT - from unittest.mock import MagicMock + + def test_response_body_not_a_string(): """Test when a response body sent from the application is not a string""" app = Sanic('response_body_not_a_string') @@ -94,3 +101,107 @@ def test_stream_response_writes_correct_content_to_transport(streaming_app): app.stop() streaming_app.run(host=HOST, port=PORT) + + +@pytest.fixture +def static_file_directory(): + """The static directory to serve""" + current_file = inspect.getfile(inspect.currentframe()) + current_directory = os.path.dirname(os.path.abspath(current_file)) + static_directory = os.path.join(current_directory, 'static') + return static_directory + + +def get_file_content(static_file_directory, file_name): + """The content of the static file to check""" + with open(os.path.join(static_file_directory, file_name), 'rb') as file: + return file.read() + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) +def test_file_response(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET']) + def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + return file(file_path, mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.get('/files/{}'.format(file_name)) + assert response.status == 200 + assert response.body == get_file_content(static_file_directory, file_name) + + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt']) +def test_file_head_response(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET', 'HEAD']) + async def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + stats = await async_os.stat(file_path) + headers = dict() + headers['Accept-Ranges'] = 'bytes' + headers['Content-Length'] = str(stats.st_size) + if request.method == "HEAD": + return HTTPResponse( + headers=headers, + content_type=guess_type(file_path)[0] or 'text/plain') + else: + return file(file_path, headers=headers, + mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.head('/files/{}'.format(file_name)) + assert response.status == 200 + assert 'Accept-Ranges' in response.headers + assert 'Content-Length' in response.headers + assert int(response.headers[ + 'Content-Length']) == len( + get_file_content(static_file_directory, file_name)) + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt', 'python.png']) +def test_file_stream_response(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET']) + def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + return file_stream(file_path, chunk_size=32, + mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.get('/files/{}'.format(file_name)) + assert response.status == 200 + assert response.body == get_file_content(static_file_directory, file_name) + + +@pytest.mark.parametrize('file_name', ['test.file', 'decode me.txt']) +def test_file_stream_head_response(file_name, static_file_directory): + app = Sanic('test_file_helper') + @app.route('/files/', methods=['GET', 'HEAD']) + async def file_route(request, filename): + file_path = os.path.join(static_file_directory, filename) + file_path = os.path.abspath(unquote(file_path)) + headers = dict() + headers['Accept-Ranges'] = 'bytes' + if request.method == "HEAD": + # Return a normal HTTPResponse, not a + # StreamingHTTPResponse for a HEAD request + stats = await async_os.stat(file_path) + headers['Content-Length'] = str(stats.st_size) + return HTTPResponse( + headers=headers, + content_type=guess_type(file_path)[0] or 'text/plain') + else: + return file_stream(file_path, chunk_size=32, headers=headers, + mime_type=guess_type(file_path)[0] or 'text/plain') + + request, response = app.test_client.head('/files/{}'.format(file_name)) + assert response.status == 200 + # A HEAD request should never be streamed/chunked. + if 'Transfer-Encoding' in response.headers: + assert response.headers['Transfer-Encoding'] != "chunked" + assert 'Accept-Ranges' in response.headers + # A HEAD request should get the Content-Length too + assert 'Content-Length' in response.headers + assert int(response.headers[ + 'Content-Length']) == len( + get_file_content(static_file_directory, file_name)) \ No newline at end of file