From 1b7f40ae532420ddcf37b7a60323ac0b7cc023de Mon Sep 17 00:00:00 2001 From: Jonathan Piron Date: Mon, 13 Jun 2016 09:44:31 +0200 Subject: [PATCH 1/2] Fix some python3 encoding issues --- rest_framework_proxy/utils.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/rest_framework_proxy/utils.py b/rest_framework_proxy/utils.py index c092a92..a2b7d7e 100644 --- a/rest_framework_proxy/utils.py +++ b/rest_framework_proxy/utils.py @@ -27,12 +27,12 @@ def __iter__(self): def generator(self): for (k, v) in self.data.items(): - yield b'%s\r\n\r\n' % self.build_multipart_header(k) - yield b'%s\r\n' % v + yield self.build_multipart_header(k) + b'\r\n\r\n' + yield '{}\r\n'.format(v).encode() for (k, v) in self.files.items(): content_type = mimetypes.guess_type(v.name)[0] or 'application/octet-stream' - yield b'%s\r\n\r\n' % self.build_multipart_header(k, v.name, content_type) + yield self.build_multipart_header(k, v.name, content_type) + b'%s\r\n\r\n' # Seek back to start as __len__ has already read the file v.seek(0) @@ -48,17 +48,16 @@ def generator(self): def build_multipart_header(self, name, filename=None, content_type=None): output = [] - output.append('--%s' % self.boundary) + output.append(('--%s' % self.boundary).encode()) string = 'Content-Disposition: form-data; name="%s"' % name if filename: string += '; filename="%s"' % filename - output.append(string) + output.append(string.encode()) if content_type: - output.append('Content-Type: %s' % content_type) - + output.append(('Content-Type: %s' % content_type).encode()) return b'\r\n'.join(output) def build_multipart_footer(self): - return b'--%s--\r\n' % self.boundary + return self.boundary.encode() + b'--%s--\r\n' From 7f323a3682ddede83a7bf53cdc8d24bcc24d096c Mon Sep 17 00:00:00 2001 From: Jonathan Piron Date: Mon, 13 Jun 2016 16:30:33 +0200 Subject: [PATCH 2/2] Use requests-toolbelt to stream uploads The StreamingHTTPAdapter was not working --- requirements/requirements-development.txt | 3 +- rest_framework_proxy/adapters.py | 63 ----------------------- rest_framework_proxy/utils.py | 63 ----------------------- rest_framework_proxy/views.py | 53 ++++++------------- setup.py | 3 +- 5 files changed, 21 insertions(+), 164 deletions(-) delete mode 100644 rest_framework_proxy/adapters.py delete mode 100644 rest_framework_proxy/utils.py diff --git a/requirements/requirements-development.txt b/requirements/requirements-development.txt index d942e0a..2a19408 100644 --- a/requirements/requirements-development.txt +++ b/requirements/requirements-development.txt @@ -1,3 +1,4 @@ django>=1.7 djangorestframework>=3.1.0 -requests>=1.1.0 \ No newline at end of file +requests>=1.1.0 +requests-toolbelt>=0.6.2 diff --git a/rest_framework_proxy/adapters.py b/rest_framework_proxy/adapters.py deleted file mode 100644 index d1e39e5..0000000 --- a/rest_framework_proxy/adapters.py +++ /dev/null @@ -1,63 +0,0 @@ -import socket - -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.response import HTTPResponse -from requests.packages.urllib3.exceptions import MaxRetryError -from requests.packages.urllib3.exceptions import TimeoutError -from requests.packages.urllib3.exceptions import SSLError as _SSLError -from requests.packages.urllib3.exceptions import HTTPError as _HTTPError -from requests.exceptions import ConnectionError, Timeout, SSLError - - -class StreamingHTTPAdapter(HTTPAdapter): - def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): - """Stream PreparedRequest object. Returns Response object.""" - - conn = self.get_connection(request.url, proxies) - - self.cert_verify(conn, request.url, verify, cert) - url = self.request_url(request, proxies) - - try: - if hasattr(conn, 'proxy_pool'): - conn = conn.proxy_pool - - low_conn = conn._get_conn(timeout=timeout) - low_conn.putrequest(request.method, url, skip_accept_encoding=True) - - for header, value in request.headers.items(): - low_conn.putheader(header, value) - - low_conn.endheaders() - - for i in request.body: - low_conn.send(i) - - r = low_conn.getresponse() - resp = HTTPResponse.from_httplib(r, - pool=conn, - connection=low_conn, - preload_content=False, - decode_content=False - ) - - except socket.error as sockerr: - raise ConnectionError(sockerr) - - except MaxRetryError as e: - raise ConnectionError(e) - - except (_SSLError, _HTTPError) as e: - if isinstance(e, _SSLError): - raise SSLError(e) - elif isinstance(e, TimeoutError): - raise Timeout(e) - else: - raise Timeout('Request timed out.') - - r = self.build_response(request, resp) - - if not stream: - r.content - - return r diff --git a/rest_framework_proxy/utils.py b/rest_framework_proxy/utils.py deleted file mode 100644 index a2b7d7e..0000000 --- a/rest_framework_proxy/utils.py +++ /dev/null @@ -1,63 +0,0 @@ -import mimetypes - -from requests.packages.urllib3.filepost import choose_boundary - - -def generate_boundary(): - return choose_boundary() - -class StreamingMultipart(object): - def __init__(self, data, files, boundary, chunk_size = 1024): - self.data = data - self.files = files - self.boundary = boundary - self.itering_files = False - self.chunk_size = chunk_size - - def __len__(self): - # TODO Optimize as currently we are iterating data and files twice - # Possible solution: Cache body into file and stream from it - size = 0 - for i in self.__iter__(): - size += len(i) - return size - - def __iter__(self): - return self.generator() - - def generator(self): - for (k, v) in self.data.items(): - yield self.build_multipart_header(k) + b'\r\n\r\n' - yield '{}\r\n'.format(v).encode() - - for (k, v) in self.files.items(): - content_type = mimetypes.guess_type(v.name)[0] or 'application/octet-stream' - yield self.build_multipart_header(k, v.name, content_type) + b'%s\r\n\r\n' - - # Seek back to start as __len__ has already read the file - v.seek(0) - - # Read file chunk by chunk - while True: - data = v.read(self.chunk_size) - if not data: - break - yield data - yield b'\r\n' - yield self.build_multipart_footer() - - def build_multipart_header(self, name, filename=None, content_type=None): - output = [] - output.append(('--%s' % self.boundary).encode()) - - string = 'Content-Disposition: form-data; name="%s"' % name - if filename: - string += '; filename="%s"' % filename - output.append(string.encode()) - - if content_type: - output.append(('Content-Type: %s' % content_type).encode()) - return b'\r\n'.join(output) - - def build_multipart_footer(self): - return self.boundary.encode() + b'--%s--\r\n' diff --git a/rest_framework_proxy/views.py b/rest_framework_proxy/views.py index 0fec330..92b4bb9 100644 --- a/rest_framework_proxy/views.py +++ b/rest_framework_proxy/views.py @@ -5,6 +5,7 @@ from django.utils import six from django.utils.six import BytesIO as StringIO from requests.exceptions import ConnectionError, SSLError, Timeout +from requests_toolbelt.multipart.encoder import MultipartEncoder from requests import sessions from django.http import HttpResponse from rest_framework.response import Response @@ -13,8 +14,6 @@ from rest_framework.exceptions import UnsupportedMediaType from rest_framework_proxy.settings import api_proxy_settings -from rest_framework_proxy.adapters import StreamingHTTPAdapter -from rest_framework_proxy.utils import StreamingMultipart, generate_boundary class BaseProxyView(APIView): @@ -158,40 +157,22 @@ def proxy(self, request, *args, **kwargs): cookies = self.get_cookies(request) try: - if files: - """ - By default requests library uses chunked upload for files - but it is much more easier for servers to handle streamed - uploads. - - This new implementation is also lightweight as files are not - read entirely into memory. - """ - boundary = generate_boundary() - headers['Content-Type'] = 'multipart/form-data; boundary=%s' % boundary - - body = StreamingMultipart(data, files, boundary) - - session = sessions.Session() - session.mount('http://', StreamingHTTPAdapter()) - session.mount('https://', StreamingHTTPAdapter()) - - response = session.request(request.method, url, - params=params, - data=body, - headers=headers, - timeout=self.proxy_settings.TIMEOUT, - verify=verify_ssl, - cookies=cookies) - else: - response = requests.request(request.method, url, - params=params, - data=data, - files=files, - headers=headers, - timeout=self.proxy_settings.TIMEOUT, - verify=verify_ssl, - cookies=cookies) + if not type(data) is str and bool(data): + # django rest framework request data attribute contains file and non file inputs + # need to process file differently to add their names + data={k: v for k, v in data.items() if k not in files.keys()} + for k, v in files.items(): + data[k] = (v.name, v) + data = MultipartEncoder(fields=data) + headers['Content-Type'] = data.content_type + + response = requests.request(request.method, url, + params=params, + data=data, + headers=headers, + timeout=self.proxy_settings.TIMEOUT, + verify=verify_ssl, + cookies=cookies) except (ConnectionError, SSLError): status = requests.status_codes.codes.bad_gateway return self.create_error_response({ diff --git a/setup.py b/setup.py index 0d77e3d..5b1de74 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ install_requires = [ 'django>=1.7', 'djangorestframework>=3.1.0', - 'requests>=1.1.0' + 'requests>=1.1.0', + 'requests-toolbelt>=0.6.2' ] classifiers = [ 'Environment :: Web Environment',