From d37beaf6059e82cc82a2f06018e666922602e6a8 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Mon, 16 Jan 2023 17:27:58 +0100 Subject: [PATCH 01/48] Enhancement #2124: added the samesite parameter to unset_cookie. Tests: added a test case to test_unset_cookies and to on_get in CookieUnset --- falcon/response.py | 7 +++++-- tests/test_cookies.py | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/falcon/response.py b/falcon/response.py index a74d49255..8753aef59 100644 --- a/falcon/response.py +++ b/falcon/response.py @@ -526,7 +526,7 @@ def set_cookie( self._cookies[name]['samesite'] = same_site.capitalize() - def unset_cookie(self, name, domain=None, path=None): + def unset_cookie(self, name, samesite='Lax', domain=None, path=None): """Unset a cookie in the response. Clears the contents of the cookie, and instructs the user @@ -548,6 +548,9 @@ def unset_cookie(self, name, domain=None, path=None): name (str): Cookie name Keyword Args: + samesite (str): Allows to override the default 'Lax' same_site + setting for the unset cookie. + domain (str): Restricts the cookie to a specific domain and any subdomains of that domain. By default, the user agent will return the cookie only to the origin server. @@ -591,7 +594,7 @@ def unset_cookie(self, name, domain=None, path=None): # NOTE(CaselIT): Set SameSite to Lax to avoid setting invalid cookies. # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite#Fixing_common_warnings # noqa: E501 - self._cookies[name]['samesite'] = 'Lax' + self._cookies[name]['samesite'] = samesite if domain: self._cookies[name]['domain'] = domain diff --git a/tests/test_cookies.py b/tests/test_cookies.py index 0312d5aca..aaf991a08 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime, timedelta, tzinfo from http import cookies as http_cookies import re @@ -75,6 +76,7 @@ def on_get(self, req, resp): resp.unset_cookie('bar', path='/bar') resp.unset_cookie('baz', domain='www.example.com') resp.unset_cookie('foobar', path='/foo', domain='www.example.com') + resp.unset_cookie('barfoo', samesite='none', path='/foo', domain='www.example.com') @pytest.fixture @@ -169,20 +171,20 @@ def test_response_complex_case(client): def test_unset_cookies(client): result = client.simulate_get('/unset-cookie') + assert len(result.cookies) == 5 - assert len(result.cookies) == 4 - - def test(cookie, path, domain): + def test(cookie, path, domain, samesite=None): assert cookie.value == '' # An unset cookie has an empty value assert cookie.domain == domain assert cookie.path == path - assert cookie.same_site == 'Lax' + assert cookie.same_site == samesite or 'Lax' assert cookie.expires < datetime.utcnow() test(result.cookies['foo'], path=None, domain=None) test(result.cookies['bar'], path='/bar', domain=None) test(result.cookies['baz'], path=None, domain='www.example.com') test(result.cookies['foobar'], path='/foo', domain='www.example.com') + test(result.cookies['barfoo'], path='/foo', domain='www.example.com', samesite='none') def test_cookie_expires_naive(client): From f89c31c5d63a6dd2ce0a6bc86eea009a10a2dd21 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Mon, 16 Jan 2023 17:44:22 +0100 Subject: [PATCH 02/48] Added new option to documentation in cookies.srt --- docs/api/cookies.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/api/cookies.rst b/docs/api/cookies.rst index 3b8d3ef13..93e5d5665 100644 --- a/docs/api/cookies.rst +++ b/docs/api/cookies.rst @@ -164,3 +164,7 @@ default, although this may change in a future release. .. _RFC 6265, Section 4.1.2.5: https://tools.ietf.org/html/rfc6265#section-4.1.2.5 + +When unsetting a cookie, :py:meth:`~falcon.Response.unset_cookie`, +the default `SameSite` setting of the unset cookie is ``'Lax'``, but can be changed +by setting the 'samsite' kwarg. From 32daa63049a6a6a6b5886323228a9c439c50ee7a Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Tue, 17 Jan 2023 17:32:08 +0100 Subject: [PATCH 03/48] Added tests, CookiesUnsetSameSite class and test_unset_cookies_samesite; added towncrier news fragment. --- docs/_newsfragments/2124.newandimproved.rst | 1 + docs/api/cookies.rst | 2 +- docs/changes/4.0.0.rst | 47 +++++++++++++- falcon/response.py | 6 +- tests/test_cookies.py | 70 +++++++++++++++++++-- 5 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 docs/_newsfragments/2124.newandimproved.rst diff --git a/docs/_newsfragments/2124.newandimproved.rst b/docs/_newsfragments/2124.newandimproved.rst new file mode 100644 index 000000000..dae847273 --- /dev/null +++ b/docs/_newsfragments/2124.newandimproved.rst @@ -0,0 +1 @@ +Added kwarg samesite to :py:meth:`~falcon.Response.unset_cookie` to allow override of default ``Lax`` setting of `SameSite` on the unset cookie \ No newline at end of file diff --git a/docs/api/cookies.rst b/docs/api/cookies.rst index 93e5d5665..c54be24ea 100644 --- a/docs/api/cookies.rst +++ b/docs/api/cookies.rst @@ -167,4 +167,4 @@ default, although this may change in a future release. When unsetting a cookie, :py:meth:`~falcon.Response.unset_cookie`, the default `SameSite` setting of the unset cookie is ``'Lax'``, but can be changed -by setting the 'samsite' kwarg. +by setting the 'samesite' kwarg. diff --git a/docs/changes/4.0.0.rst b/docs/changes/4.0.0.rst index b5b7c107b..77df2af1e 100644 --- a/docs/changes/4.0.0.rst +++ b/docs/changes/4.0.0.rst @@ -15,9 +15,54 @@ Changes to Supported Platforms - CPython 3.11 is now fully supported. (`#2072 `__) - End-of-life Python 3.5 & 3.6 are no longer supported. (`#2074 `__) - .. towncrier release notes start + +Breaking Changes +---------------- + +- The deprecated ``has_representation()`` method for :class:`~falcon.HTTPError` was + removed, along with the ``NoRepresentation`` and ``OptionalRepresentation`` + classes. (`#1853 `__) +- The deprecated ``api_helpers`` was removed in favor of the ``app_helpers`` + module. In addition, the deprecated ``body`` + attributes for the :class:`~falcon.HTTPResponse`, + :class:`~falcon.asgi.HTTPResponse`, and :class:`~falcon.HTTPStatus` classes. (`#2090 `__) + + +New & Improved +-------------- + +- Add link-extension to :meth:`falcon.Response.append_link` as specified in + `RFC 8288 Sec. 3.4.2 `__. (`#228 `__) +- A new ``path`` :class:`converter <~falcon.routing.PathConverter>` + capable of matching segments that include ``/`` was added. (`#648 `__) +- :class:`FloatConverter`:Modified existing IntConverter class and added FloatConverter class to convert string to float at runtime. (`#2022 `__) +- An informative representation was added to :class:`testing.Result ` + for easier development and interpretation of failed tests. The form of ``__repr__`` is as follows: + ``Result<{status_code} {content-type header} {content}>``, where the content part will reflect + up to 40 bytes of the result's content. (`#2044 `__) +- A new method :meth:`falcon.Request.get_header_as_int` was implemented. (`#2060 `__) +- A new property, :attr:`~falcon.Request.headers_lower`, was added to provide a + unified, self-documenting way to get a copy of all request headers with + lowercase names to facilitate case-insensitive matching. This is especially + useful for middleware components that need to be compatible with both WSGI and + ASGI. :attr:`~falcon.Request.headers_lower` was added in lieu of introducing a + breaking change to the WSGI :attr:`~falcon.Request.headers` property that + returns uppercase header names from the WSGI ``environ`` dictionary. (`#2063 `__) +- A new ``status_code`` attribute was added to the :attr:`falcon.Response `, + :attr:`falcon.asgi.Response `, + :attr:`HTTPStatus `, + and :attr:`HTTPError ` classes. (`#2108 `__) + + +Fixed +----- + +- The web servers used for tests are now run through :any:`sys.executable` in + order to ensure that they respect the virtualenv in which tests are being run. (`#2047 `__) + + Contributors to this Release ---------------------------- diff --git a/falcon/response.py b/falcon/response.py index 8753aef59..37941991f 100644 --- a/falcon/response.py +++ b/falcon/response.py @@ -548,9 +548,9 @@ def unset_cookie(self, name, samesite='Lax', domain=None, path=None): name (str): Cookie name Keyword Args: - samesite (str): Allows to override the default 'Lax' same_site - setting for the unset cookie. - + samesite (str): Allows to override the default 'Lax' same_site + setting for the unset cookie. + domain (str): Restricts the cookie to a specific domain and any subdomains of that domain. By default, the user agent will return the cookie only to the origin server. diff --git a/tests/test_cookies.py b/tests/test_cookies.py index aaf991a08..b3aed7cf1 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -59,6 +59,7 @@ def on_get(self, req, resp): class CookieResourceSameSite: def on_get(self, req, resp): resp.set_cookie('foo', 'bar', same_site='Lax') + resp.set_cookie('barz', 'barz', same_site='') def on_post(self, req, resp): resp.set_cookie('bar', 'foo', same_site='STRICT') @@ -76,7 +77,21 @@ def on_get(self, req, resp): resp.unset_cookie('bar', path='/bar') resp.unset_cookie('baz', domain='www.example.com') resp.unset_cookie('foobar', path='/foo', domain='www.example.com') - resp.unset_cookie('barfoo', samesite='none', path='/foo', domain='www.example.com') + resp.unset_cookie( + 'barfoo', samesite='none', path='/foo', domain='www.example.com' + ) + + +class CookieUnsetSameSite: + def on_get(self, req, resp): + # change lax to strict + resp.unset_cookie('foo', samesite='Strict') + # change strict to lax + resp.unset_cookie('bar') + # change none to '' + resp.unset_cookie('baz', samesite='') + # change '' to none + resp.unset_cookie('barz', samesite='None') @pytest.fixture @@ -86,6 +101,7 @@ def client(asgi): app.add_route('/test-convert', CookieResourceMaxAgeFloatString()) app.add_route('/same-site', CookieResourceSameSite()) app.add_route('/unset-cookie', CookieUnset()) + app.add_route('/unset-cookie-same-site', CookieUnsetSameSite()) return testing.TestClient(app) @@ -173,18 +189,64 @@ def test_unset_cookies(client): result = client.simulate_get('/unset-cookie') assert len(result.cookies) == 5 - def test(cookie, path, domain, samesite=None): + def test(cookie, path, domain, samesite='Lax'): assert cookie.value == '' # An unset cookie has an empty value assert cookie.domain == domain assert cookie.path == path - assert cookie.same_site == samesite or 'Lax' + assert cookie.same_site == samesite assert cookie.expires < datetime.utcnow() test(result.cookies['foo'], path=None, domain=None) test(result.cookies['bar'], path='/bar', domain=None) test(result.cookies['baz'], path=None, domain='www.example.com') test(result.cookies['foobar'], path='/foo', domain='www.example.com') - test(result.cookies['barfoo'], path='/foo', domain='www.example.com', samesite='none') + test( + result.cookies['barfoo'], path='/foo', domain='www.example.com', samesite='none' + ) + + +def test_unset_cookies_samesite(client): + # Test possible different samesite values in set_cookies + # foo, bar, lax + result_set_lax_empty = client.simulate_get('/same-site') + # bar, foo, strict + result_set_strict = client.simulate_post('/same-site') + # baz, foo, none + result_set_none = client.simulate_put('/same-site') + + def test_set(cookie, value, samesite=None): + assert cookie.value == value + assert cookie.same_site == samesite + + test_set(result_set_lax_empty.cookies['foo'], 'bar', samesite='Lax') + test_set(result_set_strict.cookies['bar'], 'foo', samesite='Strict') + test_set(result_set_none.cookies['baz'], 'foo', samesite='None') + # barz gets set with '', that is None value + test_set(result_set_lax_empty.cookies['barz'], 'barz') + test_set(result_set_lax_empty.cookies['barz'], 'barz', samesite=None) + + # Unset the cookies with different samesite values + result_unset = client.simulate_get('/unset-cookie-same-site') + assert len(result_unset.cookies) == 4 + + def test_unset(cookie, samesite='Lax'): + assert cookie.value == '' # An unset cookie has an empty value + assert cookie.same_site == samesite + assert cookie.expires < datetime.utcnow() + + test_unset(result_unset.cookies['foo'], samesite='Strict') + # default: bar is unset with no samesite param, so should go to Lax + test_unset(result_unset.cookies['bar'], samesite='Lax') + test_unset(result_unset.cookies['bar']) # default in test_unset + + test_unset( + result_unset.cookies['baz'], samesite=None + ) # baz gets unset to samesite = '' + test_unset(result_unset.cookies['barz'], samesite='None') + # test for false + assert result_unset.cookies['baz'].same_site != 'Strict' + assert result_unset.cookies['foo'].same_site != 'Lax' + assert not result_unset.cookies['baz'].same_site def test_cookie_expires_naive(client): From 6b9b198ebfe8a5580dc0733eedff255d9130ea56 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Tue, 17 Jan 2023 20:29:01 +0100 Subject: [PATCH 04/48] Removed unused import logging from test_cookies --- tests/test_cookies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cookies.py b/tests/test_cookies.py index b3aed7cf1..be8188db1 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -1,4 +1,4 @@ -import logging + from datetime import datetime, timedelta, tzinfo from http import cookies as http_cookies import re From 0a17b7942c2e4b05553dcabe7f5804c15d579615 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Wed, 18 Jan 2023 17:09:07 +0100 Subject: [PATCH 05/48] Reverted changes to docs/changes4.0.0.rst --- docs/changes/4.0.0.rst | 46 ------------------------------------------ 1 file changed, 46 deletions(-) diff --git a/docs/changes/4.0.0.rst b/docs/changes/4.0.0.rst index 77df2af1e..ef7912e47 100644 --- a/docs/changes/4.0.0.rst +++ b/docs/changes/4.0.0.rst @@ -17,52 +17,6 @@ Changes to Supported Platforms .. towncrier release notes start - -Breaking Changes ----------------- - -- The deprecated ``has_representation()`` method for :class:`~falcon.HTTPError` was - removed, along with the ``NoRepresentation`` and ``OptionalRepresentation`` - classes. (`#1853 `__) -- The deprecated ``api_helpers`` was removed in favor of the ``app_helpers`` - module. In addition, the deprecated ``body`` - attributes for the :class:`~falcon.HTTPResponse`, - :class:`~falcon.asgi.HTTPResponse`, and :class:`~falcon.HTTPStatus` classes. (`#2090 `__) - - -New & Improved --------------- - -- Add link-extension to :meth:`falcon.Response.append_link` as specified in - `RFC 8288 Sec. 3.4.2 `__. (`#228 `__) -- A new ``path`` :class:`converter <~falcon.routing.PathConverter>` - capable of matching segments that include ``/`` was added. (`#648 `__) -- :class:`FloatConverter`:Modified existing IntConverter class and added FloatConverter class to convert string to float at runtime. (`#2022 `__) -- An informative representation was added to :class:`testing.Result ` - for easier development and interpretation of failed tests. The form of ``__repr__`` is as follows: - ``Result<{status_code} {content-type header} {content}>``, where the content part will reflect - up to 40 bytes of the result's content. (`#2044 `__) -- A new method :meth:`falcon.Request.get_header_as_int` was implemented. (`#2060 `__) -- A new property, :attr:`~falcon.Request.headers_lower`, was added to provide a - unified, self-documenting way to get a copy of all request headers with - lowercase names to facilitate case-insensitive matching. This is especially - useful for middleware components that need to be compatible with both WSGI and - ASGI. :attr:`~falcon.Request.headers_lower` was added in lieu of introducing a - breaking change to the WSGI :attr:`~falcon.Request.headers` property that - returns uppercase header names from the WSGI ``environ`` dictionary. (`#2063 `__) -- A new ``status_code`` attribute was added to the :attr:`falcon.Response `, - :attr:`falcon.asgi.Response `, - :attr:`HTTPStatus `, - and :attr:`HTTPError ` classes. (`#2108 `__) - - -Fixed ------ - -- The web servers used for tests are now run through :any:`sys.executable` in - order to ensure that they respect the virtualenv in which tests are being run. (`#2047 `__) - - Contributors to this Release ---------------------------- From 187aa4fe0112020b035ad85a668319865580776d Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Wed, 18 Jan 2023 17:09:28 +0100 Subject: [PATCH 06/48] Reverted changes to docs/changes4.0.0.rst --- tests/test_cookies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cookies.py b/tests/test_cookies.py index be8188db1..72759f0e5 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -1,4 +1,3 @@ - from datetime import datetime, timedelta, tzinfo from http import cookies as http_cookies import re From 9d2fe9e5fcc693d834c6ad067b6d3fa34d345434 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Thu, 9 Feb 2023 12:23:23 +0100 Subject: [PATCH 07/48] Added 'files' parameter to _simulate_request; Added tests for file upload (text, urlencoded, image, nested) and some errors. --- falcon/testing/client.py | 107 ++++- tests/files/falcon.png | Bin 0 -> 3613 bytes tests/files/loremipsum.txt | 11 + tests/test_media_multipart.py | 1 - tests/test_multipart_formdata_request.py | 560 +++++++++++++++++++++++ 5 files changed, 676 insertions(+), 3 deletions(-) create mode 100644 tests/files/falcon.png create mode 100644 tests/files/loremipsum.txt create mode 100644 tests/test_multipart_formdata_request.py diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 3bc6f93cd..c97506583 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -18,16 +18,19 @@ WSGI callable, without having to stand up a WSGI server. """ +import os import asyncio import datetime as dt import inspect import json as json_module import time +import json from typing import Dict from typing import Optional from typing import Sequence from typing import Union import warnings +from urllib3.filepost import encode_multipart_formdata, RequestField import wsgiref.validate from falcon.asgi_spec import ScopeType @@ -437,6 +440,7 @@ def simulate_request( content_type=None, body=None, json=None, + files=None, file_wrapper=None, wsgierrors=None, params=None, @@ -575,6 +579,7 @@ def simulate_request( content_type=content_type, body=body, json=json, + files=files, params=params, params_csv=params_csv, protocol=protocol, @@ -598,6 +603,7 @@ def simulate_request( headers, body, json, + files, extras, ) @@ -651,6 +657,7 @@ async def _simulate_request_asgi( content_type=None, body=None, json=None, + files=None, params=None, params_csv=True, protocol='http', @@ -774,6 +781,7 @@ async def _simulate_request_asgi( headers, body, json, + files, extras, ) @@ -2133,8 +2141,98 @@ async def __aexit__(self, exc_type, exc, tb): await self._task_req +def _encode_files(files, data=None): + """Build the body for a multipart/form-data request. + Will successfully encode files when passed as a dict or a list of + tuples. Order is retained if data is a list of tuples but arbitrary + if parameters are supplied as a dict. + The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) + or 4-tuples (filename, fileobj, contentype, custom_headers). + """ + fields = [] + if data and not isinstance(data, (list, dict)): + raise ValueError('Data must not be a list of tuples or dict.') + elif data and isinstance(data, dict): + fields = list(data.items()) + elif data: + fields = list(data) + + if not isinstance(files, (dict, list)): + raise ValueError('cannot encode objects that are not 2-tuples') + elif isinstance(files, dict): + files = list(files.items()) + + new_fields = [] + + # Append data to the other multipart parts + for field, val in fields: + if isinstance(val, str) or not hasattr(val, '__iter__'): + val = [val] + for v in val: + if v is not None: + # Don't call str() on bytestrings: in Py3 it all goes wrong. + if not isinstance(v, bytes): + v = str(v) + + new_fields.append( + ( + field.decode('utf-8') if isinstance(field, bytes) else field, + v.encode('utf-8') if isinstance(v, str) else v, + ) + ) + + for (k, v) in files: + # support for explicit filename + file_content_type = None + file_header = None + if isinstance(v, (tuple, list)): + if len(v) == 2: + file_name, file_data = v + elif len(v) == 3: + file_name, file_data, file_content_type = v + else: + file_name, file_data, file_content_type, file_header = v + if len(v) >= 3: + if file_content_type == 'multipart/mixed': + file_data, file_content_type = _encode_files( + json.loads(file_data.decode()) + ) + else: + # if v is not a tuple or iterable it has to be a filelike obj + name = getattr(v, 'name', None) + if name and isinstance(name, str) and name[0] != '<' and name[-1] != '>': + file_name = os.path.basename(name) + else: + file_name = k + file_data = v + + if file_data is None: + continue + elif hasattr(file_data, 'read'): + fdata = file_data.read() + else: + fdata = file_data + + rf = RequestField(name=k, filename=file_name, data=fdata, headers=file_header) + rf.make_multipart(content_type=file_content_type) + new_fields.append(rf) + + body, content_type = encode_multipart_formdata(new_fields) + + return body, content_type + + def _prepare_sim_args( - path, query_string, params, params_csv, content_type, headers, body, json, extras + path, + query_string, + params, + params_csv, + content_type, + headers, + body, + json, + files, + extras, ): if not path.startswith('/'): raise ValueError("path must start with '/'") @@ -2163,7 +2261,12 @@ def _prepare_sim_args( headers = headers or {} headers['Content-Type'] = content_type - if json is not None: + if files is not None: + body, content_type = _encode_files(files, json) + headers = headers or {} + headers['Content-Type'] = content_type + + elif json is not None: body = json_module.dumps(json, ensure_ascii=False) headers = headers or {} headers['Content-Type'] = MEDIA_JSON diff --git a/tests/files/falcon.png b/tests/files/falcon.png new file mode 100644 index 0000000000000000000000000000000000000000..69246223b41927dd1708b12bb6a8e9c3f35c070b GIT binary patch literal 3613 zcmV+&4&w2NP)ec=8=KAK%^SyQQxohyR zQS`-#`QgL&)~)#5xb@7J{O{iJwPEtRaQNA>`{~p1u~+)!$o0vP_R*pA!h!eHr}W2* z^~#j*q%!u5mDO(?e_ow#9>nr10*33 zX2$#Jz0(KGB}rAKw#;TUwrY!7ymOjx(6^Y96!)Kgkce{Xry}#gCfR2E!~3{ z5(^4=t9uYbBAr2ldb&q3AiNbAu-83`0pT3MFwlQk&e-#mAgo;7;}{NmY#0VU>mEiM ztX@*^MpA+XTBsV$5LnN9JiI{}K{zTK%>an0FaPj~W(19CD?=aMq@Q&FjL9$3B_bQ? zhj#@*gD4!3YNWCrEoC3xqJkhKDu<*PYhs{S^4Voc&>$05_jfZD6OxAlcU7*@Dchun zR1|`&vRLDipaDt;r4kpjhdHt|p9FiTn?Yf%^L>Ouxu>6N)F2nN15*WOM~Y1(XESff zgWy!p6%j4w0OTA<9(ocq%qi;o*njLIa+P{Pf?o{L^!$kS9X|(1rL^IqGKI$U&W6$P zwTECXr44;BDzC6rkFX!26{-x6+-2VJLn~7VyD9k*?dWJ$_@xY`7HM@Crm|`}44q4j zpQN((s4+=~KM9a`eyJEd$-L=KCdkma&IvZ^fSuIK(ECd!AZgslrytXo!IZMt!gq!S z(X1_;Fu{mLPuhXx*WMS6%edPrsIX`TXO>BYQENaJ@2d7rpe$8*)d2 zxV}io^+b7xy@5FBWglL0SP}VnmE?vNhj-nf&b9gc?y^44Geb0+V=PneI`7_kmCr7B zYD%$Z{K)X08gCPb&`Sw|>{qu}jNgIdpoW_j`b3Cc_R-}OMGBX)-Z!MuckZh;t5u2d_0~|2Y`IF#h zz^a#flc5fCVC4}ucP+mC@{!E@>8>V*29*v&HvCZYnWz=0@275L2rHGs#!Z+KtrLKI z3xuY9C~e6z5zpl&knH+L7(PyiFQ`+BCvgRl0q9wQQ>;N-6kkx0J0p@3p$0LOh=N7S z{!nDRg;HA(kctUR;k5lcRuK(uTP4jvJ846-jt?is+wb&Gj*3BIiDI%*=a&94+kVPC z^nAU=I=gfLsIM}F>9sQKW7OwJgw+I)xmshx?fA%Ye)I>T_V334F_atqb(eP?eNHZp zA-G0U6{tSOzG<@C9+1pm$1p3yquh zVebX(EHh}?Kq2Q8lCjGGtzUuD0H9lA1g^_(@Bkh^4e}u3p2pCyR6X7)98t_X5w8Rs z4`8yDRZ(DUagZO&7LT-v!~qA3wDhYkRkAVg7c zS!8Vwgsh)*+v;-BuyA>=WzUIWSOv-NCOTym>h$%rY!*oRHEN@7VPTudbAm_%*&Dq}_ z!A#ex96K$g#%r)hSO@cb-lPV`*aT|_j*boePqEbkxHQ1g%Y8 z;1Lqpcp!3Q7sE6>iLaKl>R~FbPn8637*GOq8lJ?{Tci{5 zBzDxa^}$3!7K`B+>TU>Ef;aVB98>Wm4#;pKp2Pte`qNN{awXV&6#tSbIHDRGW~cz} z*N{aGwnLfSIRLbqru@|jk`%Sy5+YESB?;XTL}u8qsd1_lfM|8rF#1KN*4EbBE8LxM z)M?wqpEeu*J7i^(3MpcpJ(-`8Um%1aWFi7dzNl=KzMFCC^4K^OJOC%x!y#NQf8vK7 zVhLrscYx|UM->cV#R~IXO?C1{-$ANKE;ZS4%pcL1b~E%{(wFD#HMUH)Zpx-nedjw~ zWc?8ErSZ={J5{?;-&VYt1{lqMwyvN^Z|IBCY$^F`lLOvOu1Q%P8YAA`;o5$TivuMb z$1Ic;#xhW0sZJ=G$UqqM7poJ?Tljzij+4reVqw{LKg##&9t{g$vj6I15~tzkZ$*5= zeK?xaaD%tQOM1xFsf43Ca1yDKu0gwFiVwCgZLLnG{-O)N(iF`}C4x1<_GVXEivN!5 z+<&PbZ)H!%WHMDWE${)u4|n^dI&`~;IgqvYf+KHJj1Ry{H-S#LS^X%v3YMO}pskAp zH;UN<`8*Xe^>td*Ezkvj6=DrjiSRe#o|`oIK5K5b2hf@XKYGt0F_cxqsddtnFa~I~ z^MfO4HeglXIiVJh76s6h17HTHOsiX7lo(AH9O`Mx!-+@8k+{sXMyMv}=a*EnC@N`m zKMXl(c77Vk7Ic_&x z-acICdPBp_sPw}~8Gh_T%3OQy9v>F>I>(7_#Y(SBviNAl&=_F~5nSSX&5eboXuVx3 zvIQ;b)zNJ|N;imZ}Kx-ad*qBMew)U{P-{Zhy$&*O?dN@^CpaogFt9J z)>m#;Hkw#Aqr`nS7AoALsCNX_Zvd_E6u@p}b7 z-toL6)VShUw;vKjC4h>@7PEXIdf_sYOz&zgK zigsNGwAa<%8!!gtAT;HN&$ScJM1=p9EqS*qs8p#pg0ur%CgN zf5*tOSG7reE-5>y2v+gC3r{57;+w}b%HECA56;1wc~ze%ULlO}Uolxfm44D7v&^!d z+hiV{i`{N&vfmze;o)p1&Cm_c>veu*Qk+zR!DudZ403ijm0;^b@u`rOo9vxZ+R-o~ zuN`Q6%gSjF2$sD$&CW_!@i98*B8^DhaMKfI3OHG3S*X~&+JeAc;MXD?UCWHZ-Pu|y_31K%q192Mm$a>=H>bC z%#)v8EA70D(ZcD{*C!a0hgc5XJ;{}C-*v%{3w>>~bP8ea6au~g3AY)?-r3F1MM2MY z?n3~28*3WfiW9{LI6CPJR&2G@sLQUX#yy3@>yld%IE58Ojv6xj&NNkvURBP}j9F$1 z^mQiPUIr4xUj7Zi^ha0!c51skoWVt68!9YRO8ch8R-CRt53^ftD`KSZ8eMK2%u)+3 z^QBZs(M$hq!>TjHOsue2DQGvI#ChPiwBm}Z*3|W$d9x)_^VBIyt^oOCXH6Fj&1+H8 zr!GP5y{iL&H#?W@|He+*$SGlxtbu29IZ zmbC&Td|oaxZUuuCK;ebYk5%d}b@*stpIYKGw@C0D`Ra%RDnj(R8KN$mf>FnsQnd2T zIC2voYff?{wZKX8g3(Jf+|ru5V}-XF+6;m~@GOffV0KJ-ctk|u6o%DB z3R$GYr1Nub3p)N@Q+n8ya)x1e5dP5zV71$?@0(W#+?lTYx5G{4FI!#?M;o0lzWCyc jFTVKVi!Z+Tq5=N^DIcZRUSct<00000NkvXXu0mjfyO8~# literal 0 HcmV?d00001 diff --git a/tests/files/loremipsum.txt b/tests/files/loremipsum.txt new file mode 100644 index 000000000..f83a7a255 --- /dev/null +++ b/tests/files/loremipsum.txt @@ -0,0 +1,11 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor +incididunt ut labore et dolore magna aliqua. Dolor sed viverra ipsum nunc +aliquet bibendum enim. In massa tempor nec feugiat. Nunc aliquet bibendum enim +facilisis gravida. Nisl nunc mi ipsum faucibus vitae aliquet nec ullamcorper. +Amet luctus venenatis lectus magna fringilla. Volutpat maecenas volutpat blandit +aliquam etiam erat velit scelerisque in. Egestas egestas fringilla phasellus +faucibus scelerisque eleifend. Sagittis orci a scelerisque purus semper eget +duis. Nulla pharetra diam sit amet nisl suscipit. Sed adipiscing diam donec +adipiscing tristique risus nec feugiat in. Fusce ut placerat orci nulla. +Pharetra vel turpis nunc eget lorem dolor. Tristique senectus et netus et +malesuada. \ No newline at end of file diff --git a/tests/test_media_multipart.py b/tests/test_media_multipart.py index 7edf51f4c..941dd7226 100644 --- a/tests/test_media_multipart.py +++ b/tests/test_media_multipart.py @@ -98,7 +98,6 @@ b'--boundary--\r\n' ) - EXAMPLES = { '5b11af82ab65407ba8cdccf37d2a9c4f': EXAMPLE1, '---------------------------1574247108204320607285918568': EXAMPLE2, diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py new file mode 100644 index 000000000..21b902efb --- /dev/null +++ b/tests/test_multipart_formdata_request.py @@ -0,0 +1,560 @@ +import cgi +import json +import pytest +import base64 + +import falcon +from falcon import media +from falcon import testing + +from _util import create_app # NOQA: I100 + + +""" +Request takes tuples like (filename, data, content_type, headers) + +""" + + +class MultipartAnalyzer: + def on_post(self, req, resp): + values = [] + for part in req.media: + data = part.data.decode() + inner_form = [] + if part.content_type.startswith('multipart/mixed'): + print(f'we are in part with {part.data}') + for nested in part.media: + print(f'we are in nested with {nested.name}') + inner_form.append( + { + 'content_type': nested.content_type, + 'data': nested.data.decode(), + 'filename': nested.filename, + 'name': nested.name, + 'text': nested.text, + } + ) + data = inner_form + + values.append( + { + 'content_type': part.content_type, + 'data': data, + 'filename': part.filename, + 'name': part.name, + 'secure_filename': part.secure_filename if part.filename else None, + 'text': part.text, + } + ) + + resp.media = values + + def on_post_media(self, req, resp): + deserialized = [] + for part in req.media: + part_media = part.get_media() + assert part_media == part.media + deserialized.append(part_media) + + resp.media = deserialized + + def on_post_image(self, req, resp): + values = [] + for part in req.media: + # Save a copy of the image, encode in base64 and re-decode to compare. This way we avoid invalid characters + # and get a more 'manageable' output. + new_filename = part.filename.split('.')[0] + '_posted.png' + f = open(new_filename, 'w+b') + f.write(part.data) + f.close() + new_file64 = base64.b64encode(open(new_filename, 'rb').read()) + values.append( + { + 'content_type': part.content_type, + 'data': new_file64.decode(), + 'filename': part.filename, + 'new_filename': new_filename, + 'name': part.name, + 'secure_filename': part.secure_filename if part.filename else None, + } + ) + + resp.media = values + + +class AsyncMultipartAnalyzer: + async def on_post(self, req, resp): + values = [] + form = await req.get_media() + async for part in form: + values.append( + { + 'content_type': part.content_type, + 'data': (await part.data).decode(), + 'filename': part.filename, + 'name': part.name, + 'secure_filename': part.secure_filename if part.filename else None, + 'text': (await part.text), + } + ) + resp.media = values + + async def on_post_media(self, req, resp): + deserialized = [] + form = await req.media + async for part in form: + part_media = await part.get_media() + assert part_media == await part.media + deserialized.append(part_media) + + resp.media = deserialized + + async def on_post_image(self, req, resp): + values = [] + form = await req.get_media() + async for part in form: + new_filename = part.filename.split('.')[0] + '_posted.png' + data = await part.data + values.append( + { + 'content_type': part.content_type, + 'data': base64.b64encode(data).decode(), + 'filename': part.filename, + 'new_filename': new_filename, + 'name': part.name, + 'secure_filename': part.secure_filename if part.filename else None, + } + ) + resp.media = values + + +class MultipartFileUpload: + def on_post(self, req, resp): + upload = cgi.FieldStorage(fp=req.stream, environ=req.env) + data = upload['file'].file.read().decode('utf-8') + resp.media = dict(data=data) + + +class AsyncMultipartFileUpload: + async def on_post(self, req, resp): + deserialized = [] + form = await req.media + async for part in form: + upload = cgi.FieldStorage(fp=part.stream, environ=req.scope) + print(f'upload is {upload.fp.read()}') + data = await upload['file'].file.read().decode('utf-8') + + print(f'the dict is {dict(data=data)} and the dump is {dict(data=data)}') + deserialized.append(dict(data=data)) + resp.media = deserialized + + +@pytest.fixture +def client(asgi): + app = create_app(asgi) + parser = media.MultipartFormHandler() + parser.parse_options.media_handlers[ + 'multipart/mixed' + ] = media.MultipartFormHandler() + app.req_options.media_handlers = media.Handlers( + { + falcon.MEDIA_JSON: media.JSONHandler(), + falcon.MEDIA_MULTIPART: parser, # media.MultipartFormHandler() + } + ) + + app.req_options.default_media_type = falcon.MEDIA_MULTIPART + resource = AsyncMultipartAnalyzer() if asgi else MultipartAnalyzer() + resourcecgi = AsyncMultipartFileUpload() if asgi else MultipartFileUpload() + app.add_route('/submit', resource) + app.add_route('/uploadcgi', resourcecgi) + app.add_route('/media', resource, suffix='media') + app.add_route('/image', resource, suffix='image') + + return testing.TestClient(app) + + +# ----- TESTING THE files PARAMETER IN simulate_request FOR DIFFERENT DATA TYPES +payload1 = b'{"debug": true, "message": "Hello, world!", "score": 7}' + +FILES1 = { + 'fileobj': 'just some stuff', + 'hello': (None, 'world'), + 'document': (None, payload1, 'application/json'), + 'file1': ('test.txt', 'Hello, world!', 'text/plain'), +} +FILES1_TUPLES = [ + ('fileobj', 'just some stuff'), + ('hello', (None, 'world')), + ('document', (None, payload1, 'application/json')), + ('file1', ('test.txt', 'Hello, world!', 'text/plain')), +] + +FILES1_RESP = [ + { + 'content_type': 'text/plain', + 'data': 'just some stuff', + 'filename': 'fileobj', + 'name': 'fileobj', + 'secure_filename': 'fileobj', + 'text': 'just some stuff', + }, + { + 'content_type': 'text/plain', + 'data': 'world', + 'filename': None, + 'name': 'hello', + 'secure_filename': None, + 'text': 'world', + }, + { + 'content_type': 'application/json', + 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', + 'filename': None, + 'name': 'document', + 'secure_filename': None, + 'text': None, + }, + { + 'content_type': 'text/plain', + 'data': 'Hello, world!', + 'filename': 'test.txt', + 'name': 'file1', + 'secure_filename': 'test.txt', + 'text': 'Hello, world!', + }, +] + + +def test_upload_multipart_dict(client): + resp = client.simulate_post('/submit', files=FILES1) + + assert resp.status_code == 200 + assert resp.json == FILES1_RESP + + +def test_upload_multipart_list(client): + resp = client.simulate_post('/submit', files=FILES1_TUPLES) + + assert resp.status_code == 200 + assert resp.json == FILES1_RESP + + +FILES3 = { + 'bytes': ('bytes', b'123456789abcdef\n' * 64 * 1024 * 2, 'application/x-falcon'), + 'empty': (None, '', 'text/plain'), +} + + +def test_body_too_large(client): + resp = client.simulate_post('/submit', files=FILES3) + assert resp.status_code == 400 + assert resp.json == { + 'description': 'body part is too large', + 'title': 'Malformed multipart/form-data request media', + } + + +FILES5 = { + 'factorials': ( + None, + '{"count": 6, "numbers": [1, 2, 6, 24, 120, 720]}', + 'application/json', + ), + 'person': ( + None, + 'name=Jane&surname=Doe&fruit=%F0%9F%8D%8F', + 'application/x-www-form-urlencoded', + ), +} + + +def test_upload_multipart_media(client): + resp = client.simulate_post('/media', files=FILES5) + + assert resp.status_code == 200 + assert resp.json == [ + {'count': 6, 'numbers': [1, 2, 6, 24, 120, 720]}, + { + 'fruit': b'\xF0\x9F\x8D\x8F'.decode('utf8'), # u"\U0001F34F", + 'name': 'Jane', + 'surname': 'Doe', + }, + ] + + +def asserts_data_types(resp): + assert resp.status_code == 200 + expected_list = [ + { + 'content_type': 'text/plain', + 'data': 'just some stuff', + 'filename': 'fileobj', + 'name': 'fileobj', + 'secure_filename': 'fileobj', + 'text': 'just some stuff', + }, + { + 'content_type': 'text/plain', + 'data': '5', + 'filename': None, + 'name': 'data1', + 'secure_filename': None, + 'text': '5', + }, + { + 'content_type': 'text/plain', + 'data': 'hello', + 'filename': None, + 'name': 'data2', + 'secure_filename': None, + 'text': 'hello', + }, + { + 'content_type': 'text/plain', + 'data': 'bonjour', + 'filename': None, + 'name': 'data2', + 'secure_filename': None, + 'text': 'bonjour', + }, + { + 'content_type': 'text/plain', + 'data': 'world', + 'filename': None, + 'name': 'hello', + 'secure_filename': None, + 'text': 'world', + }, + { + 'content_type': 'application/json', + 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', + 'filename': None, + 'name': 'document', + 'secure_filename': None, + 'text': None, + }, + { + 'content_type': 'text/plain', + 'data': 'Hello, world!', + 'filename': 'test.txt', + 'name': 'file1', + 'secure_filename': 'test.txt', + 'text': 'Hello, world!', + }, + ] + # Result will be unordered, because both fileobj and data are present. When all files are tuples, response will be + # unordered if json contains dictionaries - then resp.json == expected_list can be used. + assert len(resp.json) == len(expected_list) + assert all(map(lambda el: el in expected_list, resp.json)) + + +# -------Tests for multipart with files param and json data, where data comes in different types---------- +def test_upload_multipart_datalist(client): + resp = client.simulate_post( + '/submit', files=FILES1, json=[('data1', 5), ('data2', ['hello', 'bonjour'])] + ) + asserts_data_types(resp) + + +def test_upload_multipart_datalisttuple(client): + resp = client.simulate_post( + '/submit', files=FILES1, json=[('data1', 5), ('data2', ('hello', 'bonjour'))] + ) + asserts_data_types(resp) + + +def test_upload_multipart_datalistdict(client): + """json data list with dict""" + resp = client.simulate_post( + '/submit', files=FILES1, json=[('data1', 5), ('data2', {'hello', 'bonjour'})] + ) + asserts_data_types(resp) + + +def test_upload_multipart_datadict(client): + """json data dict with list""" + resp = client.simulate_post( + '/submit', files=FILES1, json={'data1': 5, 'data2': ['hello', 'bonjour']} + ) + asserts_data_types(resp) + + +def test_upload_multipart_datadicttuple(client): + """json data dict with tuple""" + resp = client.simulate_post( + '/submit', files=FILES1, json={'data1': 5, 'data2': ('hello', 'bonjour')} + ) + asserts_data_types(resp) + + +def test_upload_multipart_datadictdict(client): + """json data dict with dict""" + resp = client.simulate_post( + '/submit', files=FILES1, json={'data1': 5, 'data2': {'hello', 'bonjour'}} + ) + asserts_data_types(resp) + + +def test_invalid_files(client): + """invalid file type""" + with pytest.raises(ValueError): + client.simulate_post('/submit', files='heya') + + +def test_invalid_files(client): + """empty file in files""" + with pytest.raises(ValueError): + client.simulate_post('/submit', files={'file': ()}) + + +def test_invalid_dataint(client): + """invalid data type in json, int""" + with pytest.raises(ValueError): + client.simulate_post('/submit', files=FILES1, json=5) + + +def test_invalid_datastr(client): + """invalid data type in json, str""" + with pytest.raises(ValueError): + client.simulate_post('/submit', files=FILES1, json='yo') + + +def test_invalid_databyte(client): + """invalid data type in json, b''""" + with pytest.raises(ValueError): + client.simulate_post('/submit', files=FILES1, json=b'yo self') + + +# ------TEST UPLOADING ACTUAL FILES: TEXT, IMAGE ---------------------------- + +LOREM_FILE = ( + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do ' + 'eiusmod tempor\n' + 'incididunt ut labore et dolore magna aliqua. Dolor sed viverra ' + 'ipsum nunc\n' + 'aliquet bibendum enim. In massa tempor nec feugiat. Nunc aliquet ' + 'bibendum enim\n' + 'facilisis gravida. Nisl nunc mi ipsum faucibus vitae aliquet nec ' + 'ullamcorper.\n' + 'Amet luctus venenatis lectus magna fringilla. Volutpat maecenas ' + 'volutpat blandit\n' + 'aliquam etiam erat velit scelerisque in. Egestas egestas fringilla ' + 'phasellus\n' + 'faucibus scelerisque eleifend. Sagittis orci a scelerisque purus ' + 'semper eget\n' + 'duis. Nulla pharetra diam sit amet nisl suscipit. Sed adipiscing ' + 'diam donec\n' + 'adipiscing tristique risus nec feugiat in. Fusce ut placerat orci ' + 'nulla.\n' + 'Pharetra vel turpis nunc eget lorem dolor. Tristique senectus et ' + 'netus et\n' + 'malesuada.' +) + + +def test_upload_file(client): + resp = client.simulate_post( + '/submit', files={'file': open('tests/files/loremipsum.txt', 'rb')} + ) + + assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': LOREM_FILE, + 'filename': 'loremipsum.txt', + 'name': 'file', + 'secure_filename': 'loremipsum.txt', + 'text': LOREM_FILE, + }, + ] + + +def test_upload_image(client): + filename = 'tests/files/falcon.png' + imagebin = open(filename, 'rb').read() + file64 = base64.b64encode(imagebin) + + resp = client.simulate_post( + '/image', files={'image': (filename, open(filename, 'rb'))} + ) + new_filename = filename.split('.')[0] + '_posted.png' + + assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': file64.decode(), + 'filename': filename, + 'new_filename': new_filename, + 'name': 'image', + 'secure_filename': filename.replace('/', '_'), + } + ] + + +FILES6 = { + 'field1': 'Joe Blow', + 'docs': ( + None, + json.dumps( + { + 'file1': ( + 'file1.txt', + 'this is file1', + None, + {'Content-Disposition': 'attachment'}, + ), + 'file2': ( + 'file2.txt', + 'Hello, World!', + None, + {'Content-Disposition': 'attachment'}, + ), + } + ).encode(), + 'multipart/mixed', + ), + 'document': (None, payload1, 'application/json'), +} + + +def test_nested_multipart_mixed(client): + resp = client.simulate_post('/submit', files=FILES6) + print(f'response in mixed is {resp.json}') + assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': 'Joe Blow', + 'filename': 'field1', + 'name': 'field1', + 'secure_filename': 'field1', + 'text': 'Joe Blow', + }, + { + 'content_type': 'multipart/mixed', + 'data': '{"file1": ["file1.txt", "this is file1"], "file2": ["file2.txt", "Hello, World!"]}', + 'filename': None, + 'name': 'docs', + 'secure_filename': None, + 'text': None, + }, + { + 'content_type': 'application/json', + 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', + 'filename': None, + 'name': 'document', + 'secure_filename': None, + 'text': None, + }, + ] + + +# def test_upload_file_cgi(client): +# resp = client.simulate_post('/uploadcgi', files={'file': open('tests/loremipsum.txt', 'rb')}) +# +# # assert resp.status_code == 200 +# assert resp.json == LOREM_FILE From 2b9d0e0365c7b045e78977ffd47de9f1f147a38a Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Thu, 9 Feb 2023 21:33:02 +0100 Subject: [PATCH 08/48] Fixed nested mixed request problems --- falcon/testing/client.py | 21 ++- tests/test_multipart_formdata_request.py | 191 +++++++++++++---------- 2 files changed, 125 insertions(+), 87 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index c97506583..bbf66bc5d 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2185,6 +2185,7 @@ def _encode_files(files, data=None): # support for explicit filename file_content_type = None file_header = None + content_disposition = None if isinstance(v, (tuple, list)): if len(v) == 2: file_name, file_data = v @@ -2192,11 +2193,14 @@ def _encode_files(files, data=None): file_name, file_data, file_content_type = v else: file_name, file_data, file_content_type, file_header = v - if len(v) >= 3: - if file_content_type == 'multipart/mixed': - file_data, file_content_type = _encode_files( - json.loads(file_data.decode()) - ) + if ( + len(v) >= 3 + and file_content_type + and file_content_type.startswith('multipart/mixed') + ): + file_data, assigned_type = _encode_files(json.loads(file_data.decode())) + file_data = file_data + file_content_type = 'multipart/mixed; ' + (assigned_type.split('; ')[1]) else: # if v is not a tuple or iterable it has to be a filelike obj name = getattr(v, 'name', None) @@ -2212,9 +2216,12 @@ def _encode_files(files, data=None): fdata = file_data.read() else: fdata = file_data - + if file_header and 'Content-Disposition' in file_header.keys(): + content_disposition = file_header['Content-Disposition'] rf = RequestField(name=k, filename=file_name, data=fdata, headers=file_header) - rf.make_multipart(content_type=file_content_type) + rf.make_multipart( + content_type=file_content_type, content_disposition=content_disposition + ) new_fields.append(rf) body, content_type = encode_multipart_formdata(new_fields) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 21b902efb..291f92f7c 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -1,4 +1,5 @@ import cgi +import io import json import pytest import base64 @@ -20,27 +21,18 @@ class MultipartAnalyzer: def on_post(self, req, resp): values = [] for part in req.media: - data = part.data.decode() + # For mixed nested file requests + part_type = part.content_type inner_form = [] if part.content_type.startswith('multipart/mixed'): - print(f'we are in part with {part.data}') for nested in part.media: - print(f'we are in nested with {nested.name}') - inner_form.append( - { - 'content_type': nested.content_type, - 'data': nested.data.decode(), - 'filename': nested.filename, - 'name': nested.name, - 'text': nested.text, - } - ) - data = inner_form - + inner_form.append({'name': nested.name, 'text': nested.text}) + part_type = 'multipart/mixed' + # ---------------------------------------------------- values.append( { - 'content_type': part.content_type, - 'data': data, + 'content_type': part_type, + 'data': inner_form or part.data.decode(), 'filename': part.filename, 'name': part.name, 'secure_filename': part.secure_filename if part.filename else None, @@ -55,8 +47,8 @@ def on_post_media(self, req, resp): for part in req.media: part_media = part.get_media() assert part_media == part.media - deserialized.append(part_media) + deserialized.append(part_media) resp.media = deserialized def on_post_image(self, req, resp): @@ -88,10 +80,19 @@ async def on_post(self, req, resp): values = [] form = await req.get_media() async for part in form: + # For mixed nested file requests + part_type = part.content_type + inner_form = [] + if part_type.startswith('multipart/mixed'): + part_form = await part.get_media() + async for nested in part_form: + inner_form.append({'name': nested.name, 'text': await nested.text}) + part_type = 'multipart/mixed' + # ---------------------------------------------------- values.append( { - 'content_type': part.content_type, - 'data': (await part.data).decode(), + 'content_type': part_type, + 'data': inner_form or (await part.data).decode(), 'filename': part.filename, 'name': part.name, 'secure_filename': part.secure_filename if part.filename else None, @@ -153,10 +154,15 @@ async def on_post(self, req, resp): @pytest.fixture def client(asgi): app = create_app(asgi) + + # For handling mixed nested requests ----------------------- parser = media.MultipartFormHandler() parser.parse_options.media_handlers[ 'multipart/mixed' ] = media.MultipartFormHandler() + + # ------------------------------------------------------------ + app.req_options.media_handlers = media.Handlers( { falcon.MEDIA_JSON: media.JSONHandler(), @@ -167,6 +173,7 @@ def client(asgi): app.req_options.default_media_type = falcon.MEDIA_MULTIPART resource = AsyncMultipartAnalyzer() if asgi else MultipartAnalyzer() resourcecgi = AsyncMultipartFileUpload() if asgi else MultipartFileUpload() + app.add_route('/submit', resource) app.add_route('/uploadcgi', resourcecgi) app.add_route('/media', resource, suffix='media') @@ -175,7 +182,10 @@ def client(asgi): return testing.TestClient(app) -# ----- TESTING THE files PARAMETER IN simulate_request FOR DIFFERENT DATA TYPES +# region - TESTING THE files PARAMETER IN simulate_request FOR DIFFERENT DATA + +# region - TESTING CONSISTENCY OF UPLOAD OF DIFFERENT FORMAT FOR files + payload1 = b'{"debug": true, "message": "Hello, world!", "score": 7}' FILES1 = { @@ -284,6 +294,9 @@ def test_upload_multipart_media(client): ] +# endregion + +# region - TEST DIFFERENT DATA TYPES in json part def asserts_data_types(resp): assert resp.status_code == 200 expected_list = [ @@ -350,7 +363,6 @@ def asserts_data_types(resp): assert all(map(lambda el: el in expected_list, resp.json)) -# -------Tests for multipart with files param and json data, where data comes in different types---------- def test_upload_multipart_datalist(client): resp = client.simulate_post( '/submit', files=FILES1, json=[('data1', 5), ('data2', ['hello', 'bonjour'])] @@ -397,6 +409,10 @@ def test_upload_multipart_datadictdict(client): asserts_data_types(resp) +# endregion + + +# region - TEST INVALID DATA TYPES FOR FILES def test_invalid_files(client): """invalid file type""" with pytest.raises(ValueError): @@ -427,7 +443,80 @@ def test_invalid_databyte(client): client.simulate_post('/submit', files=FILES1, json=b'yo self') -# ------TEST UPLOADING ACTUAL FILES: TEXT, IMAGE ---------------------------- +# endregion + +# region - TEST NESTED FILES UPLOAD + + +FILES6 = { + 'field1': 'Joe Blow', + 'docs': ( + None, + json.dumps( + { + 'file1': ( + 'file1.txt', + 'this is file1', + None, + {'Content-Disposition': 'attachment'}, + ), + 'file2': ( + 'file2.txt', + 'Hello, World!', + None, + { + 'Content-Disposition': 'attachment; Content-Transfer-Encoding: binary' + }, + ), + } + ).encode(), + 'multipart/mixed', + ), + 'document': (None, payload1, 'application/json'), +} + + +def test_nested_multipart_mixed(client): + resp = client.simulate_post('/submit', files=FILES6) + print(f'response in mixed is {resp.json}') + # assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': 'Joe Blow', + 'filename': 'field1', + 'name': 'field1', + 'secure_filename': 'field1', + 'text': 'Joe Blow', + }, + { + 'content_type': 'multipart/mixed', + 'data': [ + {'name': 'file1', 'text': 'this is file1'}, + {'name': 'file2', 'text': 'Hello, World!'}, + ], + 'filename': None, + 'name': 'docs', + 'secure_filename': None, + 'text': None, + }, + { + 'content_type': 'application/json', + 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', + 'filename': None, + 'name': 'document', + 'secure_filename': None, + 'text': None, + }, + ] + + +# endregion + +# endregion + +# region - TEST UPLOADING ACTUAL FILES: TEXT, IMAGE + LOREM_FILE = ( 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do ' @@ -495,64 +584,6 @@ def test_upload_image(client): ] -FILES6 = { - 'field1': 'Joe Blow', - 'docs': ( - None, - json.dumps( - { - 'file1': ( - 'file1.txt', - 'this is file1', - None, - {'Content-Disposition': 'attachment'}, - ), - 'file2': ( - 'file2.txt', - 'Hello, World!', - None, - {'Content-Disposition': 'attachment'}, - ), - } - ).encode(), - 'multipart/mixed', - ), - 'document': (None, payload1, 'application/json'), -} - - -def test_nested_multipart_mixed(client): - resp = client.simulate_post('/submit', files=FILES6) - print(f'response in mixed is {resp.json}') - assert resp.status_code == 200 - assert resp.json == [ - { - 'content_type': 'text/plain', - 'data': 'Joe Blow', - 'filename': 'field1', - 'name': 'field1', - 'secure_filename': 'field1', - 'text': 'Joe Blow', - }, - { - 'content_type': 'multipart/mixed', - 'data': '{"file1": ["file1.txt", "this is file1"], "file2": ["file2.txt", "Hello, World!"]}', - 'filename': None, - 'name': 'docs', - 'secure_filename': None, - 'text': None, - }, - { - 'content_type': 'application/json', - 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', - 'filename': None, - 'name': 'document', - 'secure_filename': None, - 'text': None, - }, - ] - - # def test_upload_file_cgi(client): # resp = client.simulate_post('/uploadcgi', files={'file': open('tests/loremipsum.txt', 'rb')}) # From 835b062c4c11e9deed1508c8eb934bacb07f4fc5 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 12:26:34 +0100 Subject: [PATCH 09/48] clean up --- tests/test_multipart_formdata_request.py | 33 +----------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 291f92f7c..7621d2d15 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -130,27 +130,6 @@ async def on_post_image(self, req, resp): resp.media = values -class MultipartFileUpload: - def on_post(self, req, resp): - upload = cgi.FieldStorage(fp=req.stream, environ=req.env) - data = upload['file'].file.read().decode('utf-8') - resp.media = dict(data=data) - - -class AsyncMultipartFileUpload: - async def on_post(self, req, resp): - deserialized = [] - form = await req.media - async for part in form: - upload = cgi.FieldStorage(fp=part.stream, environ=req.scope) - print(f'upload is {upload.fp.read()}') - data = await upload['file'].file.read().decode('utf-8') - - print(f'the dict is {dict(data=data)} and the dump is {dict(data=data)}') - deserialized.append(dict(data=data)) - resp.media = deserialized - - @pytest.fixture def client(asgi): app = create_app(asgi) @@ -172,10 +151,8 @@ def client(asgi): app.req_options.default_media_type = falcon.MEDIA_MULTIPART resource = AsyncMultipartAnalyzer() if asgi else MultipartAnalyzer() - resourcecgi = AsyncMultipartFileUpload() if asgi else MultipartFileUpload() app.add_route('/submit', resource) - app.add_route('/uploadcgi', resourcecgi) app.add_route('/media', resource, suffix='media') app.add_route('/image', resource, suffix='image') @@ -478,8 +455,7 @@ def test_invalid_databyte(client): def test_nested_multipart_mixed(client): resp = client.simulate_post('/submit', files=FILES6) - print(f'response in mixed is {resp.json}') - # assert resp.status_code == 200 + assert resp.status_code == 200 assert resp.json == [ { 'content_type': 'text/plain', @@ -582,10 +558,3 @@ def test_upload_image(client): 'secure_filename': filename.replace('/', '_'), } ] - - -# def test_upload_file_cgi(client): -# resp = client.simulate_post('/uploadcgi', files={'file': open('tests/loremipsum.txt', 'rb')}) -# -# # assert resp.status_code == 200 -# assert resp.json == LOREM_FILE From 3e28a9451476320b93ee226fe255e3d3eb68c131 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 12:50:10 +0100 Subject: [PATCH 10/48] updated docstrings --- falcon/testing/client.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index bbf66bc5d..63df21018 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -532,6 +532,15 @@ def simulate_request( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. + files(dict): same as the files parameter in requests, + dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) + for multipart encoding upload. + ``file-tuple``: can be a 2-tuple ``('filename', fileobj)``, + 3-tuple ``('filename', fileobj, 'content_type')`` + or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, + where ``'content-type'`` is a string defining the content + type of the given file and ``custom_headers`` a dict-like + object containing additional headers to add for the file. file_wrapper (callable): Callable that returns an iterable, to be used as the value for *wsgi.file_wrapper* in the WSGI environ (default: ``None``). This can be used to test @@ -743,6 +752,15 @@ async def _simulate_request_asgi( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. + files(dict): same as the files parameter in requests, + dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) + for multipart encoding upload. + ``file-tuple``: can be a 2-tuple ``('filename', fileobj)``, + 3-tuple ``('filename', fileobj, 'content_type')`` + or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, + where ``'content-type'`` is a string defining the content + type of the given file and ``custom_headers`` a dict-like + object containing additional headers to add for the file. host(str): A string to use for the hostname part of the fully qualified request URL (default: 'falconframework.org') remote_addr (str): A string to use as the remote IP address for the @@ -2148,6 +2166,7 @@ def _encode_files(files, data=None): if parameters are supplied as a dict. The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) or 4-tuples (filename, fileobj, contentype, custom_headers). + Allows for content_type = ``multipart/mixed`` for submission of nested files """ fields = [] if data and not isinstance(data, (list, dict)): From 7920cdd79ff5e7482642107834a2b7d9884accc0 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 13:19:47 +0100 Subject: [PATCH 11/48] Fixes --- falcon/testing/client.py | 12 +++++++---- requirements/tests | 1 + tests/test_multipart_formdata_request.py | 27 +++++++++++++++--------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 63df21018..4718ec04f 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -18,21 +18,24 @@ WSGI callable, without having to stand up a WSGI server. """ -import os import asyncio +import os import datetime as dt import inspect import json as json_module -import time import json +import time from typing import Dict from typing import Optional from typing import Sequence from typing import Union import warnings -from urllib3.filepost import encode_multipart_formdata, RequestField import wsgiref.validate +from urllib3.filepost import encode_multipart_formdata +from urllib3.filepost import RequestField + + from falcon.asgi_spec import ScopeType from falcon.constants import COMBINED_METHODS from falcon.constants import MEDIA_JSON @@ -2164,7 +2167,8 @@ def _encode_files(files, data=None): Will successfully encode files when passed as a dict or a list of tuples. Order is retained if data is a list of tuples but arbitrary if parameters are supplied as a dict. - The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) + The tuples may be 2-tuples (filename, fileobj), + 3-tuples (filename, fileobj, contentype) or 4-tuples (filename, fileobj, contentype, custom_headers). Allows for content_type = ``multipart/mixed`` for submission of nested files """ diff --git a/requirements/tests b/requirements/tests index 187bb054e..b3642f8ad 100644 --- a/requirements/tests +++ b/requirements/tests @@ -2,6 +2,7 @@ coverage >= 4.1 pytest pyyaml requests +urllib3 # TODO(vytas): Check if testtools still brings anything to the table, and # re-enable if/when unittest2 is adjusted to support CPython 3.10. testtools; python_version < '3.10' diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 7621d2d15..130864207 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -1,8 +1,7 @@ -import cgi -import io +import base64 import json + import pytest -import base64 import falcon from falcon import media @@ -54,8 +53,8 @@ def on_post_media(self, req, resp): def on_post_image(self, req, resp): values = [] for part in req.media: - # Save a copy of the image, encode in base64 and re-decode to compare. This way we avoid invalid characters - # and get a more 'manageable' output. + # Save a copy of the image, encode in base64 and re-decode to compare. + # To avoid invalid characters and get a more 'manageable' output. new_filename = part.filename.split('.')[0] + '_posted.png' f = open(new_filename, 'w+b') f.write(part.data) @@ -86,7 +85,11 @@ async def on_post(self, req, resp): if part_type.startswith('multipart/mixed'): part_form = await part.get_media() async for nested in part_form: - inner_form.append({'name': nested.name, 'text': await nested.text}) + inner_form.append( + { + 'name': nested.name, 'text': await nested.text + } + ) part_type = 'multipart/mixed' # ---------------------------------------------------- values.append( @@ -334,8 +337,11 @@ def asserts_data_types(resp): 'text': 'Hello, world!', }, ] - # Result will be unordered, because both fileobj and data are present. When all files are tuples, response will be - # unordered if json contains dictionaries - then resp.json == expected_list can be used. + + # Result will be unordered, because both fileobj and data are present. When all files + # are tuples, response will be unordered if json contains dictionaries - t + # hen resp.json == expected_list can be used. + assert len(resp.json) == len(expected_list) assert all(map(lambda el: el in expected_list, resp.json)) @@ -396,7 +402,7 @@ def test_invalid_files(client): client.simulate_post('/submit', files='heya') -def test_invalid_files(client): +def test_invalid_files_null(client): """empty file in files""" with pytest.raises(ValueError): client.simulate_post('/submit', files={'file': ()}) @@ -442,7 +448,8 @@ def test_invalid_databyte(client): 'Hello, World!', None, { - 'Content-Disposition': 'attachment; Content-Transfer-Encoding: binary' + 'Content-Disposition': 'attachment; ' + 'Content-Transfer-Encoding: binary' }, ), } From ded1ac023b83213b37576bb60dc9f8fc442eb7c0 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 13:56:24 +0100 Subject: [PATCH 12/48] Fixes2 --- falcon/testing/client.py | 112 ++++++++++++++--------- tests/test_multipart_formdata_request.py | 6 +- 2 files changed, 69 insertions(+), 49 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 4718ec04f..5c8a23c78 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -19,8 +19,8 @@ """ import asyncio -import os import datetime as dt +import os import inspect import json as json_module import json @@ -2162,31 +2162,23 @@ async def __aexit__(self, exc_type, exc, tb): await self._task_req -def _encode_files(files, data=None): - """Build the body for a multipart/form-data request. - Will successfully encode files when passed as a dict or a list of - tuples. Order is retained if data is a list of tuples but arbitrary - if parameters are supplied as a dict. - The tuples may be 2-tuples (filename, fileobj), - 3-tuples (filename, fileobj, contentype) - or 4-tuples (filename, fileobj, contentype, custom_headers). - Allows for content_type = ``multipart/mixed`` for submission of nested files +def _prepare_data_fields(data): + + """ + Args: + data: dict or list of tuples with json data from the request + + Returns: list of 2-tuples (field-name(str), value(bytes) + """ fields = [] + new_fields = [] if data and not isinstance(data, (list, dict)): raise ValueError('Data must not be a list of tuples or dict.') elif data and isinstance(data, dict): fields = list(data.items()) elif data: fields = list(data) - - if not isinstance(files, (dict, list)): - raise ValueError('cannot encode objects that are not 2-tuples') - elif isinstance(files, dict): - files = list(files.items()) - - new_fields = [] - # Append data to the other multipart parts for field, val in fields: if isinstance(val, str) or not hasattr(val, '__iter__'): @@ -2203,35 +2195,67 @@ def _encode_files(files, data=None): v.encode('utf-8') if isinstance(v, str) else v, ) ) + return new_fields + + +def _prepare_files(k, v): + + """ + Args: + k: (str), file-name + v: fileobj or tuple (filename, data, content_type?, headers?) + + Returns: + + """ + file_content_type = None + file_header = None + if isinstance(v, (tuple, list)): + if len(v) == 2: + file_name, file_data = v + elif len(v) == 3: + file_name, file_data, file_content_type = v + else: + file_name, file_data, file_content_type, file_header = v + if ( + len(v) >= 3 + and file_content_type + and file_content_type.startswith('multipart/mixed') + ): + file_data, assigned_type = _encode_files(json.loads(file_data.decode())) + file_content_type = 'multipart/mixed; ' + (assigned_type.split('; ')[1]) + else: + # if v is not a tuple or iterable it has to be a filelike obj + name = getattr(v, 'name', None) + if name and isinstance(name, str) and name[0] != '<' and name[-1] != '>': + file_name = os.path.basename(name) + else: + file_name = k + file_data = v + return file_name, file_data, file_content_type, file_header + + +def _encode_files(files, data=None): + """Build the body for a multipart/form-data request. + + Will successfully encode files when passed as a dict or a list of + tuples. Order is retained if data is a list of tuples but arbitrary + if parameters are supplied as a dict. + The tuples may be 2-tuples (filename, fileobj), + 3-tuples (filename, fileobj, contentype) + or 4-tuples (filename, fileobj, contentype, custom_headers). + Allows for content_type = ``multipart/mixed`` for submission of nested files""" + + new_fields = _prepare_data_fields(data) + + if not isinstance(files, (dict, list)): + raise ValueError('cannot encode objects that are not 2-tuples') + elif isinstance(files, dict): + files = list(files.items()) for (k, v) in files: - # support for explicit filename - file_content_type = None - file_header = None content_disposition = None - if isinstance(v, (tuple, list)): - if len(v) == 2: - file_name, file_data = v - elif len(v) == 3: - file_name, file_data, file_content_type = v - else: - file_name, file_data, file_content_type, file_header = v - if ( - len(v) >= 3 - and file_content_type - and file_content_type.startswith('multipart/mixed') - ): - file_data, assigned_type = _encode_files(json.loads(file_data.decode())) - file_data = file_data - file_content_type = 'multipart/mixed; ' + (assigned_type.split('; ')[1]) - else: - # if v is not a tuple or iterable it has to be a filelike obj - name = getattr(v, 'name', None) - if name and isinstance(name, str) and name[0] != '<' and name[-1] != '>': - file_name = os.path.basename(name) - else: - file_name = k - file_data = v + file_name, file_data, file_content_type, file_header = _prepare_files(k, v) if file_data is None: continue diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 130864207..34bfb203b 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -85,11 +85,7 @@ async def on_post(self, req, resp): if part_type.startswith('multipart/mixed'): part_form = await part.get_media() async for nested in part_form: - inner_form.append( - { - 'name': nested.name, 'text': await nested.text - } - ) + inner_form.append({'name': nested.name, 'text': await nested.text}) part_type = 'multipart/mixed' # ---------------------------------------------------- values.append( From 574e6674c72b998afb152457ab2521242f0f9642 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 14:11:53 +0100 Subject: [PATCH 13/48] Fixes3 --- falcon/testing/client.py | 8 ++--- tests/test_multipart_formdata_request.py | 37 +++++------------------- 2 files changed, 11 insertions(+), 34 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 5c8a23c78..ff2983ea2 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -20,8 +20,8 @@ import asyncio import datetime as dt -import os import inspect +import os import json as json_module import json import time @@ -2163,8 +2163,8 @@ async def __aexit__(self, exc_type, exc, tb): def _prepare_data_fields(data): + """Prepares data fields for request body. - """ Args: data: dict or list of tuples with json data from the request @@ -2199,13 +2199,13 @@ def _prepare_data_fields(data): def _prepare_files(k, v): + """Prepares file attributes for body of request form. - """ Args: k: (str), file-name v: fileobj or tuple (filename, data, content_type?, headers?) - Returns: + Returns: file_name, file_data, file_content_type, file_header """ file_content_type = None diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 34bfb203b..1c337e9ec 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -334,9 +334,9 @@ def asserts_data_types(resp): }, ] - # Result will be unordered, because both fileobj and data are present. When all files - # are tuples, response will be unordered if json contains dictionaries - t - # hen resp.json == expected_list can be used. + # Result will be unordered, because both fileobj and data are present. + # When all files are tuples, response will be unordered if json + # contains dictionaries - then resp.json == expected_list can be used. assert len(resp.json) == len(expected_list) assert all(map(lambda el: el in expected_list, resp.json)) @@ -497,45 +497,22 @@ def test_nested_multipart_mixed(client): # region - TEST UPLOADING ACTUAL FILES: TEXT, IMAGE -LOREM_FILE = ( - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do ' - 'eiusmod tempor\n' - 'incididunt ut labore et dolore magna aliqua. Dolor sed viverra ' - 'ipsum nunc\n' - 'aliquet bibendum enim. In massa tempor nec feugiat. Nunc aliquet ' - 'bibendum enim\n' - 'facilisis gravida. Nisl nunc mi ipsum faucibus vitae aliquet nec ' - 'ullamcorper.\n' - 'Amet luctus venenatis lectus magna fringilla. Volutpat maecenas ' - 'volutpat blandit\n' - 'aliquam etiam erat velit scelerisque in. Egestas egestas fringilla ' - 'phasellus\n' - 'faucibus scelerisque eleifend. Sagittis orci a scelerisque purus ' - 'semper eget\n' - 'duis. Nulla pharetra diam sit amet nisl suscipit. Sed adipiscing ' - 'diam donec\n' - 'adipiscing tristique risus nec feugiat in. Fusce ut placerat orci ' - 'nulla.\n' - 'Pharetra vel turpis nunc eget lorem dolor. Tristique senectus et ' - 'netus et\n' - 'malesuada.' -) - - def test_upload_file(client): resp = client.simulate_post( '/submit', files={'file': open('tests/files/loremipsum.txt', 'rb')} ) + lorem_file = open('tests/files/loremipsum.txt', 'rb').read().decode() + assert resp.status_code == 200 assert resp.json == [ { 'content_type': 'text/plain', - 'data': LOREM_FILE, + 'data': lorem_file, 'filename': 'loremipsum.txt', 'name': 'file', 'secure_filename': 'loremipsum.txt', - 'text': LOREM_FILE, + 'text': lorem_file, }, ] From ffa8d9f86287463e981686a96adf2f5138e255be Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 14:23:10 +0100 Subject: [PATCH 14/48] Fixes4 --- falcon/testing/client.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index ff2983ea2..b78e40e3a 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -21,9 +21,8 @@ import asyncio import datetime as dt import inspect -import os import json as json_module -import json +import os import time from typing import Dict from typing import Optional @@ -2163,7 +2162,7 @@ async def __aexit__(self, exc_type, exc, tb): def _prepare_data_fields(data): - """Prepares data fields for request body. + """Prepare data fields for request body. Args: data: dict or list of tuples with json data from the request @@ -2199,7 +2198,7 @@ def _prepare_data_fields(data): def _prepare_files(k, v): - """Prepares file attributes for body of request form. + """Prepare file attributes for body of request form. Args: k: (str), file-name @@ -2222,7 +2221,9 @@ def _prepare_files(k, v): and file_content_type and file_content_type.startswith('multipart/mixed') ): - file_data, assigned_type = _encode_files(json.loads(file_data.decode())) + file_data, assigned_type = _encode_files( + json_module.loads(file_data.decode()) + ) file_content_type = 'multipart/mixed; ' + (assigned_type.split('; ')[1]) else: # if v is not a tuple or iterable it has to be a filelike obj From eadb1f364d26ab2cff26dcab48ccf8bc4df2efb5 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 14:42:31 +0100 Subject: [PATCH 15/48] urllib3 in minitest requirements (?) --- requirements/mintest | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/mintest b/requirements/mintest index 8fce419e3..19a6c04ba 100644 --- a/requirements/mintest +++ b/requirements/mintest @@ -5,3 +5,4 @@ pytest pyyaml requests ujson +urllib3 From 7493b92d86bf52c569aa24cebb8344e138490e12 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 15:15:09 +0100 Subject: [PATCH 16/48] minor fix 5 --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index b78e40e3a..79e2fed9e 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2258,7 +2258,7 @@ def _encode_files(files, data=None): content_disposition = None file_name, file_data, file_content_type, file_header = _prepare_files(k, v) - if file_data is None: + if not file_data: continue elif hasattr(file_data, 'read'): fdata = file_data.read() From c49e369b844e444e52df0b78291b1e1156312005 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Fri, 10 Feb 2023 15:31:29 +0100 Subject: [PATCH 17/48] minor fix 6 --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 79e2fed9e..9b388b51e 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2183,7 +2183,7 @@ def _prepare_data_fields(data): if isinstance(val, str) or not hasattr(val, '__iter__'): val = [val] for v in val: - if v is not None: + if v: # Don't call str() on bytestrings: in Py3 it all goes wrong. if not isinstance(v, bytes): v = str(v) From 235ed3e3fa5a934ea5eaf4682cda4561f094d6e2 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 12 Feb 2023 16:31:02 +0100 Subject: [PATCH 18/48] Removed urllib3; creating encoded bodystring in _encode_files. --- falcon/testing/client.py | 75 ++++++++++++++++-------- requirements/mintest | 2 +- requirements/tests | 1 - tests/test_multipart_formdata_request.py | 1 + 4 files changed, 51 insertions(+), 28 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 9b388b51e..45277c673 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -19,6 +19,7 @@ """ import asyncio +import binascii import datetime as dt import inspect import json as json_module @@ -31,10 +32,6 @@ import warnings import wsgiref.validate -from urllib3.filepost import encode_multipart_formdata -from urllib3.filepost import RequestField - - from falcon.asgi_spec import ScopeType from falcon.constants import COMBINED_METHODS from falcon.constants import MEDIA_JSON @@ -2221,10 +2218,10 @@ def _prepare_files(k, v): and file_content_type and file_content_type.startswith('multipart/mixed') ): - file_data, assigned_type = _encode_files( - json_module.loads(file_data.decode()) + file_data, new_header = _encode_files(json_module.loads(file_data.decode())) + file_content_type = 'multipart/mixed; ' + ( + new_header['Content-Type'].split('; ')[1] ) - file_content_type = 'multipart/mixed; ' + (assigned_type.split('; ')[1]) else: # if v is not a tuple or iterable it has to be a filelike obj name = getattr(v, 'name', None) @@ -2233,48 +2230,76 @@ def _prepare_files(k, v): else: file_name = k file_data = v + if hasattr(file_data, 'read'): + file_data = file_data.read() return file_name, file_data, file_content_type, file_header +def _make_boundary(): + """ + Create random boundary to be used in multipar/form-data with files + """ + boundary = binascii.hexlify(os.urandom(16)) + boundary = boundary.decode('ascii') + return boundary + + def _encode_files(files, data=None): """Build the body for a multipart/form-data request. Will successfully encode files when passed as a dict or a list of - tuples. Order is retained if data is a list of tuples but arbitrary - if parameters are supplied as a dict. + tuples. ``data`` fields are added first. The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) or 4-tuples (filename, fileobj, contentype, custom_headers). - Allows for content_type = ``multipart/mixed`` for submission of nested files""" + Allows for content_type = ``multipart/mixed`` for submission of nested files - new_fields = _prepare_data_fields(data) + Returns: (encoded body string, headers dict) + """ + content_disposition = None + boundary = _make_boundary() + body_string = b'--' + boundary.encode() + b'\r\n' + header = {'Content-Type': 'multipart/form-data; boundary=' + boundary} + # Handle whatever json data gets passed along with files + if data: + data_fields = _prepare_data_fields(data) + for f, val in data_fields: + body_string += f'Content-Disposition: ' \ + f'{content_disposition or "fom-data"}; name={f}; \r\n\r\n'.encode() + body_string += val.encode('utf-8') if isinstance(val, str) else val + body_string += b'\r\n--' + boundary.encode() + b'\r\n' + + # Deal with the files tuples if not isinstance(files, (dict, list)): raise ValueError('cannot encode objects that are not 2-tuples') elif isinstance(files, dict): files = list(files.items()) for (k, v) in files: - content_disposition = None file_name, file_data, file_content_type, file_header = _prepare_files(k, v) - if not file_data: continue - elif hasattr(file_data, 'read'): - fdata = file_data.read() - else: - fdata = file_data if file_header and 'Content-Disposition' in file_header.keys(): content_disposition = file_header['Content-Disposition'] - rf = RequestField(name=k, filename=file_name, data=fdata, headers=file_header) - rf.make_multipart( - content_type=file_content_type, content_disposition=content_disposition + + body_string += f'Content-Disposition: {content_disposition or "fom-data"}; name={k}; '.encode() + body_string += ( + f'filename={file_name or "null"}\r\n'.encode() + if file_name + else f'\r\n'.encode() + ) + body_string += ( + f'Content-Type: {file_content_type or "text/plain"}\r\n\r\n'.encode() ) - new_fields.append(rf) + body_string += ( + file_data.encode('utf-8') if isinstance(file_data, str) else file_data + ) + body_string += b'\r\n--' + boundary.encode() + b'\r\n' - body, content_type = encode_multipart_formdata(new_fields) + body_string = body_string[:-2] + b'--\r\n' - return body, content_type + return body_string, header def _prepare_sim_args( @@ -2317,9 +2342,7 @@ def _prepare_sim_args( headers['Content-Type'] = content_type if files is not None: - body, content_type = _encode_files(files, json) - headers = headers or {} - headers['Content-Type'] = content_type + body, headers = _encode_files(files, json) elif json is not None: body = json_module.dumps(json, ensure_ascii=False) diff --git a/requirements/mintest b/requirements/mintest index 19a6c04ba..50f757e50 100644 --- a/requirements/mintest +++ b/requirements/mintest @@ -5,4 +5,4 @@ pytest pyyaml requests ujson -urllib3 + diff --git a/requirements/tests b/requirements/tests index b3642f8ad..187bb054e 100644 --- a/requirements/tests +++ b/requirements/tests @@ -2,7 +2,6 @@ coverage >= 4.1 pytest pyyaml requests -urllib3 # TODO(vytas): Check if testtools still brings anything to the table, and # re-enable if/when unittest2 is adjusted to support CPython 3.10. testtools; python_version < '3.10' diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 1c337e9ec..f73f8ba02 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -60,6 +60,7 @@ def on_post_image(self, req, resp): f.write(part.data) f.close() new_file64 = base64.b64encode(open(new_filename, 'rb').read()) + values.append( { 'content_type': part.content_type, From 94ae9790e8fc7dad8fec610c568fb6efebcdc8ca Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 12 Feb 2023 16:31:28 +0100 Subject: [PATCH 19/48] blued --- falcon/testing/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 45277c673..fed5e528b 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2265,8 +2265,10 @@ def _encode_files(files, data=None): if data: data_fields = _prepare_data_fields(data) for f, val in data_fields: - body_string += f'Content-Disposition: ' \ - f'{content_disposition or "fom-data"}; name={f}; \r\n\r\n'.encode() + body_string += ( + f'Content-Disposition: ' + f'{content_disposition or "fom-data"}; name={f}; \r\n\r\n'.encode() + ) body_string += val.encode('utf-8') if isinstance(val, str) else val body_string += b'\r\n--' + boundary.encode() + b'\r\n' From c9d4f1b1fd8578c7b14c8f87c09d3c0c8496ea39 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 12 Feb 2023 16:38:42 +0100 Subject: [PATCH 20/48] formatting corrections --- falcon/testing/client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index fed5e528b..bf718dcde 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2237,7 +2237,7 @@ def _prepare_files(k, v): def _make_boundary(): """ - Create random boundary to be used in multipar/form-data with files + Create random boundary to be used in multipar/form-data with files. """ boundary = binascii.hexlify(os.urandom(16)) boundary = boundary.decode('ascii') @@ -2284,12 +2284,16 @@ def _encode_files(files, data=None): continue if file_header and 'Content-Disposition' in file_header.keys(): content_disposition = file_header['Content-Disposition'] + else: + content_disposition = 'form-data' - body_string += f'Content-Disposition: {content_disposition or "fom-data"}; name={k}; '.encode() + body_string += ( + f'Content-Disposition: {content_disposition}; name={k}; '.encode() + ) body_string += ( f'filename={file_name or "null"}\r\n'.encode() if file_name - else f'\r\n'.encode() + else '\r\n'.encode() ) body_string += ( f'Content-Type: {file_content_type or "text/plain"}\r\n\r\n'.encode() From 9a9e58d37cede11fdb9281b3a6ba53fdae8f4f89 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 12 Feb 2023 18:26:34 +0100 Subject: [PATCH 21/48] Corrections after comments --- falcon/testing/client.py | 26 +-- tests/files/falcon.png | Bin 3613 -> 0 bytes tests/files/loremipsum.txt | 11 - tests/test_multipart_formdata_request.py | 261 ++++++++++++++++++----- 4 files changed, 216 insertions(+), 82 deletions(-) delete mode 100644 tests/files/falcon.png delete mode 100644 tests/files/loremipsum.txt diff --git a/falcon/testing/client.py b/falcon/testing/client.py index bf718dcde..d859bbd85 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2205,16 +2205,17 @@ def _prepare_files(k, v): """ file_content_type = None - file_header = None + file_data = None + file_name = None + if not v: + raise ValueError(f'No file provided for {k}') if isinstance(v, (tuple, list)): if len(v) == 2: file_name, file_data = v elif len(v) == 3: file_name, file_data, file_content_type = v - else: - file_name, file_data, file_content_type, file_header = v if ( - len(v) >= 3 + len(v) == 3 and file_content_type and file_content_type.startswith('multipart/mixed') ): @@ -2232,7 +2233,7 @@ def _prepare_files(k, v): file_data = v if hasattr(file_data, 'read'): file_data = file_data.read() - return file_name, file_data, file_content_type, file_header + return file_name, file_data, file_content_type def _make_boundary(): @@ -2249,9 +2250,8 @@ def _encode_files(files, data=None): Will successfully encode files when passed as a dict or a list of tuples. ``data`` fields are added first. - The tuples may be 2-tuples (filename, fileobj), - 3-tuples (filename, fileobj, contentype) - or 4-tuples (filename, fileobj, contentype, custom_headers). + The tuples may be 2-tuples (filename, fileobj) or + 3-tuples (filename, fileobj, contentype). Allows for content_type = ``multipart/mixed`` for submission of nested files Returns: (encoded body string, headers dict) @@ -2279,17 +2279,11 @@ def _encode_files(files, data=None): files = list(files.items()) for (k, v) in files: - file_name, file_data, file_content_type, file_header = _prepare_files(k, v) + file_name, file_data, file_content_type = _prepare_files(k, v) if not file_data: continue - if file_header and 'Content-Disposition' in file_header.keys(): - content_disposition = file_header['Content-Disposition'] - else: - content_disposition = 'form-data' - body_string += ( - f'Content-Disposition: {content_disposition}; name={k}; '.encode() - ) + body_string += f'Content-Disposition: form-data; name={k}; '.encode() body_string += ( f'filename={file_name or "null"}\r\n'.encode() if file_name diff --git a/tests/files/falcon.png b/tests/files/falcon.png deleted file mode 100644 index 69246223b41927dd1708b12bb6a8e9c3f35c070b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3613 zcmV+&4&w2NP)ec=8=KAK%^SyQQxohyR zQS`-#`QgL&)~)#5xb@7J{O{iJwPEtRaQNA>`{~p1u~+)!$o0vP_R*pA!h!eHr}W2* z^~#j*q%!u5mDO(?e_ow#9>nr10*33 zX2$#Jz0(KGB}rAKw#;TUwrY!7ymOjx(6^Y96!)Kgkce{Xry}#gCfR2E!~3{ z5(^4=t9uYbBAr2ldb&q3AiNbAu-83`0pT3MFwlQk&e-#mAgo;7;}{NmY#0VU>mEiM ztX@*^MpA+XTBsV$5LnN9JiI{}K{zTK%>an0FaPj~W(19CD?=aMq@Q&FjL9$3B_bQ? zhj#@*gD4!3YNWCrEoC3xqJkhKDu<*PYhs{S^4Voc&>$05_jfZD6OxAlcU7*@Dchun zR1|`&vRLDipaDt;r4kpjhdHt|p9FiTn?Yf%^L>Ouxu>6N)F2nN15*WOM~Y1(XESff zgWy!p6%j4w0OTA<9(ocq%qi;o*njLIa+P{Pf?o{L^!$kS9X|(1rL^IqGKI$U&W6$P zwTECXr44;BDzC6rkFX!26{-x6+-2VJLn~7VyD9k*?dWJ$_@xY`7HM@Crm|`}44q4j zpQN((s4+=~KM9a`eyJEd$-L=KCdkma&IvZ^fSuIK(ECd!AZgslrytXo!IZMt!gq!S z(X1_;Fu{mLPuhXx*WMS6%edPrsIX`TXO>BYQENaJ@2d7rpe$8*)d2 zxV}io^+b7xy@5FBWglL0SP}VnmE?vNhj-nf&b9gc?y^44Geb0+V=PneI`7_kmCr7B zYD%$Z{K)X08gCPb&`Sw|>{qu}jNgIdpoW_j`b3Cc_R-}OMGBX)-Z!MuckZh;t5u2d_0~|2Y`IF#h zz^a#flc5fCVC4}ucP+mC@{!E@>8>V*29*v&HvCZYnWz=0@275L2rHGs#!Z+KtrLKI z3xuY9C~e6z5zpl&knH+L7(PyiFQ`+BCvgRl0q9wQQ>;N-6kkx0J0p@3p$0LOh=N7S z{!nDRg;HA(kctUR;k5lcRuK(uTP4jvJ846-jt?is+wb&Gj*3BIiDI%*=a&94+kVPC z^nAU=I=gfLsIM}F>9sQKW7OwJgw+I)xmshx?fA%Ye)I>T_V334F_atqb(eP?eNHZp zA-G0U6{tSOzG<@C9+1pm$1p3yquh zVebX(EHh}?Kq2Q8lCjGGtzUuD0H9lA1g^_(@Bkh^4e}u3p2pCyR6X7)98t_X5w8Rs z4`8yDRZ(DUagZO&7LT-v!~qA3wDhYkRkAVg7c zS!8Vwgsh)*+v;-BuyA>=WzUIWSOv-NCOTym>h$%rY!*oRHEN@7VPTudbAm_%*&Dq}_ z!A#ex96K$g#%r)hSO@cb-lPV`*aT|_j*boePqEbkxHQ1g%Y8 z;1Lqpcp!3Q7sE6>iLaKl>R~FbPn8637*GOq8lJ?{Tci{5 zBzDxa^}$3!7K`B+>TU>Ef;aVB98>Wm4#;pKp2Pte`qNN{awXV&6#tSbIHDRGW~cz} z*N{aGwnLfSIRLbqru@|jk`%Sy5+YESB?;XTL}u8qsd1_lfM|8rF#1KN*4EbBE8LxM z)M?wqpEeu*J7i^(3MpcpJ(-`8Um%1aWFi7dzNl=KzMFCC^4K^OJOC%x!y#NQf8vK7 zVhLrscYx|UM->cV#R~IXO?C1{-$ANKE;ZS4%pcL1b~E%{(wFD#HMUH)Zpx-nedjw~ zWc?8ErSZ={J5{?;-&VYt1{lqMwyvN^Z|IBCY$^F`lLOvOu1Q%P8YAA`;o5$TivuMb z$1Ic;#xhW0sZJ=G$UqqM7poJ?Tljzij+4reVqw{LKg##&9t{g$vj6I15~tzkZ$*5= zeK?xaaD%tQOM1xFsf43Ca1yDKu0gwFiVwCgZLLnG{-O)N(iF`}C4x1<_GVXEivN!5 z+<&PbZ)H!%WHMDWE${)u4|n^dI&`~;IgqvYf+KHJj1Ry{H-S#LS^X%v3YMO}pskAp zH;UN<`8*Xe^>td*Ezkvj6=DrjiSRe#o|`oIK5K5b2hf@XKYGt0F_cxqsddtnFa~I~ z^MfO4HeglXIiVJh76s6h17HTHOsiX7lo(AH9O`Mx!-+@8k+{sXMyMv}=a*EnC@N`m zKMXl(c77Vk7Ic_&x z-acICdPBp_sPw}~8Gh_T%3OQy9v>F>I>(7_#Y(SBviNAl&=_F~5nSSX&5eboXuVx3 zvIQ;b)zNJ|N;imZ}Kx-ad*qBMew)U{P-{Zhy$&*O?dN@^CpaogFt9J z)>m#;Hkw#Aqr`nS7AoALsCNX_Zvd_E6u@p}b7 z-toL6)VShUw;vKjC4h>@7PEXIdf_sYOz&zgK zigsNGwAa<%8!!gtAT;HN&$ScJM1=p9EqS*qs8p#pg0ur%CgN zf5*tOSG7reE-5>y2v+gC3r{57;+w}b%HECA56;1wc~ze%ULlO}Uolxfm44D7v&^!d z+hiV{i`{N&vfmze;o)p1&Cm_c>veu*Qk+zR!DudZ403ijm0;^b@u`rOo9vxZ+R-o~ zuN`Q6%gSjF2$sD$&CW_!@i98*B8^DhaMKfI3OHG3S*X~&+JeAc;MXD?UCWHZ-Pu|y_31K%q192Mm$a>=H>bC z%#)v8EA70D(ZcD{*C!a0hgc5XJ;{}C-*v%{3w>>~bP8ea6au~g3AY)?-r3F1MM2MY z?n3~28*3WfiW9{LI6CPJR&2G@sLQUX#yy3@>yld%IE58Ojv6xj&NNkvURBP}j9F$1 z^mQiPUIr4xUj7Zi^ha0!c51skoWVt68!9YRO8ch8R-CRt53^ftD`KSZ8eMK2%u)+3 z^QBZs(M$hq!>TjHOsue2DQGvI#ChPiwBm}Z*3|W$d9x)_^VBIyt^oOCXH6Fj&1+H8 zr!GP5y{iL&H#?W@|He+*$SGlxtbu29IZ zmbC&Td|oaxZUuuCK;ebYk5%d}b@*stpIYKGw@C0D`Ra%RDnj(R8KN$mf>FnsQnd2T zIC2voYff?{wZKX8g3(Jf+|ru5V}-XF+6;m~@GOffV0KJ-ctk|u6o%DB z3R$GYr1Nub3p)N@Q+n8ya)x1e5dP5zV71$?@0(W#+?lTYx5G{4FI!#?M;o0lzWCyc jFTVKVi!Z+Tq5=N^DIcZRUSct<00000NkvXXu0mjfyO8~# diff --git a/tests/files/loremipsum.txt b/tests/files/loremipsum.txt deleted file mode 100644 index f83a7a255..000000000 --- a/tests/files/loremipsum.txt +++ /dev/null @@ -1,11 +0,0 @@ -Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor -incididunt ut labore et dolore magna aliqua. Dolor sed viverra ipsum nunc -aliquet bibendum enim. In massa tempor nec feugiat. Nunc aliquet bibendum enim -facilisis gravida. Nisl nunc mi ipsum faucibus vitae aliquet nec ullamcorper. -Amet luctus venenatis lectus magna fringilla. Volutpat maecenas volutpat blandit -aliquam etiam erat velit scelerisque in. Egestas egestas fringilla phasellus -faucibus scelerisque eleifend. Sagittis orci a scelerisque purus semper eget -duis. Nulla pharetra diam sit amet nisl suscipit. Sed adipiscing diam donec -adipiscing tristique risus nec feugiat in. Fusce ut placerat orci nulla. -Pharetra vel turpis nunc eget lorem dolor. Tristique senectus et netus et -malesuada. \ No newline at end of file diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index f73f8ba02..c19fb4477 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -17,7 +17,8 @@ class MultipartAnalyzer: - def on_post(self, req, resp): + @staticmethod + def on_post(req, resp): values = [] for part in req.media: # For mixed nested file requests @@ -41,7 +42,8 @@ def on_post(self, req, resp): resp.media = values - def on_post_media(self, req, resp): + @staticmethod + def on_post_media(req, resp): deserialized = [] for part in req.media: part_media = part.get_media() @@ -50,23 +52,15 @@ def on_post_media(self, req, resp): deserialized.append(part_media) resp.media = deserialized - def on_post_image(self, req, resp): + @staticmethod + def on_post_image(req, resp): values = [] for part in req.media: - # Save a copy of the image, encode in base64 and re-decode to compare. - # To avoid invalid characters and get a more 'manageable' output. - new_filename = part.filename.split('.')[0] + '_posted.png' - f = open(new_filename, 'w+b') - f.write(part.data) - f.close() - new_file64 = base64.b64encode(open(new_filename, 'rb').read()) - values.append( { 'content_type': part.content_type, - 'data': new_file64.decode(), + 'data': base64.b64encode(part.data).decode(), 'filename': part.filename, - 'new_filename': new_filename, 'name': part.name, 'secure_filename': part.secure_filename if part.filename else None, } @@ -76,7 +70,8 @@ def on_post_image(self, req, resp): class AsyncMultipartAnalyzer: - async def on_post(self, req, resp): + @staticmethod + async def on_post(req, resp): values = [] form = await req.get_media() async for part in form: @@ -101,7 +96,8 @@ async def on_post(self, req, resp): ) resp.media = values - async def on_post_media(self, req, resp): + @staticmethod + async def on_post_media(req, resp): deserialized = [] form = await req.media async for part in form: @@ -111,18 +107,17 @@ async def on_post_media(self, req, resp): resp.media = deserialized - async def on_post_image(self, req, resp): + @staticmethod + async def on_post_image(req, resp): values = [] form = await req.get_media() async for part in form: - new_filename = part.filename.split('.')[0] + '_posted.png' data = await part.data values.append( { 'content_type': part.content_type, 'data': base64.b64encode(data).decode(), 'filename': part.filename, - 'new_filename': new_filename, 'name': part.name, 'secure_filename': part.secure_filename if part.filename else None, } @@ -434,20 +429,10 @@ def test_invalid_databyte(client): None, json.dumps( { - 'file1': ( - 'file1.txt', - 'this is file1', - None, - {'Content-Disposition': 'attachment'}, - ), + 'file1': ('file1.txt', 'this is file1'), 'file2': ( 'file2.txt', 'Hello, World!', - None, - { - 'Content-Disposition': 'attachment; ' - 'Content-Transfer-Encoding: binary' - }, ), } ).encode(), @@ -497,36 +482,203 @@ def test_nested_multipart_mixed(client): # region - TEST UPLOADING ACTUAL FILES: TEXT, IMAGE - -def test_upload_file(client): - resp = client.simulate_post( - '/submit', files={'file': open('tests/files/loremipsum.txt', 'rb')} - ) - - lorem_file = open('tests/files/loremipsum.txt', 'rb').read().decode() - - assert resp.status_code == 200 - assert resp.json == [ - { - 'content_type': 'text/plain', - 'data': lorem_file, - 'filename': 'loremipsum.txt', - 'name': 'file', - 'secure_filename': 'loremipsum.txt', - 'text': lorem_file, - }, - ] +IMAGE_FILE = ( + b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xe1\x00\x00' + b'\x00\xe1\x08\x03\x00\x00\x00\tm"H\x00\x00\x00uPLTE\xff\xff\xff' + b'\xf0\xadN\xf0\xacK\xef\xa9A\xf0\xaaF\xef\xa8?\xf0\xaaE\xef\xa8' + b'=\xff\xfe\xfc\xfe\xfb\xf6\xfd\xf6\xed\xfe\xf8\xf1\xf1\xb3' + b'\\\xfb\xea\xd5\xfd\xf3\xe6\xfa\xe6\xcd\xf3\xbdu\xf2\xb9k\xf0' + b'\xafQ\xf4\xc5\x88\xf9\xe1\xc3\xf7\xd6\xad\xf8\xdc\xb8\xf5\xcc' + b'\x97\xfc\xef\xde\xf1\xb5a\xf2\xbbp\xf8\xd9\xb2\xfb\xe9\xd3\xf1' + b'\xb1W\xfa\xe4\xc8\xf5\xc9\x90\xf6\xd1\xa1\xf4\xc2\x81\xf7\xd4' + b'\xa7\xf4\xc7\x8b\xf5\xca\x94\xef\xa42\xf6\xd0\x9e_\xd7\x99\xa5' + b'\x00\x00\rcIDATx\x9c\xed]i\x97\xaa8\x10\x95JHX\x14A\x11Q\xdb' + b'\x05\xed\xf6\xff\xff\xc4aS\x11\x03$!\x10f\xc6\xfb\xe9\xbd\xd3' + b'\x07\xcc%IU\xa5\xb6\xccf_|\xf1\xc5\x17_|\xf1\xc5\x17\xff38\xba' + b'\x0700\x1c?\xd0=\x84a\x11\xafh\xa4{' + b'\x0c\x83"\xc6@-\xdd\x83\x18\x12\x0b\nx\xab{' + b'\x10C"\x9dA\x83z\xbaG1 ' + b'\xbc\x15\x18\xb0\xd7=\x8a\x01\xe1\x1c\xc10\xd0\x7fX\xce\xd8' + b'\xf3\x94 \xac\\\xdd\xe3\x18\x0e{' + b'l\x18\x06>\xeb\x1eF\x1b\xac^R\xf0FR\x82\x06Z\xa8\x1a\xcd\x10X' + b'\xcf{<\xbcA\x19A8*\x1b\xcd\x00\x88\xa9/\xff\xf0\x89f\x04\x8di' + b'+C\x1f\xdd\xa4\x9fu\x00\x8c\xc9/\xd2%"\x1b\xe9\x87w\x05A\x83(' + b'\x1c\x90j\xa4\xb2\x1e-e\x1f\xde\xa2\x82 ' + b'$*\x87\xa4\x18kb\xa0X\xf2\xd9eI\xd0 ' + b'\x13V\xf7w3\x15\x13\x92\x87\x02wU\xae\xd1)\xdb\xa4\x87T\x14' + b'\x82\xac\xb2X\xe3\x92\xa0\x01J\x07\xa5\x12\x17\xb3\x879\xb24' + b'\x9f\x04{\xa8\x9bAa\xad\xf3}\x84B\xb9\xa7\x9fk\xd4 ' + b'\x17\xb5\x03S\x05gG\x8aM$g3o\xc9\x83\xe0T\xcf\x15\x11-\xe6\x00' + b'\xe4\x1cH\x1ez\x124\xcc)\xea\xfb\xd8\x7f\xec"r\x95zA\x82_\x0c' + b'\xd1\xf4\xfc\x88\xf6\x1d?\x07H\xa5\xb4\xe1\xa22\x85\xc6\xf4' + b'\xce\x86\xd1\xf1\xb5\x87`.\xa5\r}0*\xaf\xb0U\x8f\xb0\x1f\xa2' + b'\x15\xaa\x0c\x8f\xdce\xde\xf1C+S\x08\xbb)\xf9\x11\xed\xe8hV' + b'\xf8\xa5\x0c\xa5\x16iu\x17\xa6\xb2j:\x0c\x9dK\x8d\x9f\xa4\xb2' + b'\xf6\xa81I\x86?\t\x90w~\xa9\x18<\xc9\xbc\xe9N&\xc8\xd0\xb9\xce' + b'\t6\xea\x80\x9d\xd4\xcb\xd0\xfbK& ' + b'i\xdc\xc8\xa7\x1f\xd3\x97\xc1\x94\xb2\xd8\xc2w\x86\x06\xd1\xac' + b'-\x9c0\xc1\x88EO\xda\x81\xe4\xd7\xde&\xa7R\x15\xc1\x8b|@\x9f' + b'\xab\xf314\xa9)t>6\xb36\xbb4\xde\x04\xd00{' + b'\xc5\x14\xca\x99\xa4\xf5Ej\xe0_\xc5\x03\xe7\x82\x15_\x8e\xec' + b'\xbd\xf7\x82\xf9#\xf5\xeau}MH\x9f\xa1\xe5a/\x7fW]\xf4\xd2O\xbf' + b'\x96{\xf9\xfc\xe3\xc5#{' + b'1\x9cp\xbd2q\x17\xbd\xf4\xcb\x1brG\x82\xb8\xbeH\xc7\xf5Dy\x87' + b'\xbd\x818\xe8e\x1f^rX\x11\xf9x\x95\xe4\x86\x16\x87w\xdd\xa1' + b'\xce\xb5\xf9\xfc\xee\xb2>\xce3C4\x9bc,' + b'S\xef:\xef\xdez\x95\xcf.wjJ\xb1g\xfc\xc8\xf0\x9e\x1ao\x13\x88' + b'\xd0K\t\x82\xecW\xb7W\x8c\xdf\x81\xe3\xa0\x86\x9b\x15\xfaD\x88' + b'^\xf6\xd1\xe5\x14E\n\x97\xb2\xde7\xa4\xd2w\xee\xab6\xad\xce' + b'\x86\x9c1\x93#f2\x1c\xce\xfa^\xf8\x94Or\xbe\r\x87J\x9d\x99\n' + b'\x9c>\x95E\x06$\x1f\xc0j\xfd\xb5=\x15\xa6\x97\xed\xc1\x1e\x04g' + b'\x11\x9b\xa1\x01\x03\x1c0\xd2\xf9\x93\xe0g\xc0\xaa\x97{' + b'\x93\xa1\x0es`\xe5\x116w-\xbe\xfd\xf2\x91\xcc\xfb\xe9\xae&\x86' + b'\x06\x95\x0eC6\xfcP\xf3\x99\xa8\x15\xa8\xef\xa7nd\x08+\x95\xc2' + b'\xc6M\x98\x12\xad\x13\x80{\x0b\x84\xa6}(' + b"m\xc93\x11\xcf\xe5&\x90\xec\xfaG\x18>N\x87/\xa8S\x8a'q\x05" + b'\x91\x01\xe8Y\x81S\xacA[' + b'\x14_P\x91;#\x92\x12\xa1\x061\x94\x88\x82E\xcb\xfePd\xbc\x85R[' + b'\x10\x90\x8a\tL\xe1\xb4\xfd\xc6\xbei\xb2\xdb\x1e\x90\xcc_\xc9\xd0*\xc6' + b'\x180\x83!"\n^\xe7(' + b'\xa4\xc5\xcd\x8f\x98&\x84\x81N\xa5L_\xd4;\xa8T\xec|6\xfbt6\xb7' + b"\x01\x05C%\x81\x9c\xba\xa5\x81\x99\xc8\xa8'!1\x03H.O\x86\x0b" + b'\x1c\xe2\x80\x04\x12\xc7P\x11U\x08\x862\x1b\x86\x81\x03\x87' + b']\x05 ,o6\x02\xe6\x1a\x9e\x0f\x9a\xa6\xc4gwP\xc1\xfcq\xeb\xc8' + b'?\x85d\xe8tH\x9eI\xcc\x8e\xa3B+\xb5\xf9l\xfd\x81\x1e\xc9\xf7' + b"\x9c\xb0v\\\x9f\x1b\xb0\x80\xda\xe0|g\xfe\xe9\x06'XI\xf1\xee" + b'\xfa\xd8{\xeeilq\x1e\xd4\t\x8e\x928\xff\xdbe\xd8\x94\x00\x93W' + b'\xa6wk\xd9\x12X.EF\x18;^\xfb\n\xed\xb8\x0c\xab6\xdf\xc1\xfb7' + b'\x1b6\n\xf4B\xd7\x11\xa32$r\xe5P\xff\xdc\xa7&\xe5G,' + b'\xec\xf7r%\xf1\x9a\xa4\xa4oX\xab7\xb8Ox\xe5\x94\x94%o\xbc' + b'\xaaBoI@\x01\xc1\xe0mY\x11\xc6\xe7\xd5J\x81t\xf3\x9b\t\x0c\xb6' + b'\x1cr\x16u\xe0\x9e\xc2i\xb45Xsn\xa9r\xcc\xd9$r\xbb\x81\xf5o' + b'\xc3\x0c\x96Ph%\xdb\x89.\xf7\x03\x92\xb5\x13\xaa\xc1\xce>mb' + b'\x98\xce!\x7f\xc0\x10\xe9\xaf\x1f\xcb!\x12\xe2L\xf7\xe1\x92' + b'\xfb\x8b\x8c\xeb\xbeh\x81#\xe0\xd8\x08D\x82\xda\xd3\xe9D\xf5' + b'\xcbK\x11\x0c\xc7\n\xf8\xbfG\x8f.:\x8a\xd1\x1d\x03/\xe7\xc4pf' + b'\xbe\x80\xe8\xa5\xd3ik\x90\xf0,' + b'<@\xbee\xefDt\x8bf\xb3\xbb\n\x8e\x08\x0b\x10\xe3:[' + b'\x08%?M\xa9$\xbeK\xc3\x01\xa1\xab\xad=\xdb\x8a%qO\xc3\xa2' + b')\x110\x87\x0e\x00\x98 ' + b'DQpYdm\x0f\x84\xac\x9ft\xdb\xear\xd1\xb0py\xafe\xcf\x89aX\x05' + b'\xc9\xef&:e.k\xe7\xdaZ\xda\xca\x84\xde\xf3}\r\xcf\xe6C)9\x13' + b'\xe6\xfb\xdb5\\x\x0f\x83\xc4]^vb%Z%C\xfdG\xa7\x17\xca0\x12\xa6' + b'\xabd{zQ\xf3\xe2\xe8\x92\xec\x88)A/\xc3tD\xe9,' + b'oH\x90N\xde\xef\xb2 ' + b'\xe7\xc6\xcb\xcd\xd9\xdf\x1d\xc1L\xd7\xaa\x1c;-\xa5\xc6m\xd8#s' + b'\x9dg|\xc6\xd1v\x9f\xcaNDRj\xb2\xdcJ\x86\xd3j\xb6u\xbd\xb83' + b'\xfbt\x0f(\xe5,\xd8\xed\xc6\xd4ZO\xba\x07\x1f\xf1\x14[' + b'\xf3cZ]\xfd\xe2D\\\x19t\x01M\xe3\xf8\x9b\xc3\xf3\x19=T\xfa3' + b'\x9cN\xdf\xc2+\x1d\x80\xdf4\xfcl9\xecD8\x11\x9d\x0b\x92}t\xd4' + b'\xc3\r\x04\xadM^\xe0\x11\x12\xd9x ' + b'rd\x17\xc34<\x89\xaf\x96\xb4\xeaa*\xaeO\x95\x04o\x18P\x02t\x1a' + b'\x9e\xc4\xcb@\x9bp:\xed\xc2\xc5J\tD\xa0\xb2\xfe\xb6\x0f\x86[' + b'\xa4\x13\xf1$v\xd4\xb4\xf5\xc1DB\x16\x8b\xe1\x18\xean\x08W' + b'\x827\xf5[\x1cS\xf1$\x0e\xc8p"\x9e\xc4\x01\x19\xfa\xd3P\x87r' + b'%\xd8|\x14\xff\x92)8\xa2\x1a\x1bf\xa8\x00\xee\xd7\x90E\r\xb6C' + b'\x99\xdd9\x00\xb4\x9b\xa6\xfc\xd5\t\x92\x14\xb5\xdf\x12"P\x97' + b'%\t\xdd\x11Df\xd8\xb0\xa9q\xaa\x14\x80hu\xd60\xfaE\xa6\xd6\xd6' + b'\xd6\xdb+\xdc\x9dp\xd4i\xdb\xc4\x9f6\x1b\xfe;de\x93\n)b\x9d' + b'=\x99\x9f\x91_ \x84 ' + b'd"\x02I\xbe\xa8l\x95\xbe\x9bq\xea\xf2\xd88\x14<\x00\'\xd7\xc3' + b'!\\.\x7f\xe2\x87\x1db\te\xbaw\x80\xea;G\x15\x0ca\xc5\n\xf3]Mu' + b'\xf2F\xdfA\xaaH.5\xd9q\xcc\x1f\xd1\x98v3\xf4]\xd2\x97\xe7\xec5' + b'\xb6L\xb6n\xca\xa6Q}\xe7;^d\xfd\x10\xf0\xa5\xf1\xcf?;U\xbbQ' + b'\xdfV\xbc\x99\x06\x18\xcd\x7f\xb6\xae\xa0Ho\xe8\x8b\xd2l)\xf9k' + b'\x93\x03\xdeM\xaeIY\x1d\x1a#\xde\xde\xe1\xda~\x8c\x8b\x03%\x1c' + b'\xc7,\x94\x15\xc62Pa\xa9N(\x9a\xc8@\x18\xf4\x17\xab\x13\xcb[' + b'\xf8\x80\x02\x8e\x93\xca\x90ba\xd9w?\xca\xf7\xea\x1e\r\x0b_' + b'\xb2\xff\xeac\x12\xa7\xe1\xe7oE|\xc3}8\x9a\xd3p\x83\xb7\xc3Kz' + b'\xc8\xd5\xa9\x84\xa3:p\x12\xa9\x92\xaeA\xbbc\x8a\x0f\xb6/m' + b'\xadN\xa6\xfe\xa2\x0b\xbf\xd2\x14\xcdI%\x825\xc1\xf6fWY\x8a' + b'\xff\x8e\xeb\xdc\x7f\xa9\x1foeO\xc7d2U4-\xf0\x01\xc3\x0fw\xfb' + b'\xa4:t\xbb\x889\x90\xb5\xf7\x82#oS\x8c\x0f\xc0J7\x81N\xb8Y' + b'\xfd(\xb9\n\x96\x9e\xbe\xa0\xad\x8b\x047\x8a\xd9\x03\xf9<\x152' + b'\xf5uZ\xd4-\xd0\x05\x7f\x15b\rS\x89\xf07\xe2\x9e\x9b4\xf8>kn' + b'\xb7\x07\xd0\x9a\x04?z\xcf!1\x94U\xc3\xa9u\xd2\x940\x06h\xb5' + b'\xf3\x83#i6`U\xdf9\xa1\x16\x8f\x16\x05\xd0\x94\x03`\x06\xa7L' + b'\xab[^\x94\x18M\x17\x1c\xeai\xca\xc3\x89G\xc8\x91\xb8\xcc\xb4F' + b'\xa8&\xe8\xe7\x97T\xb2(' + b'*it?\x10\x9e\xb4H\xec1\xe6\x10p-D\xe0\\\x08\xcb\xfd\x81&\x92' + b']\xcb\xc0\xb3\xbf\x17Z|\\\x1a\xc9 ' + b'\x98\xe1\xc4*\x04\x98\xec9\xea\x95aD\x16\x0c\xb3\x869n7\\\xde' + b'>\\\xe7zC\xc3\xcd\xa8\xf4\xc3H\x19~\xecC\xca\\{' + b'\xee\x1e\x1f\x16\xf7:\xc7\x89n\xc5J\xafK\xb2\xf8h\xc5\xd0\x18a' + b'\n\x11\\\xe2{\xcd\x8d\x85\xa6h\xbd]*\xb2\x05-\xea\xd5\xd1m=J7' + b'\x88n\xe3\xda<\xca\xde\xe58 ' + b'.\xd5I#^\xbd\xcd\x10j\xd3\xe3\xd6\x1d\xd1{' + b'|~\xe3\xa8\xaf\xcfY\x03\xceo\x99\x1b\xd8\xae%\xe3t\xad:\xe7F' + b'\xe1\xba\xbf\xd5\xe0=\xcc[\xb0\xc8\xae\xb6\xc5\x98 ' + b'\xfa\xb7\xbbF\x9b\xf2\x02{H\xcd\xe9\xe3|>\x0fn\xf1{' + b'\x05?\xde\xf1\xcd\x86\x9b\x98\xe6nyy\x9e<\x906\x1f\xb1[' + b')&\xcd\xd2\x89\x8a\xff\x91\xe0\xfa,' + b'a\x7f;\x1e\xe2\x1d\xb7\xfe\x0e\rb\xdeNO\xcf\xb9\x82{' + b'\x8a\xe4\xc0<\xde\xe2\x8ax\x7f\xeb\xa7\x8f\xe7\x02\x06\x8a' + b'\xb3O\x15\xe3\xf2\xf4H,' + b'C\x17\xe5\x83\xe7B\x84\xb2N\x185\xc3\xb3\x1av\xf0+b\x9f\x04b' + b'\x16\xd8\x9d\x02\xba\xc5\xe72X\xa7I\xf3\x87\x7f\xc7\xc8\xb2W' + b'\xb5I|.);\xa9\x08V\xf1\xbb\x0bO$\xdd\xe2\x9b\xc74\xca\xde\x8d' + b'\xd2\x0f\xce\xc1\x9ayU\x9f(^!\x8c\xf1_1Y?\x95~\xd2 ' + b'\xb3\xcc\xb2\x9e\xdbd\x1e\x9d\x8b\xddnj\xb2\xdf\x1ew\xe1\xe1l' + b'&\xcd\xd0\r\xcf\xebu~eR\x9cT\x82\xc1h.u\x0crv8\x95`\xebC\xf1' + b'\xa9\x90\x96\x9b\xec\x9dR\xda\xd10"\xaf\x1dh{' + b'\xcb\xcai\x0f\x08\x96\xbd9\xcd\xceJW\xf11:\xe7"\x8dH\xddp\xd3' + b'\x13e\n8YgY\xa8\xd4\xf5\xc8*\x03\xe0W\x0f\x10@\xc6E\xde&\xc9' + b')\x02\xb9\x84\xb9\xa9J\xf8\xd5\x8d2\x14u\n\xd9A5\xa6\xe9F\xb9' + b'}t\xc9\x00\xbf\x9f\xc7\xac8k"?\xceSY\x01\x8f\xee\x7f\xcb\xed' + b'\xea\xc2+\xb1_\xcf\xecO\xfdx\xec\xbb\xb0\x9c\xfc\x93\xe1\xf9' + b'"\xca\xd6\x85\x94\xc4\xea\x85\xcc!\x03f.F,' + b'fCV\xbc\xedK\xb10+\x00E^\xde@\x0c\xed\xc7=j\xa4f\xa7\xf9\xbc' + b'\xfe\xa9\xb8g\x08\xbd\xafS|\xecs\xcdt\x86\xf2Dm\xae\xedmF\x16' + b'\x8b\xdf\xc5\xd4\x07A\xf5z\xadL\xec\x00\nk\xed\x91\xe1\xafo' + b'\x82Li\xbd\x93\xb9\xb3\xcc%\x18MF Date: Sun, 12 Feb 2023 19:40:10 +0100 Subject: [PATCH 22/48] Added test_upload_fileobj --- tests/test_multipart_formdata_request.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index c19fb4477..7754a03f9 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -1,4 +1,5 @@ import base64 +import io import json import pytest @@ -690,3 +691,21 @@ def test_upload_image(client): 'secure_filename': filename.replace('/', '_'), } ] + + +def test_upload_fileobj(client): + fileobj = io.BytesIO(IMAGE_FILE) + fileobj.name = '/tests/img_readable' + + resp = client.simulate_post('/image', files={'image': fileobj}) + + assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': base64.b64encode(IMAGE_FILE).decode(), + 'filename': 'img_readable', + 'name': 'image', + 'secure_filename': 'img_readable', + } + ] From 5c6886805225c8aaf25b5e3965e5942b03c20f5a Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Mon, 13 Feb 2023 01:15:47 +0100 Subject: [PATCH 23/48] typo fom-data --- falcon/testing/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index d859bbd85..7b899b375 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2267,7 +2267,8 @@ def _encode_files(files, data=None): for f, val in data_fields: body_string += ( f'Content-Disposition: ' - f'{content_disposition or "fom-data"}; name={f}; \r\n\r\n'.encode() + f'{content_disposition or "form-data"}; name={f}; ' + f'\r\n\r\n'.encode() ) body_string += val.encode('utf-8') if isinstance(val, str) else val body_string += b'\r\n--' + boundary.encode() + b'\r\n' From f052dcac5fc33c16b2e27a4a0113210df65873aa Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 19 Feb 2023 01:37:57 +0100 Subject: [PATCH 24/48] removed conditional where object type bytes is not possible --- falcon/testing/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 7b899b375..227a56423 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2181,10 +2181,8 @@ def _prepare_data_fields(data): val = [val] for v in val: if v: - # Don't call str() on bytestrings: in Py3 it all goes wrong. - if not isinstance(v, bytes): - v = str(v) - + # v has to come from json serializable obj, so can't be bytes + v = str(v) new_fields.append( ( field.decode('utf-8') if isinstance(field, bytes) else field, From b527de7b258cef6ae26f286710a1b61b77da6cf9 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 19 Feb 2023 01:55:01 +0100 Subject: [PATCH 25/48] added fct testing null data value --- falcon/testing/client.py | 12 ++++++--- tests/test_multipart_formdata_request.py | 32 +++++++++++++++++++----- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 227a56423..95109929c 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2169,12 +2169,13 @@ def _prepare_data_fields(data): """ fields = [] new_fields = [] - if data and not isinstance(data, (list, dict)): + if not isinstance(data, (list, dict)): raise ValueError('Data must not be a list of tuples or dict.') - elif data and isinstance(data, dict): + elif isinstance(data, dict): fields = list(data.items()) - elif data: + else: fields = list(data) + # Append data to the other multipart parts for field, val in fields: if isinstance(val, str) or not hasattr(val, '__iter__'): @@ -2189,6 +2190,11 @@ def _prepare_data_fields(data): v.encode('utf-8') if isinstance(v, str) else v, ) ) + else: + new_fields.append( + (field.decode('utf-8') if isinstance(field, bytes) else field, b'') + ) + return new_fields diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 7754a03f9..1a4d1c6a5 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -305,6 +305,14 @@ def asserts_data_types(resp): 'secure_filename': None, 'text': 'bonjour', }, + { + 'content_type': 'text/plain', + 'data': '', + 'filename': None, + 'name': 'empty', + 'secure_filename': None, + 'text': '', + }, { 'content_type': 'text/plain', 'data': 'world', @@ -341,14 +349,18 @@ def asserts_data_types(resp): def test_upload_multipart_datalist(client): resp = client.simulate_post( - '/submit', files=FILES1, json=[('data1', 5), ('data2', ['hello', 'bonjour'])] + '/submit', + files=FILES1, + json=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)], ) asserts_data_types(resp) def test_upload_multipart_datalisttuple(client): resp = client.simulate_post( - '/submit', files=FILES1, json=[('data1', 5), ('data2', ('hello', 'bonjour'))] + '/submit', + files=FILES1, + json=[('data1', 5), ('data2', ('hello', 'bonjour')), ('empty', None)], ) asserts_data_types(resp) @@ -356,7 +368,9 @@ def test_upload_multipart_datalisttuple(client): def test_upload_multipart_datalistdict(client): """json data list with dict""" resp = client.simulate_post( - '/submit', files=FILES1, json=[('data1', 5), ('data2', {'hello', 'bonjour'})] + '/submit', + files=FILES1, + json=[('data1', 5), ('data2', {'hello', 'bonjour'}), ('empty', None)], ) asserts_data_types(resp) @@ -364,7 +378,9 @@ def test_upload_multipart_datalistdict(client): def test_upload_multipart_datadict(client): """json data dict with list""" resp = client.simulate_post( - '/submit', files=FILES1, json={'data1': 5, 'data2': ['hello', 'bonjour']} + '/submit', + files=FILES1, + json={'data1': 5, 'data2': ['hello', 'bonjour'], 'empty': None}, ) asserts_data_types(resp) @@ -372,7 +388,9 @@ def test_upload_multipart_datadict(client): def test_upload_multipart_datadicttuple(client): """json data dict with tuple""" resp = client.simulate_post( - '/submit', files=FILES1, json={'data1': 5, 'data2': ('hello', 'bonjour')} + '/submit', + files=FILES1, + json={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, ) asserts_data_types(resp) @@ -380,7 +398,9 @@ def test_upload_multipart_datadicttuple(client): def test_upload_multipart_datadictdict(client): """json data dict with dict""" resp = client.simulate_post( - '/submit', files=FILES1, json={'data1': 5, 'data2': {'hello', 'bonjour'}} + '/submit', + files=FILES1, + json={'data1': 5, 'data2': {'hello', 'bonjour'}, 'empty': None}, ) asserts_data_types(resp) From 1465241accfe4d974f8f8c4a9d0eb5ffddb865cf Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 19 Feb 2023 02:12:05 +0100 Subject: [PATCH 26/48] another unneccesay if removed --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 95109929c..8cb127cec 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2216,7 +2216,7 @@ def _prepare_files(k, v): if isinstance(v, (tuple, list)): if len(v) == 2: file_name, file_data = v - elif len(v) == 3: + else: file_name, file_data, file_content_type = v if ( len(v) == 3 From e4cb4c9c727e612261bdf200e86f37b25149b529 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 12:55:54 +0100 Subject: [PATCH 27/48] Update falcon/testing/client.py ) in docstring Co-authored-by: Federico Caselli --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 8cb127cec..306f1872b 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2164,7 +2164,7 @@ def _prepare_data_fields(data): Args: data: dict or list of tuples with json data from the request - Returns: list of 2-tuples (field-name(str), value(bytes) + Returns: list of 2-tuples (field-name(str), value(bytes)) """ fields = [] From dba5eb9edc7eacdaf9c77cea11e61446af4da145 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 12:59:11 +0100 Subject: [PATCH 28/48] Update falcon/testing/client.py wrong docstring in _prepare_data_fields Co-authored-by: Federico Caselli --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 306f1872b..865043bee 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2205,7 +2205,7 @@ def _prepare_files(k, v): k: (str), file-name v: fileobj or tuple (filename, data, content_type?, headers?) - Returns: file_name, file_data, file_content_type, file_header + Returns: file_name, file_data, file_content_type """ file_content_type = None From 51ea53ee605298754c2f202178c80da509439711 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 13:00:20 +0100 Subject: [PATCH 29/48] Update falcon/testing/client.py Co-authored-by: Federico Caselli --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 865043bee..5280579b3 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2290,7 +2290,7 @@ def _encode_files(files, data=None): body_string += f'Content-Disposition: form-data; name={k}; '.encode() body_string += ( - f'filename={file_name or "null"}\r\n'.encode() + f'filename={file_name}\r\n'.encode() if file_name else '\r\n'.encode() ) From 8a663468ba7ab8692dc19ba603d3e4f152f494b8 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 12:42:30 +0100 Subject: [PATCH 30/48] typo --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 5280579b3..d5cc76949 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2170,7 +2170,7 @@ def _prepare_data_fields(data): fields = [] new_fields = [] if not isinstance(data, (list, dict)): - raise ValueError('Data must not be a list of tuples or dict.') + raise ValueError('Data must be a list of tuples or dict.') elif isinstance(data, dict): fields = list(data.items()) else: From 30415832021d3243e0be8aa7266a1494feac4a75 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 12:57:19 +0100 Subject: [PATCH 31/48] check for datatype in json changed --- falcon/testing/client.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index d5cc76949..fba6c326e 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2167,11 +2167,8 @@ def _prepare_data_fields(data): Returns: list of 2-tuples (field-name(str), value(bytes)) """ - fields = [] new_fields = [] - if not isinstance(data, (list, dict)): - raise ValueError('Data must be a list of tuples or dict.') - elif isinstance(data, dict): + if isinstance(data, dict): fields = list(data.items()) else: fields = list(data) @@ -2242,7 +2239,7 @@ def _prepare_files(k, v): def _make_boundary(): """ - Create random boundary to be used in multipar/form-data with files. + Create random boundary to be used in multipart/form-data with files. """ boundary = binascii.hexlify(os.urandom(16)) boundary = boundary.decode('ascii') From 77f350b6ed6e18ef8bb35224498b49642287bfc5 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 13:41:50 +0100 Subject: [PATCH 32/48] Corrections after code-review (part1) --- falcon/testing/client.py | 31 ++++++------------------ tests/test_multipart_formdata_request.py | 27 +-------------------- 2 files changed, 9 insertions(+), 49 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index fba6c326e..b7cf835ac 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -534,12 +534,10 @@ def simulate_request( files(dict): same as the files parameter in requests, dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. - ``file-tuple``: can be a 2-tuple ``('filename', fileobj)``, + ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a 3-tuple ``('filename', fileobj, 'content_type')`` - or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string defining the content - type of the given file and ``custom_headers`` a dict-like - object containing additional headers to add for the file. + type of the given file. file_wrapper (callable): Callable that returns an iterable, to be used as the value for *wsgi.file_wrapper* in the WSGI environ (default: ``None``). This can be used to test @@ -754,12 +752,10 @@ async def _simulate_request_asgi( files(dict): same as the files parameter in requests, dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. - ``file-tuple``: can be a 2-tuple ``('filename', fileobj)``, - 3-tuple ``('filename', fileobj, 'content_type')`` - or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, + ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a + 3-tuple ``('filename', fileobj, 'content_type')``, where ``'content-type'`` is a string defining the content - type of the given file and ``custom_headers`` a dict-like - object containing additional headers to add for the file. + type of the given file. host(str): A string to use for the hostname part of the fully qualified request URL (default: 'falconframework.org') remote_addr (str): A string to use as the remote IP address for the @@ -2187,10 +2183,6 @@ def _prepare_data_fields(data): v.encode('utf-8') if isinstance(v, str) else v, ) ) - else: - new_fields.append( - (field.decode('utf-8') if isinstance(field, bytes) else field, b'') - ) return new_fields @@ -2200,14 +2192,12 @@ def _prepare_files(k, v): Args: k: (str), file-name - v: fileobj or tuple (filename, data, content_type?, headers?) + v: fileobj or tuple (filename, data, content_type?) Returns: file_name, file_data, file_content_type """ file_content_type = None - file_data = None - file_name = None if not v: raise ValueError(f'No file provided for {k}') if isinstance(v, (tuple, list)): @@ -2241,8 +2231,7 @@ def _make_boundary(): """ Create random boundary to be used in multipart/form-data with files. """ - boundary = binascii.hexlify(os.urandom(16)) - boundary = boundary.decode('ascii') + boundary = os.urandom(16).hex() return boundary @@ -2257,8 +2246,6 @@ def _encode_files(files, data=None): Returns: (encoded body string, headers dict) """ - - content_disposition = None boundary = _make_boundary() body_string = b'--' + boundary.encode() + b'\r\n' header = {'Content-Type': 'multipart/form-data; boundary=' + boundary} @@ -2267,9 +2254,7 @@ def _encode_files(files, data=None): data_fields = _prepare_data_fields(data) for f, val in data_fields: body_string += ( - f'Content-Disposition: ' - f'{content_disposition or "form-data"}; name={f}; ' - f'\r\n\r\n'.encode() + f'Content-Disposition: form-data; name={f}; ' f'\r\n\r\n'.encode() ) body_string += val.encode('utf-8') if isinstance(val, str) else val body_string += b'\r\n--' + boundary.encode() + b'\r\n' diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 1a4d1c6a5..5fe81ac14 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -305,14 +305,6 @@ def asserts_data_types(resp): 'secure_filename': None, 'text': 'bonjour', }, - { - 'content_type': 'text/plain', - 'data': '', - 'filename': None, - 'name': 'empty', - 'secure_filename': None, - 'text': '', - }, { 'content_type': 'text/plain', 'data': 'world', @@ -353,6 +345,7 @@ def test_upload_multipart_datalist(client): files=FILES1, json=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)], ) + print(resp.json) asserts_data_types(resp) @@ -421,24 +414,6 @@ def test_invalid_files_null(client): client.simulate_post('/submit', files={'file': ()}) -def test_invalid_dataint(client): - """invalid data type in json, int""" - with pytest.raises(ValueError): - client.simulate_post('/submit', files=FILES1, json=5) - - -def test_invalid_datastr(client): - """invalid data type in json, str""" - with pytest.raises(ValueError): - client.simulate_post('/submit', files=FILES1, json='yo') - - -def test_invalid_databyte(client): - """invalid data type in json, b''""" - with pytest.raises(ValueError): - client.simulate_post('/submit', files=FILES1, json=b'yo self') - - # endregion # region - TEST NESTED FILES UPLOAD From 4bc299ad47b80d91984b1d21298a495f82ec50f1 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 13:48:00 +0100 Subject: [PATCH 33/48] blued --- falcon/testing/client.py | 4 +--- tests/test_multipart_formdata_request.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index b7cf835ac..01a580be1 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2272,9 +2272,7 @@ def _encode_files(files, data=None): body_string += f'Content-Disposition: form-data; name={k}; '.encode() body_string += ( - f'filename={file_name}\r\n'.encode() - if file_name - else '\r\n'.encode() + f'filename={file_name}\r\n'.encode() if file_name else '\r\n'.encode() ) body_string += ( f'Content-Type: {file_content_type or "text/plain"}\r\n\r\n'.encode() diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 5fe81ac14..6cfbd3f7c 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -345,7 +345,6 @@ def test_upload_multipart_datalist(client): files=FILES1, json=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)], ) - print(resp.json) asserts_data_types(resp) From df37dbd84fbc533f39ba1f6d84a877643ccb0a14 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 5 Mar 2023 14:28:26 +0100 Subject: [PATCH 34/48] unnecessary import removed --- falcon/testing/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 01a580be1..1cdd56ead 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -19,7 +19,6 @@ """ import asyncio -import binascii import datetime as dt import inspect import json as json_module From 2149b0cdae577f7aaf4b6185bf8f8d5675fe0516 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sat, 11 Mar 2023 17:05:02 +0100 Subject: [PATCH 35/48] Handling of new data parameter; TODO: more tests --- falcon/testing/client.py | 96 +++++++++++++++++------- tests/test_multipart_formdata_request.py | 52 +++++++++++-- 2 files changed, 111 insertions(+), 37 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 1cdd56ead..86282f794 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -29,12 +29,13 @@ from typing import Sequence from typing import Union import warnings +from urllib.parse import urlencode import wsgiref.validate from falcon.asgi_spec import ScopeType from falcon.constants import COMBINED_METHODS -from falcon.constants import MEDIA_JSON -from falcon.errors import CompatibilityError +from falcon.constants import MEDIA_JSON, MEDIA_URLENCODED +from falcon.errors import CompatibilityError, HTTPBadRequest from falcon.testing import helpers from falcon.testing.srmock import StartResponseMock from falcon.util import async_to_sync @@ -96,7 +97,7 @@ class Cookie: or ``None`` if not specified. max_age (int): The lifetime of the cookie in seconds, or ``None`` if not specified. - secure (bool): Whether or not the cookie may only only be + secure (bool): Whether or not the cookie may only be transmitted from the client via HTTPS. http_only (bool): Whether or not the cookie may only be included in unscripted requests from the client. @@ -439,6 +440,7 @@ def simulate_request( body=None, json=None, files=None, + data=None, file_wrapper=None, wsgierrors=None, params=None, @@ -530,6 +532,8 @@ def simulate_request( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. + Can only be used if data and files are null, otherwise an exception + is thrown. files(dict): same as the files parameter in requests, dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. @@ -537,6 +541,11 @@ def simulate_request( 3-tuple ``('filename', fileobj, 'content_type')`` where ``'content-type'`` is a string defining the content type of the given file. + If both files and json are present, an exception is thrown. To pass + additional form-data with files, use data. + data : list of tuples or dict with additional data to be passed with + files (or alone if files is null), to be treated as urlencoded form data. + If both data and json are present, an exception is thrown. file_wrapper (callable): Callable that returns an iterable, to be used as the value for *wsgi.file_wrapper* in the WSGI environ (default: ``None``). This can be used to test @@ -585,6 +594,7 @@ def simulate_request( body=body, json=json, files=files, + data=data, params=params, params_csv=params_csv, protocol=protocol, @@ -609,6 +619,7 @@ def simulate_request( body, json, files, + data, extras, ) @@ -633,7 +644,7 @@ def simulate_request( # NOTE(vytas): Even given the duct tape nature of overriding # arbitrary environ variables, changing the method can potentially # be very confusing, particularly when using specialized - # simulate_get/post/patch etc methods. + # simulate_get/post/patch etc. methods. raise ValueError( 'WSGI environ extras may not override the request method. ' 'Please use the method parameter.' @@ -663,6 +674,7 @@ async def _simulate_request_asgi( body=None, json=None, files=None, + data=None, params=None, params_csv=True, protocol='http', @@ -748,6 +760,8 @@ async def _simulate_request_asgi( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. + Can only be use if files and data are null, otherwise an exception + is thrown. files(dict): same as the files parameter in requests, dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. @@ -755,6 +769,11 @@ async def _simulate_request_asgi( 3-tuple ``('filename', fileobj, 'content_type')``, where ``'content-type'`` is a string defining the content type of the given file. + If both files and json are present, an exception is thrown. To pass + additional form-data with files, use data. + data : list of tuples or dict with additional data to be passed with + files (or alone if files is null), to be treated as urlencoded form data. + If both data and json are present, an exception is thrown. host(str): A string to use for the hostname part of the fully qualified request URL (default: 'falconframework.org') remote_addr (str): A string to use as the remote IP address for the @@ -794,6 +813,7 @@ async def _simulate_request_asgi( body, json, files, + data, extras, ) @@ -2153,7 +2173,7 @@ async def __aexit__(self, exc_type, exc, tb): await self._task_req -def _prepare_data_fields(data): +def _prepare_data_fields(data, boundary=None, urlenc=False): """Prepare data fields for request body. Args: @@ -2162,8 +2182,13 @@ def _prepare_data_fields(data): Returns: list of 2-tuples (field-name(str), value(bytes)) """ - new_fields = [] - if isinstance(data, dict): + urlresult = [] + body_part = b'' + if isinstance(data, (str, bytes)): + fields = list(json.loads(data).items()) + elif hasattr(data, "read"): + fields = list(json.load(data).items()) + elif isinstance(data, dict): fields = list(data.items()) else: fields = list(data) @@ -2172,18 +2197,28 @@ def _prepare_data_fields(data): for field, val in fields: if isinstance(val, str) or not hasattr(val, '__iter__'): val = [val] - for v in val: - if v: - # v has to come from json serializable obj, so can't be bytes - v = str(v) - new_fields.append( - ( - field.decode('utf-8') if isinstance(field, bytes) else field, - v.encode('utf-8') if isinstance(v, str) else v, + # if no files are passed, make urlencoded form + if urlenc: + for v in val: + if v: + urlresult.append( + ( + field.encode("utf-8") if isinstance(field, str) else field, + v.encode("utf-8") if isinstance(v, str) else v, + ) ) - ) - - return new_fields + # if files and data are passed, concat data to files body like in requests + else: + for v in val: + body_part += f'Content-Disposition: form-data; name={field}; ' f'\r\n\r\n'.encode() + if v: + if not isinstance(v, bytes): + v = str(v) + body_part += v.encode('utf-8') if isinstance(v, str) else v + body_part += b'\r\n--' + boundary.encode() + b'\r\n' + else: + body_part += b'\r\n--' + boundary.encode() + b'\r\n' + return body_part if not urlenc else urlencode(urlresult, doseq=True) def _prepare_files(k, v): @@ -2248,15 +2283,6 @@ def _encode_files(files, data=None): boundary = _make_boundary() body_string = b'--' + boundary.encode() + b'\r\n' header = {'Content-Type': 'multipart/form-data; boundary=' + boundary} - # Handle whatever json data gets passed along with files - if data: - data_fields = _prepare_data_fields(data) - for f, val in data_fields: - body_string += ( - f'Content-Disposition: form-data; name={f}; ' f'\r\n\r\n'.encode() - ) - body_string += val.encode('utf-8') if isinstance(val, str) else val - body_string += b'\r\n--' + boundary.encode() + b'\r\n' # Deal with the files tuples if not isinstance(files, (dict, list)): @@ -2281,6 +2307,10 @@ def _encode_files(files, data=None): ) body_string += b'\r\n--' + boundary.encode() + b'\r\n' + # Handle whatever json data gets passed along with files + if data: + body_string += _prepare_data_fields(data, boundary) + body_string = body_string[:-2] + b'--\r\n' return body_string, header @@ -2296,6 +2326,7 @@ def _prepare_sim_args( body, json, files, + data, extras, ): if not path.startswith('/'): @@ -2325,8 +2356,15 @@ def _prepare_sim_args( headers = headers or {} headers['Content-Type'] = content_type - if files is not None: - body, headers = _encode_files(files, json) + if files or data: + if json: + raise HTTPBadRequest(description="Cannot process both json and (files or data) args") + elif files: + body, headers = _encode_files(files, data) + else: + body = _prepare_data_fields(data, None, True) + headers = headers or {} + headers["Content-Type"] = MEDIA_URLENCODED elif json is not None: body = json_module.dumps(json, ensure_ascii=False) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 6cfbd3f7c..a39247ad5 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -8,6 +8,8 @@ from falcon import media from falcon import testing +from falcon.errors import HTTPBadRequest + from _util import create_app # NOQA: I100 @@ -43,6 +45,11 @@ def on_post(req, resp): resp.media = values + @staticmethod + def on_post_data(req, resp): + deserialized = req.get_media() + resp.media = deserialized + @staticmethod def on_post_media(req, resp): deserialized = [] @@ -97,6 +104,11 @@ async def on_post(req, resp): ) resp.media = values + @staticmethod + async def on_post_data(req, resp): + data = await req.get_media() + resp.media = data + @staticmethod async def on_post_media(req, resp): deserialized = [] @@ -141,7 +153,8 @@ def client(asgi): app.req_options.media_handlers = media.Handlers( { falcon.MEDIA_JSON: media.JSONHandler(), - falcon.MEDIA_MULTIPART: parser, # media.MultipartFormHandler() + falcon.MEDIA_URLENCODED: media.URLEncodedFormHandler(), + falcon.MEDIA_MULTIPART: parser, } ) @@ -150,6 +163,7 @@ def client(asgi): app.add_route('/submit', resource) app.add_route('/media', resource, suffix='media') + app.add_route('/data', resource, suffix='data') app.add_route('/image', resource, suffix='image') return testing.TestClient(app) @@ -260,13 +274,19 @@ def test_upload_multipart_media(client): assert resp.json == [ {'count': 6, 'numbers': [1, 2, 6, 24, 120, 720]}, { - 'fruit': b'\xF0\x9F\x8D\x8F'.decode('utf8'), # u"\U0001F34F", + 'fruit': b'\xF0\x9F\x8D\x8F'.decode('utf8'), 'name': 'Jane', 'surname': 'Doe', }, ] +def test_upload_only_data(client): + resp = client.simulate_post('/data', data=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)]) + assert resp.status_code == 200 + assert resp.json == {"data1": "5", "data2": ["hello", "bonjour"]} + + # endregion # region - TEST DIFFERENT DATA TYPES in json part @@ -305,6 +325,14 @@ def asserts_data_types(resp): 'secure_filename': None, 'text': 'bonjour', }, + { + 'content_type': 'text/plain', + 'data': '', + 'filename': None, + 'name': 'empty', + 'secure_filename': None, + 'text': '', + }, { 'content_type': 'text/plain', 'data': 'world', @@ -343,7 +371,7 @@ def test_upload_multipart_datalist(client): resp = client.simulate_post( '/submit', files=FILES1, - json=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)], + data=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)], ) asserts_data_types(resp) @@ -352,7 +380,7 @@ def test_upload_multipart_datalisttuple(client): resp = client.simulate_post( '/submit', files=FILES1, - json=[('data1', 5), ('data2', ('hello', 'bonjour')), ('empty', None)], + data=[('data1', 5), ('data2', ('hello', 'bonjour')), ('empty', None)], ) asserts_data_types(resp) @@ -362,7 +390,7 @@ def test_upload_multipart_datalistdict(client): resp = client.simulate_post( '/submit', files=FILES1, - json=[('data1', 5), ('data2', {'hello', 'bonjour'}), ('empty', None)], + data=[('data1', 5), ('data2', {'hello', 'bonjour'}), ('empty', None)], ) asserts_data_types(resp) @@ -372,7 +400,7 @@ def test_upload_multipart_datadict(client): resp = client.simulate_post( '/submit', files=FILES1, - json={'data1': 5, 'data2': ['hello', 'bonjour'], 'empty': None}, + data={'data1': 5, 'data2': ['hello', 'bonjour'], 'empty': None}, ) asserts_data_types(resp) @@ -382,7 +410,7 @@ def test_upload_multipart_datadicttuple(client): resp = client.simulate_post( '/submit', files=FILES1, - json={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, + data={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, ) asserts_data_types(resp) @@ -392,7 +420,7 @@ def test_upload_multipart_datadictdict(client): resp = client.simulate_post( '/submit', files=FILES1, - json={'data1': 5, 'data2': {'hello', 'bonjour'}, 'empty': None}, + data={'data1': 5, 'data2': {'hello', 'bonjour'}, 'empty': None}, ) asserts_data_types(resp) @@ -413,6 +441,14 @@ def test_invalid_files_null(client): client.simulate_post('/submit', files={'file': ()}) +def test_invalid_files_data_json(client): + """empty json and data and files""" + with pytest.raises(HTTPBadRequest): + client.simulate_post('/submit', files=FILES1, + data={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, + json={'badrequest': 'should fail'}) + + # endregion # region - TEST NESTED FILES UPLOAD From 6f8b3ae6f64fcfa54130c9b78ece2d0d61bf6211 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sat, 11 Mar 2023 17:05:34 +0100 Subject: [PATCH 36/48] blued --- falcon/testing/client.py | 19 ++++++++++++------- tests/test_multipart_formdata_request.py | 15 ++++++++++----- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 86282f794..b82f9b6e6 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2186,7 +2186,7 @@ def _prepare_data_fields(data, boundary=None, urlenc=False): body_part = b'' if isinstance(data, (str, bytes)): fields = list(json.loads(data).items()) - elif hasattr(data, "read"): + elif hasattr(data, 'read'): fields = list(json.load(data).items()) elif isinstance(data, dict): fields = list(data.items()) @@ -2203,14 +2203,17 @@ def _prepare_data_fields(data, boundary=None, urlenc=False): if v: urlresult.append( ( - field.encode("utf-8") if isinstance(field, str) else field, - v.encode("utf-8") if isinstance(v, str) else v, + field.encode('utf-8') if isinstance(field, str) else field, + v.encode('utf-8') if isinstance(v, str) else v, ) ) # if files and data are passed, concat data to files body like in requests else: for v in val: - body_part += f'Content-Disposition: form-data; name={field}; ' f'\r\n\r\n'.encode() + body_part += ( + f'Content-Disposition: form-data; name={field}; ' + f'\r\n\r\n'.encode() + ) if v: if not isinstance(v, bytes): v = str(v) @@ -2218,7 +2221,7 @@ def _prepare_data_fields(data, boundary=None, urlenc=False): body_part += b'\r\n--' + boundary.encode() + b'\r\n' else: body_part += b'\r\n--' + boundary.encode() + b'\r\n' - return body_part if not urlenc else urlencode(urlresult, doseq=True) + return body_part if not urlenc else urlencode(urlresult, doseq=True) def _prepare_files(k, v): @@ -2358,13 +2361,15 @@ def _prepare_sim_args( if files or data: if json: - raise HTTPBadRequest(description="Cannot process both json and (files or data) args") + raise HTTPBadRequest( + description='Cannot process both json and (files or data) args' + ) elif files: body, headers = _encode_files(files, data) else: body = _prepare_data_fields(data, None, True) headers = headers or {} - headers["Content-Type"] = MEDIA_URLENCODED + headers['Content-Type'] = MEDIA_URLENCODED elif json is not None: body = json_module.dumps(json, ensure_ascii=False) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index a39247ad5..8abe0f8f9 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -282,9 +282,11 @@ def test_upload_multipart_media(client): def test_upload_only_data(client): - resp = client.simulate_post('/data', data=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)]) + resp = client.simulate_post( + '/data', data=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)] + ) assert resp.status_code == 200 - assert resp.json == {"data1": "5", "data2": ["hello", "bonjour"]} + assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} # endregion @@ -444,9 +446,12 @@ def test_invalid_files_null(client): def test_invalid_files_data_json(client): """empty json and data and files""" with pytest.raises(HTTPBadRequest): - client.simulate_post('/submit', files=FILES1, - data={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, - json={'badrequest': 'should fail'}) + client.simulate_post( + '/submit', + files=FILES1, + data={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, + json={'badrequest': 'should fail'}, + ) # endregion From 632317c4e007a4b6f80b81527a675c1aebb6e65d Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sat, 11 Mar 2023 17:19:23 +0100 Subject: [PATCH 37/48] utf-8 corrected --- falcon/testing/client.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index b82f9b6e6..b8669acb2 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -28,8 +28,8 @@ from typing import Optional from typing import Sequence from typing import Union -import warnings from urllib.parse import urlencode +import warnings import wsgiref.validate from falcon.asgi_spec import ScopeType @@ -532,15 +532,18 @@ def simulate_request( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. - Can only be used if data and files are null, otherwise an exception - is thrown. + + Note: + Can only be used if data and files are null, otherwise an exception + is thrown. + files(dict): same as the files parameter in requests, - dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) - for multipart encoding upload. + dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) + for multipart encoding upload. ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a - 3-tuple ``('filename', fileobj, 'content_type')`` - where ``'content-type'`` is a string defining the content - type of the given file. + 3-tuple ``('filename', fileobj, 'content_type')`` + where ``'content-type'`` is a string defining the content + type of the given file. If both files and json are present, an exception is thrown. To pass additional form-data with files, use data. data : list of tuples or dict with additional data to be passed with @@ -760,15 +763,18 @@ async def _simulate_request_asgi( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. - Can only be use if files and data are null, otherwise an exception - is thrown. + + Note: + Can only be used if data and files are null, otherwise an exception + is thrown. + files(dict): same as the files parameter in requests, - dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) - for multipart encoding upload. + dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) + for multipart encoding upload. ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a - 3-tuple ``('filename', fileobj, 'content_type')``, - where ``'content-type'`` is a string defining the content - type of the given file. + 3-tuple ``('filename', fileobj, 'content_type')``, + where ``'content-type'`` is a string defining the content + type of the given file. If both files and json are present, an exception is thrown. To pass additional form-data with files, use data. data : list of tuples or dict with additional data to be passed with @@ -2185,9 +2191,9 @@ def _prepare_data_fields(data, boundary=None, urlenc=False): urlresult = [] body_part = b'' if isinstance(data, (str, bytes)): - fields = list(json.loads(data).items()) + fields = list(json_module.loads(data).items()) elif hasattr(data, 'read'): - fields = list(json.load(data).items()) + fields = list(json_module.load(data).items()) elif isinstance(data, dict): fields = list(data.items()) else: From 3b2edd8650edea62b2216cffb9131fdfa4048b34 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sat, 11 Mar 2023 17:56:47 +0100 Subject: [PATCH 38/48] more tests --- falcon/testing/client.py | 4 +--- tests/test_multipart_formdata_request.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index b8669acb2..9602a4d45 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2190,10 +2190,8 @@ def _prepare_data_fields(data, boundary=None, urlenc=False): """ urlresult = [] body_part = b'' - if isinstance(data, (str, bytes)): + if isinstance(data, (str, bytes)) or hasattr(data, 'read'): fields = list(json_module.loads(data).items()) - elif hasattr(data, 'read'): - fields = list(json_module.load(data).items()) elif isinstance(data, dict): fields = list(data.items()) else: diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 8abe0f8f9..ccde9e11d 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -289,7 +289,23 @@ def test_upload_only_data(client): assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} -# endregion +def test_upload_only_datadict(client): + resp = client.simulate_post( + '/data', data={'data1': 5, 'data2': ['hello', 'bonjour']} + ) + assert resp.status_code == 200 + assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} + + +def test_upload_only_data_str(client): + resp = client.simulate_post( + '/data', data=b'{"data1": 5, "data2": ["hello", "bonjour"]}' + ) + assert resp.status_code == 200 + assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} + + +# endregion{ # region - TEST DIFFERENT DATA TYPES in json part def asserts_data_types(resp): From bfa6a4c6f62b51b2c31b54e3ce17a6ab77c46261 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sat, 11 Mar 2023 18:05:36 +0100 Subject: [PATCH 39/48] test for string data --- tests/test_multipart_formdata_request.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index ccde9e11d..d1e3d07b5 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -297,7 +297,7 @@ def test_upload_only_datadict(client): assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} -def test_upload_only_data_str(client): +def test_upload_only_data_bstr(client): resp = client.simulate_post( '/data', data=b'{"data1": 5, "data2": ["hello", "bonjour"]}' ) @@ -305,6 +305,14 @@ def test_upload_only_data_str(client): assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} +def test_upload_only_data_str(client): + resp = client.simulate_post( + '/data', data='{"data1": 5, "data2": ["hello", "bonjour"]}' + ) + assert resp.status_code == 200 + assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} + + # endregion{ # region - TEST DIFFERENT DATA TYPES in json part From e1d983496069f40bcd99dd65a33d038e498cf884 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 12 Mar 2023 00:37:15 +0100 Subject: [PATCH 40/48] testing bool and float data types --- falcon/testing/client.py | 1 + tests/test_multipart_formdata_request.py | 96 ++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 9602a4d45..203162ece 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2225,6 +2225,7 @@ def _prepare_data_fields(data, boundary=None, urlenc=False): body_part += b'\r\n--' + boundary.encode() + b'\r\n' else: body_part += b'\r\n--' + boundary.encode() + b'\r\n' + return body_part if not urlenc else urlencode(urlresult, doseq=True) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index d1e3d07b5..6a7773020 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -313,8 +313,104 @@ def test_upload_only_data_str(client): assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} +def asserts_data_types_bool(resp): + assert resp.status_code == 200 + expected_list = [ + { + 'content_type': 'text/plain', + 'data': 'just some stuff', + 'filename': 'fileobj', + 'name': 'fileobj', + 'secure_filename': 'fileobj', + 'text': 'just some stuff', + }, + { + 'content_type': 'text/plain', + 'data': 'True', + 'filename': None, + 'name': 'data1', + 'secure_filename': None, + 'text': 'True', + }, + { + 'content_type': 'text/plain', + 'data': '3.14', + 'filename': None, + 'name': 'data3', + 'secure_filename': None, + 'text': '3.14', + }, + { + 'content_type': 'text/plain', + 'data': 'hello', + 'filename': None, + 'name': 'data2', + 'secure_filename': None, + 'text': 'hello', + }, + { + 'content_type': 'text/plain', + 'data': 'bonjour', + 'filename': None, + 'name': 'data2', + 'secure_filename': None, + 'text': 'bonjour', + }, + { + 'content_type': 'text/plain', + 'data': '', + 'filename': None, + 'name': 'empty', + 'secure_filename': None, + 'text': '', + }, + { + 'content_type': 'text/plain', + 'data': 'world', + 'filename': None, + 'name': 'hello', + 'secure_filename': None, + 'text': 'world', + }, + { + 'content_type': 'application/json', + 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', + 'filename': None, + 'name': 'document', + 'secure_filename': None, + 'text': None, + }, + { + 'content_type': 'text/plain', + 'data': 'Hello, world!', + 'filename': 'test.txt', + 'name': 'file1', + 'secure_filename': 'test.txt', + 'text': 'Hello, world!', + }, + ] + + # Result will be unordered, because both fileobj and data are present. + # When all files are tuples, response will be unordered if json + # contains dictionaries - then resp.json == expected_list can be used. + + assert len(resp.json) == len(expected_list) + assert all(map(lambda el: el in expected_list, resp.json)) + + +def test_upload_data_bool(client): + resp = client.simulate_post( + '/submit', + files=FILES1, + data=[('data1', True), ('data3', 3.14), ('data2', ['hello', 'bonjour']), ('empty', None)], + ) + print(resp.json) + asserts_data_types_bool(resp) + + # endregion{ + # region - TEST DIFFERENT DATA TYPES in json part def asserts_data_types(resp): assert resp.status_code == 200 From 514e9367b366162bbc299b8e094dc894ad23e96e Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 12 Mar 2023 00:45:30 +0100 Subject: [PATCH 41/48] blue & pep8 --- tests/test_multipart_formdata_request.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 6a7773020..10013d8ed 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -402,7 +402,12 @@ def test_upload_data_bool(client): resp = client.simulate_post( '/submit', files=FILES1, - data=[('data1', True), ('data3', 3.14), ('data2', ['hello', 'bonjour']), ('empty', None)], + data=[ + ('data1', True), + ('data3', 3.14), + ('data2', ['hello', 'bonjour']), + ('empty', None), + ], ) print(resp.json) asserts_data_types_bool(resp) From dc744568556f0b6770929de5fdba6b0fafe0e090 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 12 Mar 2023 12:35:12 +0100 Subject: [PATCH 42/48] Docstrings updated --- falcon/testing/client.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 203162ece..8c14ca11c 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -544,11 +544,18 @@ def simulate_request( 3-tuple ``('filename', fileobj, 'content_type')`` where ``'content-type'`` is a string defining the content type of the given file. - If both files and json are present, an exception is thrown. To pass - additional form-data with files, use data. - data : list of tuples or dict with additional data to be passed with - files (or alone if files is null), to be treated as urlencoded form data. - If both data and json are present, an exception is thrown. + + Note: + If both data and json are present, an exception is thrown. + To pass additional form-data with files, use data. + + data : list of tuples, dict or (b)string, with additional data + to be passed with files (or alone if files is null), to be treated + as urlencoded form data. + + Note: + If both data and json are present, an exception is thrown. + file_wrapper (callable): Callable that returns an iterable, to be used as the value for *wsgi.file_wrapper* in the WSGI environ (default: ``None``). This can be used to test @@ -775,11 +782,17 @@ async def _simulate_request_asgi( 3-tuple ``('filename', fileobj, 'content_type')``, where ``'content-type'`` is a string defining the content type of the given file. - If both files and json are present, an exception is thrown. To pass - additional form-data with files, use data. - data : list of tuples or dict with additional data to be passed with + + Mote: + If both files and json are present, an exception is thrown. To pass + additional form-data with files, use data. + + data : list of tuples, dict or (b)string with additional data to be passed with files (or alone if files is null), to be treated as urlencoded form data. - If both data and json are present, an exception is thrown. + + Note: + If both data and json are present, an exception is thrown. + host(str): A string to use for the hostname part of the fully qualified request URL (default: 'falconframework.org') remote_addr (str): A string to use as the remote IP address for the From 6d8fd964743214574e1b1e83ae719a62cb353fd6 Mon Sep 17 00:00:00 2001 From: TigreModerata Date: Sun, 26 Mar 2023 17:27:25 +0200 Subject: [PATCH 43/48] data string (not json) treated like body --- falcon/testing/client.py | 9 +- tests/test_multipart_formdata_request.py | 115 ++++++++++++++++++++++- 2 files changed, 120 insertions(+), 4 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 8c14ca11c..cf86485a5 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2204,7 +2204,11 @@ def _prepare_data_fields(data, boundary=None, urlenc=False): urlresult = [] body_part = b'' if isinstance(data, (str, bytes)) or hasattr(data, 'read'): - fields = list(json_module.loads(data).items()) + try: + fields = list(json_module.loads(data).items()) + except ValueError: + # if it's not a json, then treat as body + return data elif isinstance(data, dict): fields = list(data.items()) else: @@ -2387,7 +2391,8 @@ def _prepare_sim_args( else: body = _prepare_data_fields(data, None, True) headers = headers or {} - headers['Content-Type'] = MEDIA_URLENCODED + if not headers: + headers['Content-Type'] = MEDIA_URLENCODED elif json is not None: body = json_module.dumps(json, ensure_ascii=False) diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py index 10013d8ed..c1ae42b3a 100644 --- a/tests/test_multipart_formdata_request.py +++ b/tests/test_multipart_formdata_request.py @@ -42,7 +42,6 @@ def on_post(req, resp): 'text': part.text, } ) - resp.media = values @staticmethod @@ -159,6 +158,12 @@ def client(asgi): ) app.req_options.default_media_type = falcon.MEDIA_MULTIPART + app.resp_options.media_handlers = media.Handlers( + { + falcon.MEDIA_JSON: media.JSONHandler(), + } + ) + resource = AsyncMultipartAnalyzer() if asgi else MultipartAnalyzer() app.add_route('/submit', resource) @@ -313,6 +318,113 @@ def test_upload_only_data_str(client): assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} +bstring_body = ( + b'--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' + b'Content-Disposition: form-data; name="hello"\r\n\r\n' + b'world\r\n' + b'--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' + b'Content-Disposition: form-data; name="document"\r\n' + b'Content-Type: application/json\r\n\r\n' + b'{"debug": true, "message": "Hello, world!", "score": 7}\r\n' + b'--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' + b'Content-Disposition: form-data; name="file1"; filename="test.txt"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, world!\n\r\n' + b'--5b11af82ab65407ba8cdccf37d2a9c4f--\r\n' +) + +string_body = ( + '--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' + 'Content-Disposition: form-data; name="hello"\r\n\r\n' + 'world\r\n' + '--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' + 'Content-Disposition: form-data; name="document"\r\n' + 'Content-Type: application/json\r\n\r\n' + '{"debug": true, "message": "Hello, world!", "score": 7}\r\n' + '--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' + 'Content-Disposition: form-data; name="file1"; filename="test.txt"\r\n' + 'Content-Type: text/plain\r\n\r\n' + 'Hello, world!\n\r\n' + '--5b11af82ab65407ba8cdccf37d2a9c4f--\r\n' +) + + +def test_upload_only_data_bstrnojson(client): + resp = client.simulate_post( + '/submit', + data=bstring_body, + headers={ + 'Content-Type': 'multipart/form-data; ' + 'boundary=5b11af82ab65407ba8cdccf37d2a9c4f', + }, + ) + assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': 'world', + 'filename': None, + 'name': 'hello', + 'secure_filename': None, + 'text': 'world', + }, + { + 'content_type': 'application/json', + 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', + 'filename': None, + 'name': 'document', + 'secure_filename': None, + 'text': None, + }, + { + 'content_type': 'text/plain', + 'data': 'Hello, world!\n', + 'filename': 'test.txt', + 'name': 'file1', + 'secure_filename': 'test.txt', + 'text': 'Hello, world!\n', + }, + ] + + +def test_upload_only_data_strnojson(client): + resp = client.simulate_post( + '/submit', + data=string_body, + headers={ + 'Content-Type': 'multipart/form-data; ' + 'boundary=5b11af82ab65407ba8cdccf37d2a9c4f', + }, + ) + assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': 'world', + 'filename': None, + 'name': 'hello', + 'secure_filename': None, + 'text': 'world', + }, + { + 'content_type': 'application/json', + 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', + 'filename': None, + 'name': 'document', + 'secure_filename': None, + 'text': None, + }, + { + 'content_type': 'text/plain', + 'data': 'Hello, world!\n', + 'filename': 'test.txt', + 'name': 'file1', + 'secure_filename': 'test.txt', + 'text': 'Hello, world!\n', + }, + ] + + def asserts_data_types_bool(resp): assert resp.status_code == 200 expected_list = [ @@ -409,7 +521,6 @@ def test_upload_data_bool(client): ('empty', None), ], ) - print(resp.json) asserts_data_types_bool(resp) From aef3e0dc0cf960957ff8172cd171de9a21bb9acf Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Mon, 3 Jul 2023 00:00:58 +0200 Subject: [PATCH 44/48] Update 4.0.0.rst --- docs/changes/4.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changes/4.0.0.rst b/docs/changes/4.0.0.rst index 595b909a4..457343885 100644 --- a/docs/changes/4.0.0.rst +++ b/docs/changes/4.0.0.rst @@ -15,6 +15,7 @@ Changes to Supported Platforms - CPython 3.11 is now fully supported. (`#2072 `__) - End-of-life Python 3.5 & 3.6 are no longer supported. (`#2074 `__) + .. towncrier release notes start Contributors to this Release From 3c2b4fc124ecb113bdc24e4bfa92996edd174eb4 Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Mon, 3 Jul 2023 08:27:57 +0200 Subject: [PATCH 45/48] chore(requirements): remove extraneous whitespace from `mintest` --- requirements/mintest | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/mintest b/requirements/mintest index 50f757e50..8fce419e3 100644 --- a/requirements/mintest +++ b/requirements/mintest @@ -5,4 +5,3 @@ pytest pyyaml requests ujson - From 62275cc9cce611eddf8ad03789a302551e19fe14 Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Wed, 19 Jul 2023 16:08:22 +0200 Subject: [PATCH 46/48] refactor: restore some stuff from master, temp remove 1 file for now --- tests/test_media_multipart.py | 1 + tests/test_multipart_formdata_request.py | 982 ----------------------- 2 files changed, 1 insertion(+), 982 deletions(-) delete mode 100644 tests/test_multipart_formdata_request.py diff --git a/tests/test_media_multipart.py b/tests/test_media_multipart.py index 941dd7226..7edf51f4c 100644 --- a/tests/test_media_multipart.py +++ b/tests/test_media_multipart.py @@ -98,6 +98,7 @@ b'--boundary--\r\n' ) + EXAMPLES = { '5b11af82ab65407ba8cdccf37d2a9c4f': EXAMPLE1, '---------------------------1574247108204320607285918568': EXAMPLE2, diff --git a/tests/test_multipart_formdata_request.py b/tests/test_multipart_formdata_request.py deleted file mode 100644 index c1ae42b3a..000000000 --- a/tests/test_multipart_formdata_request.py +++ /dev/null @@ -1,982 +0,0 @@ -import base64 -import io -import json - -import pytest - -import falcon -from falcon import media -from falcon import testing - -from falcon.errors import HTTPBadRequest - -from _util import create_app # NOQA: I100 - - -""" -Request takes tuples like (filename, data, content_type, headers) - -""" - - -class MultipartAnalyzer: - @staticmethod - def on_post(req, resp): - values = [] - for part in req.media: - # For mixed nested file requests - part_type = part.content_type - inner_form = [] - if part.content_type.startswith('multipart/mixed'): - for nested in part.media: - inner_form.append({'name': nested.name, 'text': nested.text}) - part_type = 'multipart/mixed' - # ---------------------------------------------------- - values.append( - { - 'content_type': part_type, - 'data': inner_form or part.data.decode(), - 'filename': part.filename, - 'name': part.name, - 'secure_filename': part.secure_filename if part.filename else None, - 'text': part.text, - } - ) - resp.media = values - - @staticmethod - def on_post_data(req, resp): - deserialized = req.get_media() - resp.media = deserialized - - @staticmethod - def on_post_media(req, resp): - deserialized = [] - for part in req.media: - part_media = part.get_media() - assert part_media == part.media - - deserialized.append(part_media) - resp.media = deserialized - - @staticmethod - def on_post_image(req, resp): - values = [] - for part in req.media: - values.append( - { - 'content_type': part.content_type, - 'data': base64.b64encode(part.data).decode(), - 'filename': part.filename, - 'name': part.name, - 'secure_filename': part.secure_filename if part.filename else None, - } - ) - - resp.media = values - - -class AsyncMultipartAnalyzer: - @staticmethod - async def on_post(req, resp): - values = [] - form = await req.get_media() - async for part in form: - # For mixed nested file requests - part_type = part.content_type - inner_form = [] - if part_type.startswith('multipart/mixed'): - part_form = await part.get_media() - async for nested in part_form: - inner_form.append({'name': nested.name, 'text': await nested.text}) - part_type = 'multipart/mixed' - # ---------------------------------------------------- - values.append( - { - 'content_type': part_type, - 'data': inner_form or (await part.data).decode(), - 'filename': part.filename, - 'name': part.name, - 'secure_filename': part.secure_filename if part.filename else None, - 'text': (await part.text), - } - ) - resp.media = values - - @staticmethod - async def on_post_data(req, resp): - data = await req.get_media() - resp.media = data - - @staticmethod - async def on_post_media(req, resp): - deserialized = [] - form = await req.media - async for part in form: - part_media = await part.get_media() - assert part_media == await part.media - deserialized.append(part_media) - - resp.media = deserialized - - @staticmethod - async def on_post_image(req, resp): - values = [] - form = await req.get_media() - async for part in form: - data = await part.data - values.append( - { - 'content_type': part.content_type, - 'data': base64.b64encode(data).decode(), - 'filename': part.filename, - 'name': part.name, - 'secure_filename': part.secure_filename if part.filename else None, - } - ) - resp.media = values - - -@pytest.fixture -def client(asgi): - app = create_app(asgi) - - # For handling mixed nested requests ----------------------- - parser = media.MultipartFormHandler() - parser.parse_options.media_handlers[ - 'multipart/mixed' - ] = media.MultipartFormHandler() - - # ------------------------------------------------------------ - - app.req_options.media_handlers = media.Handlers( - { - falcon.MEDIA_JSON: media.JSONHandler(), - falcon.MEDIA_URLENCODED: media.URLEncodedFormHandler(), - falcon.MEDIA_MULTIPART: parser, - } - ) - - app.req_options.default_media_type = falcon.MEDIA_MULTIPART - app.resp_options.media_handlers = media.Handlers( - { - falcon.MEDIA_JSON: media.JSONHandler(), - } - ) - - resource = AsyncMultipartAnalyzer() if asgi else MultipartAnalyzer() - - app.add_route('/submit', resource) - app.add_route('/media', resource, suffix='media') - app.add_route('/data', resource, suffix='data') - app.add_route('/image', resource, suffix='image') - - return testing.TestClient(app) - - -# region - TESTING THE files PARAMETER IN simulate_request FOR DIFFERENT DATA - -# region - TESTING CONSISTENCY OF UPLOAD OF DIFFERENT FORMAT FOR files - -payload1 = b'{"debug": true, "message": "Hello, world!", "score": 7}' - -FILES1 = { - 'fileobj': 'just some stuff', - 'hello': (None, 'world'), - 'document': (None, payload1, 'application/json'), - 'file1': ('test.txt', 'Hello, world!', 'text/plain'), -} -FILES1_TUPLES = [ - ('fileobj', 'just some stuff'), - ('hello', (None, 'world')), - ('document', (None, payload1, 'application/json')), - ('file1', ('test.txt', 'Hello, world!', 'text/plain')), -] - -FILES1_RESP = [ - { - 'content_type': 'text/plain', - 'data': 'just some stuff', - 'filename': 'fileobj', - 'name': 'fileobj', - 'secure_filename': 'fileobj', - 'text': 'just some stuff', - }, - { - 'content_type': 'text/plain', - 'data': 'world', - 'filename': None, - 'name': 'hello', - 'secure_filename': None, - 'text': 'world', - }, - { - 'content_type': 'application/json', - 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', - 'filename': None, - 'name': 'document', - 'secure_filename': None, - 'text': None, - }, - { - 'content_type': 'text/plain', - 'data': 'Hello, world!', - 'filename': 'test.txt', - 'name': 'file1', - 'secure_filename': 'test.txt', - 'text': 'Hello, world!', - }, -] - - -def test_upload_multipart_dict(client): - resp = client.simulate_post('/submit', files=FILES1) - - assert resp.status_code == 200 - assert resp.json == FILES1_RESP - - -def test_upload_multipart_list(client): - resp = client.simulate_post('/submit', files=FILES1_TUPLES) - - assert resp.status_code == 200 - assert resp.json == FILES1_RESP - - -FILES3 = { - 'bytes': ('bytes', b'123456789abcdef\n' * 64 * 1024 * 2, 'application/x-falcon'), - 'empty': (None, '', 'text/plain'), -} - - -def test_body_too_large(client): - resp = client.simulate_post('/submit', files=FILES3) - assert resp.status_code == 400 - assert resp.json == { - 'description': 'body part is too large', - 'title': 'Malformed multipart/form-data request media', - } - - -FILES5 = { - 'factorials': ( - None, - '{"count": 6, "numbers": [1, 2, 6, 24, 120, 720]}', - 'application/json', - ), - 'person': ( - None, - 'name=Jane&surname=Doe&fruit=%F0%9F%8D%8F', - 'application/x-www-form-urlencoded', - ), -} - - -def test_upload_multipart_media(client): - resp = client.simulate_post('/media', files=FILES5) - - assert resp.status_code == 200 - assert resp.json == [ - {'count': 6, 'numbers': [1, 2, 6, 24, 120, 720]}, - { - 'fruit': b'\xF0\x9F\x8D\x8F'.decode('utf8'), - 'name': 'Jane', - 'surname': 'Doe', - }, - ] - - -def test_upload_only_data(client): - resp = client.simulate_post( - '/data', data=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)] - ) - assert resp.status_code == 200 - assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} - - -def test_upload_only_datadict(client): - resp = client.simulate_post( - '/data', data={'data1': 5, 'data2': ['hello', 'bonjour']} - ) - assert resp.status_code == 200 - assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} - - -def test_upload_only_data_bstr(client): - resp = client.simulate_post( - '/data', data=b'{"data1": 5, "data2": ["hello", "bonjour"]}' - ) - assert resp.status_code == 200 - assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} - - -def test_upload_only_data_str(client): - resp = client.simulate_post( - '/data', data='{"data1": 5, "data2": ["hello", "bonjour"]}' - ) - assert resp.status_code == 200 - assert resp.json == {'data1': '5', 'data2': ['hello', 'bonjour']} - - -bstring_body = ( - b'--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' - b'Content-Disposition: form-data; name="hello"\r\n\r\n' - b'world\r\n' - b'--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' - b'Content-Disposition: form-data; name="document"\r\n' - b'Content-Type: application/json\r\n\r\n' - b'{"debug": true, "message": "Hello, world!", "score": 7}\r\n' - b'--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' - b'Content-Disposition: form-data; name="file1"; filename="test.txt"\r\n' - b'Content-Type: text/plain\r\n\r\n' - b'Hello, world!\n\r\n' - b'--5b11af82ab65407ba8cdccf37d2a9c4f--\r\n' -) - -string_body = ( - '--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' - 'Content-Disposition: form-data; name="hello"\r\n\r\n' - 'world\r\n' - '--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' - 'Content-Disposition: form-data; name="document"\r\n' - 'Content-Type: application/json\r\n\r\n' - '{"debug": true, "message": "Hello, world!", "score": 7}\r\n' - '--5b11af82ab65407ba8cdccf37d2a9c4f\r\n' - 'Content-Disposition: form-data; name="file1"; filename="test.txt"\r\n' - 'Content-Type: text/plain\r\n\r\n' - 'Hello, world!\n\r\n' - '--5b11af82ab65407ba8cdccf37d2a9c4f--\r\n' -) - - -def test_upload_only_data_bstrnojson(client): - resp = client.simulate_post( - '/submit', - data=bstring_body, - headers={ - 'Content-Type': 'multipart/form-data; ' - 'boundary=5b11af82ab65407ba8cdccf37d2a9c4f', - }, - ) - assert resp.status_code == 200 - assert resp.json == [ - { - 'content_type': 'text/plain', - 'data': 'world', - 'filename': None, - 'name': 'hello', - 'secure_filename': None, - 'text': 'world', - }, - { - 'content_type': 'application/json', - 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', - 'filename': None, - 'name': 'document', - 'secure_filename': None, - 'text': None, - }, - { - 'content_type': 'text/plain', - 'data': 'Hello, world!\n', - 'filename': 'test.txt', - 'name': 'file1', - 'secure_filename': 'test.txt', - 'text': 'Hello, world!\n', - }, - ] - - -def test_upload_only_data_strnojson(client): - resp = client.simulate_post( - '/submit', - data=string_body, - headers={ - 'Content-Type': 'multipart/form-data; ' - 'boundary=5b11af82ab65407ba8cdccf37d2a9c4f', - }, - ) - assert resp.status_code == 200 - assert resp.json == [ - { - 'content_type': 'text/plain', - 'data': 'world', - 'filename': None, - 'name': 'hello', - 'secure_filename': None, - 'text': 'world', - }, - { - 'content_type': 'application/json', - 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', - 'filename': None, - 'name': 'document', - 'secure_filename': None, - 'text': None, - }, - { - 'content_type': 'text/plain', - 'data': 'Hello, world!\n', - 'filename': 'test.txt', - 'name': 'file1', - 'secure_filename': 'test.txt', - 'text': 'Hello, world!\n', - }, - ] - - -def asserts_data_types_bool(resp): - assert resp.status_code == 200 - expected_list = [ - { - 'content_type': 'text/plain', - 'data': 'just some stuff', - 'filename': 'fileobj', - 'name': 'fileobj', - 'secure_filename': 'fileobj', - 'text': 'just some stuff', - }, - { - 'content_type': 'text/plain', - 'data': 'True', - 'filename': None, - 'name': 'data1', - 'secure_filename': None, - 'text': 'True', - }, - { - 'content_type': 'text/plain', - 'data': '3.14', - 'filename': None, - 'name': 'data3', - 'secure_filename': None, - 'text': '3.14', - }, - { - 'content_type': 'text/plain', - 'data': 'hello', - 'filename': None, - 'name': 'data2', - 'secure_filename': None, - 'text': 'hello', - }, - { - 'content_type': 'text/plain', - 'data': 'bonjour', - 'filename': None, - 'name': 'data2', - 'secure_filename': None, - 'text': 'bonjour', - }, - { - 'content_type': 'text/plain', - 'data': '', - 'filename': None, - 'name': 'empty', - 'secure_filename': None, - 'text': '', - }, - { - 'content_type': 'text/plain', - 'data': 'world', - 'filename': None, - 'name': 'hello', - 'secure_filename': None, - 'text': 'world', - }, - { - 'content_type': 'application/json', - 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', - 'filename': None, - 'name': 'document', - 'secure_filename': None, - 'text': None, - }, - { - 'content_type': 'text/plain', - 'data': 'Hello, world!', - 'filename': 'test.txt', - 'name': 'file1', - 'secure_filename': 'test.txt', - 'text': 'Hello, world!', - }, - ] - - # Result will be unordered, because both fileobj and data are present. - # When all files are tuples, response will be unordered if json - # contains dictionaries - then resp.json == expected_list can be used. - - assert len(resp.json) == len(expected_list) - assert all(map(lambda el: el in expected_list, resp.json)) - - -def test_upload_data_bool(client): - resp = client.simulate_post( - '/submit', - files=FILES1, - data=[ - ('data1', True), - ('data3', 3.14), - ('data2', ['hello', 'bonjour']), - ('empty', None), - ], - ) - asserts_data_types_bool(resp) - - -# endregion{ - - -# region - TEST DIFFERENT DATA TYPES in json part -def asserts_data_types(resp): - assert resp.status_code == 200 - expected_list = [ - { - 'content_type': 'text/plain', - 'data': 'just some stuff', - 'filename': 'fileobj', - 'name': 'fileobj', - 'secure_filename': 'fileobj', - 'text': 'just some stuff', - }, - { - 'content_type': 'text/plain', - 'data': '5', - 'filename': None, - 'name': 'data1', - 'secure_filename': None, - 'text': '5', - }, - { - 'content_type': 'text/plain', - 'data': 'hello', - 'filename': None, - 'name': 'data2', - 'secure_filename': None, - 'text': 'hello', - }, - { - 'content_type': 'text/plain', - 'data': 'bonjour', - 'filename': None, - 'name': 'data2', - 'secure_filename': None, - 'text': 'bonjour', - }, - { - 'content_type': 'text/plain', - 'data': '', - 'filename': None, - 'name': 'empty', - 'secure_filename': None, - 'text': '', - }, - { - 'content_type': 'text/plain', - 'data': 'world', - 'filename': None, - 'name': 'hello', - 'secure_filename': None, - 'text': 'world', - }, - { - 'content_type': 'application/json', - 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', - 'filename': None, - 'name': 'document', - 'secure_filename': None, - 'text': None, - }, - { - 'content_type': 'text/plain', - 'data': 'Hello, world!', - 'filename': 'test.txt', - 'name': 'file1', - 'secure_filename': 'test.txt', - 'text': 'Hello, world!', - }, - ] - - # Result will be unordered, because both fileobj and data are present. - # When all files are tuples, response will be unordered if json - # contains dictionaries - then resp.json == expected_list can be used. - - assert len(resp.json) == len(expected_list) - assert all(map(lambda el: el in expected_list, resp.json)) - - -def test_upload_multipart_datalist(client): - resp = client.simulate_post( - '/submit', - files=FILES1, - data=[('data1', 5), ('data2', ['hello', 'bonjour']), ('empty', None)], - ) - asserts_data_types(resp) - - -def test_upload_multipart_datalisttuple(client): - resp = client.simulate_post( - '/submit', - files=FILES1, - data=[('data1', 5), ('data2', ('hello', 'bonjour')), ('empty', None)], - ) - asserts_data_types(resp) - - -def test_upload_multipart_datalistdict(client): - """json data list with dict""" - resp = client.simulate_post( - '/submit', - files=FILES1, - data=[('data1', 5), ('data2', {'hello', 'bonjour'}), ('empty', None)], - ) - asserts_data_types(resp) - - -def test_upload_multipart_datadict(client): - """json data dict with list""" - resp = client.simulate_post( - '/submit', - files=FILES1, - data={'data1': 5, 'data2': ['hello', 'bonjour'], 'empty': None}, - ) - asserts_data_types(resp) - - -def test_upload_multipart_datadicttuple(client): - """json data dict with tuple""" - resp = client.simulate_post( - '/submit', - files=FILES1, - data={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, - ) - asserts_data_types(resp) - - -def test_upload_multipart_datadictdict(client): - """json data dict with dict""" - resp = client.simulate_post( - '/submit', - files=FILES1, - data={'data1': 5, 'data2': {'hello', 'bonjour'}, 'empty': None}, - ) - asserts_data_types(resp) - - -# endregion - - -# region - TEST INVALID DATA TYPES FOR FILES -def test_invalid_files(client): - """invalid file type""" - with pytest.raises(ValueError): - client.simulate_post('/submit', files='heya') - - -def test_invalid_files_null(client): - """empty file in files""" - with pytest.raises(ValueError): - client.simulate_post('/submit', files={'file': ()}) - - -def test_invalid_files_data_json(client): - """empty json and data and files""" - with pytest.raises(HTTPBadRequest): - client.simulate_post( - '/submit', - files=FILES1, - data={'data1': 5, 'data2': ('hello', 'bonjour'), 'empty': None}, - json={'badrequest': 'should fail'}, - ) - - -# endregion - -# region - TEST NESTED FILES UPLOAD - - -FILES6 = { - 'field1': 'Joe Blow', - 'docs': ( - None, - json.dumps( - { - 'file1': ('file1.txt', 'this is file1'), - 'file2': ( - 'file2.txt', - 'Hello, World!', - ), - } - ).encode(), - 'multipart/mixed', - ), - 'document': (None, payload1, 'application/json'), -} - - -def test_nested_multipart_mixed(client): - resp = client.simulate_post('/submit', files=FILES6) - assert resp.status_code == 200 - assert resp.json == [ - { - 'content_type': 'text/plain', - 'data': 'Joe Blow', - 'filename': 'field1', - 'name': 'field1', - 'secure_filename': 'field1', - 'text': 'Joe Blow', - }, - { - 'content_type': 'multipart/mixed', - 'data': [ - {'name': 'file1', 'text': 'this is file1'}, - {'name': 'file2', 'text': 'Hello, World!'}, - ], - 'filename': None, - 'name': 'docs', - 'secure_filename': None, - 'text': None, - }, - { - 'content_type': 'application/json', - 'data': '{"debug": true, "message": "Hello, world!", "score": 7}', - 'filename': None, - 'name': 'document', - 'secure_filename': None, - 'text': None, - }, - ] - - -# endregion - -# endregion - -# region - TEST UPLOADING ACTUAL FILES: TEXT, IMAGE - -IMAGE_FILE = ( - b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xe1\x00\x00' - b'\x00\xe1\x08\x03\x00\x00\x00\tm"H\x00\x00\x00uPLTE\xff\xff\xff' - b'\xf0\xadN\xf0\xacK\xef\xa9A\xf0\xaaF\xef\xa8?\xf0\xaaE\xef\xa8' - b'=\xff\xfe\xfc\xfe\xfb\xf6\xfd\xf6\xed\xfe\xf8\xf1\xf1\xb3' - b'\\\xfb\xea\xd5\xfd\xf3\xe6\xfa\xe6\xcd\xf3\xbdu\xf2\xb9k\xf0' - b'\xafQ\xf4\xc5\x88\xf9\xe1\xc3\xf7\xd6\xad\xf8\xdc\xb8\xf5\xcc' - b'\x97\xfc\xef\xde\xf1\xb5a\xf2\xbbp\xf8\xd9\xb2\xfb\xe9\xd3\xf1' - b'\xb1W\xfa\xe4\xc8\xf5\xc9\x90\xf6\xd1\xa1\xf4\xc2\x81\xf7\xd4' - b'\xa7\xf4\xc7\x8b\xf5\xca\x94\xef\xa42\xf6\xd0\x9e_\xd7\x99\xa5' - b'\x00\x00\rcIDATx\x9c\xed]i\x97\xaa8\x10\x95JHX\x14A\x11Q\xdb' - b'\x05\xed\xf6\xff\xff\xc4aS\x11\x03$!\x10f\xc6\xfb\xe9\xbd\xd3' - b'\x07\xcc%IU\xa5\xb6\xccf_|\xf1\xc5\x17_|\xf1\xc5\x17\xff38\xba' - b'\x0700\x1c?\xd0=\x84a\x11\xafh\xa4{' - b'\x0c\x83"\xc6@-\xdd\x83\x18\x12\x0b\nx\xab{' - b'\x10C"\x9dA\x83z\xbaG1 ' - b'\xbc\x15\x18\xb0\xd7=\x8a\x01\xe1\x1c\xc10\xd0\x7fX\xce\xd8' - b'\xf3\x94 \xac\\\xdd\xe3\x18\x0e{' - b'l\x18\x06>\xeb\x1eF\x1b\xac^R\xf0FR\x82\x06Z\xa8\x1a\xcd\x10X' - b'\xcf{<\xbcA\x19A8*\x1b\xcd\x00\x88\xa9/\xff\xf0\x89f\x04\x8di' - b'+C\x1f\xdd\xa4\x9fu\x00\x8c\xc9/\xd2%"\x1b\xe9\x87w\x05A\x83(' - b'\x1c\x90j\xa4\xb2\x1e-e\x1f\xde\xa2\x82 ' - b'$*\x87\xa4\x18kb\xa0X\xf2\xd9eI\xd0 ' - b'\x13V\xf7w3\x15\x13\x92\x87\x02wU\xae\xd1)\xdb\xa4\x87T\x14' - b'\x82\xac\xb2X\xe3\x92\xa0\x01J\x07\xa5\x12\x17\xb3\x879\xb24' - b'\x9f\x04{\xa8\x9bAa\xad\xf3}\x84B\xb9\xa7\x9fk\xd4 ' - b'\x17\xb5\x03S\x05gG\x8aM$g3o\xc9\x83\xe0T\xcf\x15\x11-\xe6\x00' - b'\xe4\x1cH\x1ez\x124\xcc)\xea\xfb\xd8\x7f\xec"r\x95zA\x82_\x0c' - b'\xd1\xf4\xfc\x88\xf6\x1d?\x07H\xa5\xb4\xe1\xa22\x85\xc6\xf4' - b'\xce\x86\xd1\xf1\xb5\x87`.\xa5\r}0*\xaf\xb0U\x8f\xb0\x1f\xa2' - b'\x15\xaa\x0c\x8f\xdce\xde\xf1C+S\x08\xbb)\xf9\x11\xed\xe8hV' - b'\xf8\xa5\x0c\xa5\x16iu\x17\xa6\xb2j:\x0c\x9dK\x8d\x9f\xa4\xb2' - b'\xf6\xa81I\x86?\t\x90w~\xa9\x18<\xc9\xbc\xe9N&\xc8\xd0\xb9\xce' - b'\t6\xea\x80\x9d\xd4\xcb\xd0\xfbK& ' - b'i\xdc\xc8\xa7\x1f\xd3\x97\xc1\x94\xb2\xd8\xc2w\x86\x06\xd1\xac' - b'-\x9c0\xc1\x88EO\xda\x81\xe4\xd7\xde&\xa7R\x15\xc1\x8b|@\x9f' - b'\xab\xf314\xa9)t>6\xb36\xbb4\xde\x04\xd00{' - b'\xc5\x14\xca\x99\xa4\xf5Ej\xe0_\xc5\x03\xe7\x82\x15_\x8e\xec' - b'\xbd\xf7\x82\xf9#\xf5\xeau}MH\x9f\xa1\xe5a/\x7fW]\xf4\xd2O\xbf' - b'\x96{\xf9\xfc\xe3\xc5#{' - b'1\x9cp\xbd2q\x17\xbd\xf4\xcb\x1brG\x82\xb8\xbeH\xc7\xf5Dy\x87' - b'\xbd\x818\xe8e\x1f^rX\x11\xf9x\x95\xe4\x86\x16\x87w\xdd\xa1' - b'\xce\xb5\xf9\xfc\xee\xb2>\xce3C4\x9bc,' - b'S\xef:\xef\xdez\x95\xcf.wjJ\xb1g\xfc\xc8\xf0\x9e\x1ao\x13\x88' - b'\xd0K\t\x82\xecW\xb7W\x8c\xdf\x81\xe3\xa0\x86\x9b\x15\xfaD\x88' - b'^\xf6\xd1\xe5\x14E\n\x97\xb2\xde7\xa4\xd2w\xee\xab6\xad\xce' - b'\x86\x9c1\x93#f2\x1c\xce\xfa^\xf8\x94Or\xbe\r\x87J\x9d\x99\n' - b'\x9c>\x95E\x06$\x1f\xc0j\xfd\xb5=\x15\xa6\x97\xed\xc1\x1e\x04g' - b'\x11\x9b\xa1\x01\x03\x1c0\xd2\xf9\x93\xe0g\xc0\xaa\x97{' - b'\x93\xa1\x0es`\xe5\x116w-\xbe\xfd\xf2\x91\xcc\xfb\xe9\xae&\x86' - b'\x06\x95\x0eC6\xfcP\xf3\x99\xa8\x15\xa8\xef\xa7nd\x08+\x95\xc2' - b'\xc6M\x98\x12\xad\x13\x80{\x0b\x84\xa6}(' - b"m\xc93\x11\xcf\xe5&\x90\xec\xfaG\x18>N\x87/\xa8S\x8a'q\x05" - b'\x91\x01\xe8Y\x81S\xacA[' - b'\x14_P\x91;#\x92\x12\xa1\x061\x94\x88\x82E\xcb\xfePd\xbc\x85R[' - b'\x10\x90\x8a\tL\xe1\xb4\xfd\xc6\xbei\xb2\xdb\x1e\x90\xcc_\xc9\xd0*\xc6' - b'\x180\x83!"\n^\xe7(' - b'\xa4\xc5\xcd\x8f\x98&\x84\x81N\xa5L_\xd4;\xa8T\xec|6\xfbt6\xb7' - b"\x01\x05C%\x81\x9c\xba\xa5\x81\x99\xc8\xa8'!1\x03H.O\x86\x0b" - b'\x1c\xe2\x80\x04\x12\xc7P\x11U\x08\x862\x1b\x86\x81\x03\x87' - b']\x05 ,o6\x02\xe6\x1a\x9e\x0f\x9a\xa6\xc4gwP\xc1\xfcq\xeb\xc8' - b'?\x85d\xe8tH\x9eI\xcc\x8e\xa3B+\xb5\xf9l\xfd\x81\x1e\xc9\xf7' - b"\x9c\xb0v\\\x9f\x1b\xb0\x80\xda\xe0|g\xfe\xe9\x06'XI\xf1\xee" - b'\xfa\xd8{\xeeilq\x1e\xd4\t\x8e\x928\xff\xdbe\xd8\x94\x00\x93W' - b'\xa6wk\xd9\x12X.EF\x18;^\xfb\n\xed\xb8\x0c\xab6\xdf\xc1\xfb7' - b'\x1b6\n\xf4B\xd7\x11\xa32$r\xe5P\xff\xdc\xa7&\xe5G,' - b'\xec\xf7r%\xf1\x9a\xa4\xa4oX\xab7\xb8Ox\xe5\x94\x94%o\xbc' - b'\xaaBoI@\x01\xc1\xe0mY\x11\xc6\xe7\xd5J\x81t\xf3\x9b\t\x0c\xb6' - b'\x1cr\x16u\xe0\x9e\xc2i\xb45Xsn\xa9r\xcc\xd9$r\xbb\x81\xf5o' - b'\xc3\x0c\x96Ph%\xdb\x89.\xf7\x03\x92\xb5\x13\xaa\xc1\xce>mb' - b'\x98\xce!\x7f\xc0\x10\xe9\xaf\x1f\xcb!\x12\xe2L\xf7\xe1\x92' - b'\xfb\x8b\x8c\xeb\xbeh\x81#\xe0\xd8\x08D\x82\xda\xd3\xe9D\xf5' - b'\xcbK\x11\x0c\xc7\n\xf8\xbfG\x8f.:\x8a\xd1\x1d\x03/\xe7\xc4pf' - b'\xbe\x80\xe8\xa5\xd3ik\x90\xf0,' - b'<@\xbee\xefDt\x8bf\xb3\xbb\n\x8e\x08\x0b\x10\xe3:[' - b'\x08%?M\xa9$\xbeK\xc3\x01\xa1\xab\xad=\xdb\x8a%qO\xc3\xa2' - b')\x110\x87\x0e\x00\x98 ' - b'DQpYdm\x0f\x84\xac\x9ft\xdb\xear\xd1\xb0py\xafe\xcf\x89aX\x05' - b'\xc9\xef&:e.k\xe7\xdaZ\xda\xca\x84\xde\xf3}\r\xcf\xe6C)9\x13' - b'\xe6\xfb\xdb5\\x\x0f\x83\xc4]^vb%Z%C\xfdG\xa7\x17\xca0\x12\xa6' - b'\xabd{zQ\xf3\xe2\xe8\x92\xec\x88)A/\xc3tD\xe9,' - b'oH\x90N\xde\xef\xb2 ' - b'\xe7\xc6\xcb\xcd\xd9\xdf\x1d\xc1L\xd7\xaa\x1c;-\xa5\xc6m\xd8#s' - b'\x9dg|\xc6\xd1v\x9f\xcaNDRj\xb2\xdcJ\x86\xd3j\xb6u\xbd\xb83' - b'\xfbt\x0f(\xe5,\xd8\xed\xc6\xd4ZO\xba\x07\x1f\xf1\x14[' - b'\xf3cZ]\xfd\xe2D\\\x19t\x01M\xe3\xf8\x9b\xc3\xf3\x19=T\xfa3' - b'\x9cN\xdf\xc2+\x1d\x80\xdf4\xfcl9\xecD8\x11\x9d\x0b\x92}t\xd4' - b'\xc3\r\x04\xadM^\xe0\x11\x12\xd9x ' - b'rd\x17\xc34<\x89\xaf\x96\xb4\xeaa*\xaeO\x95\x04o\x18P\x02t\x1a' - b'\x9e\xc4\xcb@\x9bp:\xed\xc2\xc5J\tD\xa0\xb2\xfe\xb6\x0f\x86[' - b'\xa4\x13\xf1$v\xd4\xb4\xf5\xc1DB\x16\x8b\xe1\x18\xean\x08W' - b'\x827\xf5[\x1cS\xf1$\x0e\xc8p"\x9e\xc4\x01\x19\xfa\xd3P\x87r' - b'%\xd8|\x14\xff\x92)8\xa2\x1a\x1bf\xa8\x00\xee\xd7\x90E\r\xb6C' - b'\x99\xdd9\x00\xb4\x9b\xa6\xfc\xd5\t\x92\x14\xb5\xdf\x12"P\x97' - b'%\t\xdd\x11Df\xd8\xb0\xa9q\xaa\x14\x80hu\xd60\xfaE\xa6\xd6\xd6' - b'\xd6\xdb+\xdc\x9dp\xd4i\xdb\xc4\x9f6\x1b\xfe;de\x93\n)b\x9d' - b'=\x99\x9f\x91_ \x84 ' - b'd"\x02I\xbe\xa8l\x95\xbe\x9bq\xea\xf2\xd88\x14<\x00\'\xd7\xc3' - b'!\\.\x7f\xe2\x87\x1db\te\xbaw\x80\xea;G\x15\x0ca\xc5\n\xf3]Mu' - b'\xf2F\xdfA\xaaH.5\xd9q\xcc\x1f\xd1\x98v3\xf4]\xd2\x97\xe7\xec5' - b'\xb6L\xb6n\xca\xa6Q}\xe7;^d\xfd\x10\xf0\xa5\xf1\xcf?;U\xbbQ' - b'\xdfV\xbc\x99\x06\x18\xcd\x7f\xb6\xae\xa0Ho\xe8\x8b\xd2l)\xf9k' - b'\x93\x03\xdeM\xaeIY\x1d\x1a#\xde\xde\xe1\xda~\x8c\x8b\x03%\x1c' - b'\xc7,\x94\x15\xc62Pa\xa9N(\x9a\xc8@\x18\xf4\x17\xab\x13\xcb[' - b'\xf8\x80\x02\x8e\x93\xca\x90ba\xd9w?\xca\xf7\xea\x1e\r\x0b_' - b'\xb2\xff\xeac\x12\xa7\xe1\xe7oE|\xc3}8\x9a\xd3p\x83\xb7\xc3Kz' - b'\xc8\xd5\xa9\x84\xa3:p\x12\xa9\x92\xaeA\xbbc\x8a\x0f\xb6/m' - b'\xadN\xa6\xfe\xa2\x0b\xbf\xd2\x14\xcdI%\x825\xc1\xf6fWY\x8a' - b'\xff\x8e\xeb\xdc\x7f\xa9\x1foeO\xc7d2U4-\xf0\x01\xc3\x0fw\xfb' - b'\xa4:t\xbb\x889\x90\xb5\xf7\x82#oS\x8c\x0f\xc0J7\x81N\xb8Y' - b'\xfd(\xb9\n\x96\x9e\xbe\xa0\xad\x8b\x047\x8a\xd9\x03\xf9<\x152' - b'\xf5uZ\xd4-\xd0\x05\x7f\x15b\rS\x89\xf07\xe2\x9e\x9b4\xf8>kn' - b'\xb7\x07\xd0\x9a\x04?z\xcf!1\x94U\xc3\xa9u\xd2\x940\x06h\xb5' - b'\xf3\x83#i6`U\xdf9\xa1\x16\x8f\x16\x05\xd0\x94\x03`\x06\xa7L' - b'\xab[^\x94\x18M\x17\x1c\xeai\xca\xc3\x89G\xc8\x91\xb8\xcc\xb4F' - b'\xa8&\xe8\xe7\x97T\xb2(' - b'*it?\x10\x9e\xb4H\xec1\xe6\x10p-D\xe0\\\x08\xcb\xfd\x81&\x92' - b']\xcb\xc0\xb3\xbf\x17Z|\\\x1a\xc9 ' - b'\x98\xe1\xc4*\x04\x98\xec9\xea\x95aD\x16\x0c\xb3\x869n7\\\xde' - b'>\\\xe7zC\xc3\xcd\xa8\xf4\xc3H\x19~\xecC\xca\\{' - b'\xee\x1e\x1f\x16\xf7:\xc7\x89n\xc5J\xafK\xb2\xf8h\xc5\xd0\x18a' - b'\n\x11\\\xe2{\xcd\x8d\x85\xa6h\xbd]*\xb2\x05-\xea\xd5\xd1m=J7' - b'\x88n\xe3\xda<\xca\xde\xe58 ' - b'.\xd5I#^\xbd\xcd\x10j\xd3\xe3\xd6\x1d\xd1{' - b'|~\xe3\xa8\xaf\xcfY\x03\xceo\x99\x1b\xd8\xae%\xe3t\xad:\xe7F' - b'\xe1\xba\xbf\xd5\xe0=\xcc[\xb0\xc8\xae\xb6\xc5\x98 ' - b'\xfa\xb7\xbbF\x9b\xf2\x02{H\xcd\xe9\xe3|>\x0fn\xf1{' - b'\x05?\xde\xf1\xcd\x86\x9b\x98\xe6nyy\x9e<\x906\x1f\xb1[' - b')&\xcd\xd2\x89\x8a\xff\x91\xe0\xfa,' - b'a\x7f;\x1e\xe2\x1d\xb7\xfe\x0e\rb\xdeNO\xcf\xb9\x82{' - b'\x8a\xe4\xc0<\xde\xe2\x8ax\x7f\xeb\xa7\x8f\xe7\x02\x06\x8a' - b'\xb3O\x15\xe3\xf2\xf4H,' - b'C\x17\xe5\x83\xe7B\x84\xb2N\x185\xc3\xb3\x1av\xf0+b\x9f\x04b' - b'\x16\xd8\x9d\x02\xba\xc5\xe72X\xa7I\xf3\x87\x7f\xc7\xc8\xb2W' - b'\xb5I|.);\xa9\x08V\xf1\xbb\x0bO$\xdd\xe2\x9b\xc74\xca\xde\x8d' - b'\xd2\x0f\xce\xc1\x9ayU\x9f(^!\x8c\xf1_1Y?\x95~\xd2 ' - b'\xb3\xcc\xb2\x9e\xdbd\x1e\x9d\x8b\xddnj\xb2\xdf\x1ew\xe1\xe1l' - b'&\xcd\xd0\r\xcf\xebu~eR\x9cT\x82\xc1h.u\x0crv8\x95`\xebC\xf1' - b'\xa9\x90\x96\x9b\xec\x9dR\xda\xd10"\xaf\x1dh{' - b'\xcb\xcai\x0f\x08\x96\xbd9\xcd\xceJW\xf11:\xe7"\x8dH\xddp\xd3' - b'\x13e\n8YgY\xa8\xd4\xf5\xc8*\x03\xe0W\x0f\x10@\xc6E\xde&\xc9' - b')\x02\xb9\x84\xb9\xa9J\xf8\xd5\x8d2\x14u\n\xd9A5\xa6\xe9F\xb9' - b'}t\xc9\x00\xbf\x9f\xc7\xac8k"?\xceSY\x01\x8f\xee\x7f\xcb\xed' - b'\xea\xc2+\xb1_\xcf\xecO\xfdx\xec\xbb\xb0\x9c\xfc\x93\xe1\xf9' - b'"\xca\xd6\x85\x94\xc4\xea\x85\xcc!\x03f.F,' - b'fCV\xbc\xedK\xb10+\x00E^\xde@\x0c\xed\xc7=j\xa4f\xa7\xf9\xbc' - b'\xfe\xa9\xb8g\x08\xbd\xafS|\xecs\xcdt\x86\xf2Dm\xae\xedmF\x16' - b'\x8b\xdf\xc5\xd4\x07A\xf5z\xadL\xec\x00\nk\xed\x91\xe1\xafo' - b'\x82Li\xbd\x93\xb9\xb3\xcc%\x18MF Date: Fri, 29 Dec 2023 11:37:36 +0100 Subject: [PATCH 47/48] feat(testing): add a new parameter to simulate form --- falcon/testing/client.py | 268 +++++++-------------------------- tests/test_media_multipart.py | 39 +++++ tests/test_media_urlencoded.py | 9 ++ 3 files changed, 105 insertions(+), 211 deletions(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index c5dab80c8..29f28858b 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -20,9 +20,9 @@ import asyncio import datetime as dt +import hashlib import inspect import json as json_module -import os import time from typing import Dict from typing import Optional @@ -34,8 +34,8 @@ from falcon.asgi_spec import ScopeType from falcon.constants import COMBINED_METHODS -from falcon.constants import MEDIA_JSON, MEDIA_URLENCODED -from falcon.errors import CompatibilityError, HTTPBadRequest +from falcon.constants import MEDIA_JSON, MEDIA_MULTIPART, MEDIA_URLENCODED +from falcon.errors import CompatibilityError from falcon.testing import helpers from falcon.testing.srmock import StartResponseMock from falcon.util import async_to_sync @@ -439,8 +439,7 @@ def simulate_request( content_type=None, body=None, json=None, - files=None, - data=None, + form=None, file_wrapper=None, wsgierrors=None, params=None, @@ -532,30 +531,6 @@ def simulate_request( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. - - Note: - Can only be used if data and files are null, otherwise an exception - is thrown. - - files(dict): same as the files parameter in requests, - dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) - for multipart encoding upload. - ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a - 3-tuple ``('filename', fileobj, 'content_type')`` - where ``'content-type'`` is a string defining the content - type of the given file. - - Note: - If both data and json are present, an exception is thrown. - To pass additional form-data with files, use data. - - data : list of tuples, dict or (b)string, with additional data - to be passed with files (or alone if files is null), to be treated - as urlencoded form data. - - Note: - If both data and json are present, an exception is thrown. - file_wrapper (callable): Callable that returns an iterable, to be used as the value for *wsgi.file_wrapper* in the WSGI environ (default: ``None``). This can be used to test @@ -603,8 +578,7 @@ def simulate_request( content_type=content_type, body=body, json=json, - files=files, - data=data, + form=form, params=params, params_csv=params_csv, protocol=protocol, @@ -628,8 +602,7 @@ def simulate_request( headers, body, json, - files, - data, + form, extras, ) @@ -683,8 +656,7 @@ async def _simulate_request_asgi( content_type=None, body=None, json=None, - files=None, - data=None, + form=None, params=None, params_csv=True, protocol='http', @@ -770,29 +742,9 @@ async def _simulate_request_asgi( overrides `body` and sets the Content-Type header to ``'application/json'``, overriding any value specified by either the `content_type` or `headers` arguments. - - Note: - Can only be used if data and files are null, otherwise an exception - is thrown. - - files(dict): same as the files parameter in requests, - dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) - for multipart encoding upload. - ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a - 3-tuple ``('filename', fileobj, 'content_type')``, - where ``'content-type'`` is a string defining the content - type of the given file. - - Mote: - If both files and json are present, an exception is thrown. To pass - additional form-data with files, use data. - - data : list of tuples, dict or (b)string with additional data to be passed with - files (or alone if files is null), to be treated as urlencoded form data. - - Note: - If both data and json are present, an exception is thrown. - + form (dict): A form to submit as the request's body + (default: ``None``). If present, overrides `body`, and sets the + Content-Type header. host(str): A string to use for the hostname part of the fully qualified request URL (default: 'falconframework.org') remote_addr (str): A string to use as the remote IP address for the @@ -831,8 +783,7 @@ async def _simulate_request_asgi( headers, body, json, - files, - data, + form, extras, ) @@ -2195,153 +2146,57 @@ async def __aexit__(self, exc_type, exc, tb): await self._task_req -def _prepare_data_fields(data, boundary=None, urlenc=False): - """Prepare data fields for request body. - - Args: - data: dict or list of tuples with json data from the request - - Returns: list of 2-tuples (field-name(str), value(bytes)) - - """ - urlresult = [] - body_part = b'' - if isinstance(data, (str, bytes)) or hasattr(data, 'read'): - try: - fields = list(json_module.loads(data).items()) - except ValueError: - # if it's not a json, then treat as body - return data - elif isinstance(data, dict): - fields = list(data.items()) - else: - fields = list(data) - - # Append data to the other multipart parts - for field, val in fields: - if isinstance(val, str) or not hasattr(val, '__iter__'): - val = [val] - # if no files are passed, make urlencoded form - if urlenc: - for v in val: - if v: - urlresult.append( - ( - field.encode('utf-8') if isinstance(field, str) else field, - v.encode('utf-8') if isinstance(v, str) else v, - ) - ) - # if files and data are passed, concat data to files body like in requests - else: - for v in val: - body_part += ( - f'Content-Disposition: form-data; name={field}; ' - f'\r\n\r\n'.encode() - ) - if v: - if not isinstance(v, bytes): - v = str(v) - body_part += v.encode('utf-8') if isinstance(v, str) else v - body_part += b'\r\n--' + boundary.encode() + b'\r\n' - else: - body_part += b'\r\n--' + boundary.encode() + b'\r\n' - - return body_part if not urlenc else urlencode(urlresult, doseq=True) - - -def _prepare_files(k, v): - """Prepare file attributes for body of request form. - - Args: - k: (str), file-name - v: fileobj or tuple (filename, data, content_type?) +def _encode_form(form: dict) -> tuple: + """Build the body for a URL-encoded or multipart form. - Returns: file_name, file_data, file_content_type + This utility method accepts two types of forms: a simple dict mapping + string keys to values will get URL-encoded, whereas if any value is a list + of two or three items, these will be treated as (filename, content) or + (filename, content, content_type), and encoded as a multipart form. + Returns: (encoded body bytes, Content-Type header) """ - file_content_type = None - if not v: - raise ValueError(f'No file provided for {k}') - if isinstance(v, (tuple, list)): - if len(v) == 2: - file_name, file_data = v - else: - file_name, file_data, file_content_type = v - if ( - len(v) == 3 - and file_content_type - and file_content_type.startswith('multipart/mixed') - ): - file_data, new_header = _encode_files(json_module.loads(file_data.decode())) - file_content_type = 'multipart/mixed; ' + ( - new_header['Content-Type'].split('; ')[1] - ) - else: - # if v is not a tuple or iterable it has to be a filelike obj - name = getattr(v, 'name', None) - if name and isinstance(name, str) and name[0] != '<' and name[-1] != '>': - file_name = os.path.basename(name) - else: - file_name = k - file_data = v - if hasattr(file_data, 'read'): - file_data = file_data.read() - return file_name, file_data, file_content_type - + form_items = form.items() if isinstance(form, dict) else form -def _make_boundary(): - """ - Create random boundary to be used in multipart/form-data with files. - """ - boundary = os.urandom(16).hex() - return boundary + if not any(isinstance(value, (list, tuple)) for _, value in form_items): + # URL-encoded form + return urlencode(form, doseq=True).encode(), MEDIA_URLENCODED + # Encode multipart form + body = [b''] -def _encode_files(files, data=None): - """Build the body for a multipart/form-data request. + for name, value in form_items: + data = value + filename = None + content_type = 'text/plain' - Will successfully encode files when passed as a dict or a list of - tuples. ``data`` fields are added first. - The tuples may be 2-tuples (filename, fileobj) or - 3-tuples (filename, fileobj, contentype). - Allows for content_type = ``multipart/mixed`` for submission of nested files + if isinstance(value, (list, tuple)): + try: + filename, data = value + content_type = 'application/octet-stream' + except ValueError: + filename, data, content_type = value + if isinstance(data, str): + data = data.encode() + elif not isinstance(data, bytes): + # Assume a file-like object + data = data.read() - Returns: (encoded body string, headers dict) - """ - boundary = _make_boundary() - body_string = b'--' + boundary.encode() + b'\r\n' - header = {'Content-Type': 'multipart/form-data; boundary=' + boundary} - - # Deal with the files tuples - if not isinstance(files, (dict, list)): - raise ValueError('cannot encode objects that are not 2-tuples') - elif isinstance(files, dict): - files = list(files.items()) - - for (k, v) in files: - file_name, file_data, file_content_type = _prepare_files(k, v) - if not file_data: - continue - - body_string += f'Content-Disposition: form-data; name={k}; '.encode() - body_string += ( - f'filename={file_name}\r\n'.encode() if file_name else '\r\n'.encode() - ) - body_string += ( - f'Content-Type: {file_content_type or "text/plain"}\r\n\r\n'.encode() - ) - body_string += ( - file_data.encode('utf-8') if isinstance(file_data, str) else file_data - ) - body_string += b'\r\n--' + boundary.encode() + b'\r\n' + headers = f'Content-Disposition: form-data; name="{name}"' + if filename: + headers += f'; filename="{filename}"' + headers += f'\r\nContent-Type: {content_type}\r\n\r\n' - # Handle whatever json data gets passed along with files - if data: - body_string += _prepare_data_fields(data, boundary) + body.append(headers.encode() + data + b'\r\n') - body_string = body_string[:-2] + b'--\r\n' + checksum = hashlib.sha256() + for chunk in body: + checksum.update(chunk) + boundary = checksum.hexdigest() - return body_string, header + encoded = f'--{boundary}\r\n'.encode().join(body) + encoded += f'--{boundary}--\r\n'.encode() + return encoded, f'{MEDIA_MULTIPART}; boundary={boundary}' def _prepare_sim_args( @@ -2353,8 +2208,7 @@ def _prepare_sim_args( headers, body, json, - files, - data, + form, extras, ): if not path.startswith('/'): @@ -2384,24 +2238,16 @@ def _prepare_sim_args( headers = headers or {} headers['Content-Type'] = content_type - if files or data: - if json: - raise HTTPBadRequest( - description='Cannot process both json and (files or data) args' - ) - elif files: - body, headers = _encode_files(files, data) - else: - body = _prepare_data_fields(data, None, True) - headers = headers or {} - if not headers: - headers['Content-Type'] = MEDIA_URLENCODED - elif json is not None: body = json_module.dumps(json, ensure_ascii=False) headers = headers or {} headers['Content-Type'] = MEDIA_JSON + elif form is not None: + body, content_type = _encode_form(form) + headers = headers or {} + headers['Content-Type'] = content_type + return path, query_string, headers, body, extras diff --git a/tests/test_media_multipart.py b/tests/test_media_multipart.py index 7edf51f4c..1a0f8636b 100644 --- a/tests/test_media_multipart.py +++ b/tests/test_media_multipart.py @@ -850,3 +850,42 @@ async def deserialize_async(self, stream, content_type, content_length): assert resp.status_code == 200 assert resp.json == ['', '0x48'] + + +def test_simulate_form(client): + resp = client.simulate_post( + '/submit', + form={ + 'checked': 'true', + 'file': ('test.txt', b'Hello, World!\n', 'text/plain'), + 'another': ('test.dat', io.BytesIO(b'1\n2\n3\n')), + }, + ) + + assert resp.status_code == 200 + assert resp.json == [ + { + 'content_type': 'text/plain', + 'data': 'true', + 'filename': None, + 'name': 'checked', + 'secure_filename': None, + 'text': 'true', + }, + { + 'content_type': 'text/plain', + 'data': 'Hello, World!\n', + 'filename': 'test.txt', + 'name': 'file', + 'secure_filename': 'test.txt', + 'text': 'Hello, World!\n', + }, + { + 'content_type': 'application/octet-stream', + 'data': '1\n2\n3\n', + 'filename': 'test.dat', + 'name': 'another', + 'secure_filename': 'test.dat', + 'text': None, + }, + ] diff --git a/tests/test_media_urlencoded.py b/tests/test_media_urlencoded.py index be7458773..950dbdf0d 100644 --- a/tests/test_media_urlencoded.py +++ b/tests/test_media_urlencoded.py @@ -83,3 +83,12 @@ def test_urlencoded_form(client, body, expected): headers={'Content-Type': 'application/x-www-form-urlencoded'}, ) assert resp.json == expected + + +@pytest.mark.parametrize( + 'form', [{}, {'a': '1', 'b': '2'}, (('a', '1'), ('b', '2'), ('c', '3'))] +) +def test_simulate_form(client, form): + resp = client.simulate_post('/media', form=form) + assert resp.status_code == 200 + assert resp.json == dict(form) From 93fad43258d148d74d384b68af20bca49f940294 Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Fri, 29 Dec 2023 11:49:10 +0100 Subject: [PATCH 48/48] fix(testing): fix a regression wrt passing json to simulate_request --- falcon/testing/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon/testing/client.py b/falcon/testing/client.py index 29f28858b..37d480176 100644 --- a/falcon/testing/client.py +++ b/falcon/testing/client.py @@ -2238,7 +2238,7 @@ def _prepare_sim_args( headers = headers or {} headers['Content-Type'] = content_type - elif json is not None: + if json is not None: body = json_module.dumps(json, ensure_ascii=False) headers = headers or {} headers['Content-Type'] = MEDIA_JSON