-
Notifications
You must be signed in to change notification settings - Fork 2.7k
fix: guard RESP parsers against RecursionError on deeply nested replies (#4116) #4144
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,8 @@ | |
| class _RESP3Parser(_RESPBase, PushNotificationsParser): | ||
| """RESP3 protocol implementation""" | ||
|
|
||
| MAX_NESTING_DEPTH = 100 | ||
|
|
||
| def __init__(self, socket_read_size): | ||
| super().__init__(socket_read_size) | ||
| self.pubsub_push_handler_func = self.handle_pubsub_push_response | ||
|
|
@@ -61,6 +63,7 @@ def _read_response( | |
| disable_decoding=False, | ||
| push_request=False, | ||
| timeout: Union[float, object] = SENTINEL, | ||
| _depth=0, | ||
| ): | ||
| raw = self._buffer.readline(timeout=timeout) | ||
| if not raw: | ||
|
|
@@ -106,41 +109,69 @@ def _read_response( | |
| response = self._buffer.read(int(response), timeout=timeout)[4:] | ||
| # array response | ||
| elif byte == b"*": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| response = [ | ||
| self._read_response(disable_decoding=disable_decoding, timeout=timeout) | ||
| self._read_response( | ||
| disable_decoding=disable_decoding, | ||
| timeout=timeout, | ||
| _depth=_depth + 1, | ||
| ) | ||
| for _ in range(int(response)) | ||
| ] | ||
| # set response | ||
| elif byte == b"~": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| # redis can return unhashable types (like dict) in a set, | ||
| # so we return sets as list, all the time, for predictability | ||
| response = [ | ||
| self._read_response(disable_decoding=disable_decoding, timeout=timeout) | ||
| self._read_response( | ||
| disable_decoding=disable_decoding, | ||
| timeout=timeout, | ||
| _depth=_depth + 1, | ||
| ) | ||
| for _ in range(int(response)) | ||
| ] | ||
| # map response | ||
| elif byte == b"%": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| # We cannot use a dict-comprehension to parse stream. | ||
| # Evaluation order of key:val expression in dict comprehension only | ||
| # became defined to be left-right in version 3.8 | ||
| resp_dict = {} | ||
| for _ in range(int(response)): | ||
| key = self._read_response( | ||
| disable_decoding=disable_decoding, timeout=timeout | ||
| disable_decoding=disable_decoding, | ||
| timeout=timeout, | ||
| _depth=_depth + 1, | ||
| ) | ||
| resp_dict[key] = self._read_response( | ||
| disable_decoding=disable_decoding, | ||
| push_request=push_request, | ||
| timeout=timeout, | ||
| _depth=_depth + 1, | ||
| ) | ||
| response = resp_dict | ||
| # push response | ||
| elif byte == b">": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| response = [ | ||
| self._read_response( | ||
| disable_decoding=disable_decoding, | ||
| push_request=push_request, | ||
| timeout=timeout, | ||
| _depth=_depth + 1, | ||
| ) | ||
| for _ in range(int(response)) | ||
| ] | ||
|
|
@@ -164,6 +195,8 @@ def _read_response( | |
|
|
||
|
|
||
| class _AsyncRESP3Parser(_AsyncRESPBase, AsyncPushNotificationsParser): | ||
| MAX_NESTING_DEPTH = 100 | ||
|
|
||
| def __init__(self, socket_read_size): | ||
| super().__init__(socket_read_size) | ||
| self.pubsub_push_handler_func = self.handle_pubsub_push_response | ||
|
|
@@ -190,7 +223,7 @@ async def read_response( | |
| return response | ||
|
|
||
| async def _read_response( | ||
| self, disable_decoding: bool = False, push_request: bool = False | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Push continuation resets nesting depthMedium Severity After handling a RESP3 push reply, the follow-up Additional Locations (1)Reviewed by Cursor Bugbot for commit 32859c2. Configure here. |
||
| self, disable_decoding: bool = False, push_request: bool = False, _depth: int = 0 | ||
| ) -> Union[EncodableT, ResponseError, None]: | ||
| if not self._stream or not self.encoder: | ||
| raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) | ||
|
|
@@ -240,36 +273,54 @@ async def _read_response( | |
| response = (await self._read(int(response)))[4:] | ||
| # array response | ||
| elif byte == b"*": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| response = [ | ||
| (await self._read_response(disable_decoding=disable_decoding)) | ||
| (await self._read_response(disable_decoding, _depth=_depth + 1)) | ||
| for _ in range(int(response)) | ||
| ] | ||
| # set response | ||
| elif byte == b"~": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| # redis can return unhashable types (like dict) in a set, | ||
| # so we always convert to a list, to have predictable return types | ||
| response = [ | ||
| (await self._read_response(disable_decoding=disable_decoding)) | ||
| (await self._read_response(disable_decoding, _depth=_depth + 1)) | ||
| for _ in range(int(response)) | ||
| ] | ||
| # map response | ||
| elif byte == b"%": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| # We cannot use a dict-comprehension to parse stream. | ||
| # Evaluation order of key:val expression in dict comprehension only | ||
| # became defined to be left-right in version 3.8 | ||
| resp_dict = {} | ||
| for _ in range(int(response)): | ||
| key = await self._read_response(disable_decoding=disable_decoding) | ||
| key = await self._read_response( | ||
| disable_decoding, _depth=_depth + 1 | ||
| ) | ||
| resp_dict[key] = await self._read_response( | ||
| disable_decoding=disable_decoding, push_request=push_request | ||
| disable_decoding, push_request=push_request, _depth=_depth + 1 | ||
| ) | ||
| response = resp_dict | ||
| # push response | ||
| elif byte == b">": | ||
| if _depth >= self.MAX_NESTING_DEPTH: | ||
| raise InvalidResponse( | ||
| f"Response nesting depth exceeded {self.MAX_NESTING_DEPTH}" | ||
| ) | ||
| response = [ | ||
| ( | ||
| await self._read_response( | ||
| disable_decoding=disable_decoding, push_request=push_request | ||
| disable_decoding, push_request=push_request, _depth=_depth + 1 | ||
| ) | ||
| ) | ||
| for _ in range(int(response)) | ||
|
|
||


There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
the push branch guards the recursion into the elements, but the tail call right below it (the "not a push_request, go read the actual reply" one) doesn't forward _depth, so it restarts at 0. a server/proxy dribbling out-of-band push frames back-to-back gets one stack frame per frame with the counter reset every time, which is the same RecursionError this PR is closing. same in _AsyncRESP3Parser.