From 52c5f8dca30afefb9ccfda4141d5c276df09b63a Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Sun, 9 Aug 2026 13:46:56 +0200 Subject: [PATCH 1/4] Speed up multipart callback dispatch --- CHANGELOG.md | 1 + python_multipart/multipart.py | 75 +++++++++++++++++++++++++---------- tests/test_multipart.py | 18 +++++++++ 3 files changed, 72 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aeb17fe..a43e2ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* Speed up multipart callback dispatch. * Speed up querystring callback dispatch [#316](https://github.com/Kludex/python-multipart/pull/316). ## 0.0.32 (2026-06-04) diff --git a/python_multipart/multipart.py b/python_multipart/multipart.py index 49cdd8e..2526850 100644 --- a/python_multipart/multipart.py +++ b/python_multipart/multipart.py @@ -1119,6 +1119,35 @@ def _internal_write(self, data: bytes, length: int) -> int: current_header_count = self._current_header_count current_header_size = self._current_header_size + callbacks = cast("MultipartCallbacks", self.callbacks) + on_part_begin = callbacks.get("on_part_begin") + on_part_data = callbacks.get("on_part_data") + on_part_end = callbacks.get("on_part_end") + on_header_begin = callbacks.get("on_header_begin") + on_header_field = callbacks.get("on_header_field") + on_header_value = callbacks.get("on_header_value") + on_header_end = callbacks.get("on_header_end") + on_headers_finished = callbacks.get("on_headers_finished") + on_end = callbacks.get("on_end") + if on_part_begin is None: + on_part_begin = _noop_event + if on_part_data is None: + on_part_data = _noop_data + if on_part_end is None: + on_part_end = _noop_event + if on_header_begin is None: + on_header_begin = _noop_event + if on_header_field is None: + on_header_field = _noop_data + if on_header_value is None: + on_header_value = _noop_data + if on_header_end is None: + on_header_end = _noop_event + if on_headers_finished is None: + on_headers_finished = _noop_event + if on_end is None: + on_end = _noop_event + # Our index defaults to 0. i = 0 @@ -1141,7 +1170,9 @@ def delete_mark(name: str, reset: bool = False) -> None: # end of the buffer, and reset the mark, instead of deleting it. This # is used at the end of the function to call our callbacks with any # remaining data in this chunk. - def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> None: + def data_callback( + name: CallbackName, callback: Callable[[bytes, int, int], None], end_i: int, remaining: bool = False + ) -> None: marked_index = self.marks.get(name) if marked_index is None: return @@ -1153,7 +1184,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No pass elif marked_index >= 0: # We are emitting data from the local buffer. - self.callback(name, data, marked_index, end_i) + callback(data, marked_index, end_i) else: # Some of the data comes from a partial boundary match. # and requires look-behind. @@ -1161,18 +1192,18 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # the state when we entered the loop. lookbehind_len = -marked_index if lookbehind_len <= boundary_length: - self.callback(name, boundary, 0, lookbehind_len) + callback(boundary, 0, lookbehind_len) elif self.flags & FLAG_PART_BOUNDARY: lookback = boundary + b"\r\n" - self.callback(name, lookback, 0, lookbehind_len) + callback(lookback, 0, lookbehind_len) elif self.flags & FLAG_LAST_BOUNDARY: lookback = boundary + b"--\r\n" - self.callback(name, lookback, 0, lookbehind_len) + callback(lookback, 0, lookbehind_len) else: # pragma: no cover (error case) self.logger.warning("Look-back buffer error") if end_i > 0: - self.callback(name, data, 0, end_i) + callback(data, 0, end_i) # If we're getting remaining data, we have got all the data we # can be certain is not a boundary, leaving only a partial boundary match. if remaining: @@ -1227,7 +1258,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No index = 0 # Callback for the start of a part. - self.callback("part_begin") + on_part_begin() current_header_count = 0 current_header_size = 0 @@ -1263,7 +1294,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # to stop parsing headers in the MultipartState.HEADER_FIELD state, # below. if c != CR: - self.callback("header_begin") + on_header_begin() # Move to parsing header fields. state = MultipartState.HEADER_FIELD @@ -1309,7 +1340,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # Call our callback with the header field. i = colon - data_callback("header_field", i) + data_callback("header_field", on_header_field, i) # Move to parsing the header value. state = MultipartState.HEADER_VALUE_START @@ -1336,8 +1367,8 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No advance_header_size(end - i) if cr != -1: i = cr - data_callback("header_value", i) - self.callback("header_end") + data_callback("header_value", on_header_value, i) + on_header_end() current_header_size = 0 state = MultipartState.HEADER_VALUE_ALMOST_DONE else: @@ -1364,7 +1395,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No self.logger.warning(msg) raise MultipartParseError(msg, offset=i) - self.callback("headers_finished") + on_headers_finished() state = MultipartState.PART_DATA_START elif state == MultipartState.PART_DATA_START: @@ -1451,11 +1482,11 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No flags &= ~FLAG_PART_BOUNDARY # We have identified a boundary, callback for any data before it. - data_callback("part_data", i - index) + data_callback("part_data", on_part_data, i - index) # Callback indicating that we've reached the end of # a part, and are starting a new one. - self.callback("part_end") - self.callback("part_begin") + on_part_end() + on_part_begin() current_header_count = 0 current_header_size = 0 @@ -1476,11 +1507,11 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # We need a second hyphen here. if c == HYPHEN: # We have identified a boundary, callback for any data before it. - data_callback("part_data", i - index) + data_callback("part_data", on_part_data, i - index) # Callback to end the current part, and then the # message. - self.callback("part_end") - self.callback("end") + on_part_end() + on_end() state = MultipartState.END else: # No match, so reset index. @@ -1505,7 +1536,7 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No self.logger.warning(msg) raise MultipartParseError(msg, offset=i) index += 1 - self.callback("end") + on_end() state = MultipartState.END elif state == MultipartState.END: @@ -1531,9 +1562,9 @@ def data_callback(name: CallbackName, end_i: int, remaining: bool = False) -> No # that we haven't yet reached the end of this 'thing'. So, by setting # the mark to 0, we cause any data callbacks that take place in future # calls to this function to start from the beginning of that buffer. - data_callback("header_field", length, True) - data_callback("header_value", length, True) - data_callback("part_data", length - index, True) + data_callback("header_field", on_header_field, length, True) + data_callback("header_value", on_header_value, length, True) + data_callback("part_data", on_part_data, length - index, True) # Save values to locals. self.state = state diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 949d70c..daa3d0a 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1592,6 +1592,24 @@ def test_invalid_max_size_multipart(self) -> None: with self.assertRaises(ValueError): MultipartParser(b"bound", max_size="foo") # type: ignore[arg-type] + def test_multipart_none_callbacks(self) -> None: + callbacks: Any = { + "on_part_begin": None, + "on_part_data": None, + "on_part_end": None, + "on_header_begin": None, + "on_header_field": None, + "on_header_value": None, + "on_header_end": None, + "on_headers_finished": None, + "on_end": None, + } + parser = MultipartParser(b"boundary", callbacks) + data = b"--boundary\r\nX: y\r\n\r\nbody\r\n--boundary--\r\n" + + self.assertEqual(parser.write(data), len(data)) + parser.finalize() + def test_boundary_too_long(self) -> None: with self.assertRaisesRegex(FormParserError, "Boundary length 257 exceeds maximum of 256"): MultipartParser(b"x" * 257) From c198b75e27233befeb907e1f9306a02b3a295504 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Sun, 9 Aug 2026 13:47:17 +0200 Subject: [PATCH 2/4] Link multipart optimization changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a43e2ae..f4d5849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -* Speed up multipart callback dispatch. +* Speed up multipart callback dispatch [#317](https://github.com/Kludex/python-multipart/pull/317). * Speed up querystring callback dispatch [#316](https://github.com/Kludex/python-multipart/pull/316). ## 0.0.32 (2026-06-04) From ef5fbc90e197f6153805f04e5144d5f8e86cd5c0 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Sun, 9 Aug 2026 13:54:45 +0200 Subject: [PATCH 3/4] Cache multipart callbacks on the parser --- python_multipart/multipart.py | 89 +++++++++++++++++------------------ tests/test_multipart.py | 21 +++++++++ 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/python_multipart/multipart.py b/python_multipart/multipart.py index 2526850..2ec4ec7 100644 --- a/python_multipart/multipart.py +++ b/python_multipart/multipart.py @@ -1048,6 +1048,24 @@ def __init__( self.index = self.flags = 0 self.callbacks = callbacks + on_part_begin = callbacks.get("on_part_begin") + on_part_data = callbacks.get("on_part_data") + on_part_end = callbacks.get("on_part_end") + on_header_begin = callbacks.get("on_header_begin") + on_header_field = callbacks.get("on_header_field") + on_header_value = callbacks.get("on_header_value") + on_header_end = callbacks.get("on_header_end") + on_headers_finished = callbacks.get("on_headers_finished") + on_end = callbacks.get("on_end") + self._on_part_begin = _noop_event if on_part_begin is None else on_part_begin + self._on_part_data = _noop_data if on_part_data is None else on_part_data + self._on_part_end = _noop_event if on_part_end is None else on_part_end + self._on_header_begin = _noop_event if on_header_begin is None else on_header_begin + self._on_header_field = _noop_data if on_header_field is None else on_header_field + self._on_header_value = _noop_data if on_header_value is None else on_header_value + self._on_header_end = _noop_event if on_header_end is None else on_header_end + self._on_headers_finished = _noop_event if on_headers_finished is None else on_headers_finished + self._on_end = _noop_event if on_end is None else on_end if not isinstance(max_size, Number) or max_size < 1: raise ValueError("max_size must be a positive number, not %r" % max_size) @@ -1070,6 +1088,16 @@ def __init__( raise FormParserError(f"Boundary length {len(boundary)} exceeds maximum of {MAX_BOUNDARY_LENGTH}") self.boundary = b"\r\n--" + boundary + def set_callback(self, name: CallbackName, new_func: Callable[..., Any] | None) -> None: + super().set_callback(name, new_func) + if name in ("part_data", "header_field", "header_value"): + callback = _noop_data if new_func is None else new_func + elif name in ("part_begin", "part_end", "header_begin", "header_end", "headers_finished", "end"): + callback = _noop_event if new_func is None else new_func + else: + return + setattr(self, "_on_" + name, callback) + def write(self, data: bytes) -> int: """Write some data to the parser, which will perform size verification, and then parse the data into the appropriate location (e.g. header, @@ -1119,35 +1147,6 @@ def _internal_write(self, data: bytes, length: int) -> int: current_header_count = self._current_header_count current_header_size = self._current_header_size - callbacks = cast("MultipartCallbacks", self.callbacks) - on_part_begin = callbacks.get("on_part_begin") - on_part_data = callbacks.get("on_part_data") - on_part_end = callbacks.get("on_part_end") - on_header_begin = callbacks.get("on_header_begin") - on_header_field = callbacks.get("on_header_field") - on_header_value = callbacks.get("on_header_value") - on_header_end = callbacks.get("on_header_end") - on_headers_finished = callbacks.get("on_headers_finished") - on_end = callbacks.get("on_end") - if on_part_begin is None: - on_part_begin = _noop_event - if on_part_data is None: - on_part_data = _noop_data - if on_part_end is None: - on_part_end = _noop_event - if on_header_begin is None: - on_header_begin = _noop_event - if on_header_field is None: - on_header_field = _noop_data - if on_header_value is None: - on_header_value = _noop_data - if on_header_end is None: - on_header_end = _noop_event - if on_headers_finished is None: - on_headers_finished = _noop_event - if on_end is None: - on_end = _noop_event - # Our index defaults to 0. i = 0 @@ -1258,7 +1257,7 @@ def data_callback( index = 0 # Callback for the start of a part. - on_part_begin() + self._on_part_begin() current_header_count = 0 current_header_size = 0 @@ -1294,7 +1293,7 @@ def data_callback( # to stop parsing headers in the MultipartState.HEADER_FIELD state, # below. if c != CR: - on_header_begin() + self._on_header_begin() # Move to parsing header fields. state = MultipartState.HEADER_FIELD @@ -1340,7 +1339,7 @@ def data_callback( # Call our callback with the header field. i = colon - data_callback("header_field", on_header_field, i) + data_callback("header_field", self._on_header_field, i) # Move to parsing the header value. state = MultipartState.HEADER_VALUE_START @@ -1367,8 +1366,8 @@ def data_callback( advance_header_size(end - i) if cr != -1: i = cr - data_callback("header_value", on_header_value, i) - on_header_end() + data_callback("header_value", self._on_header_value, i) + self._on_header_end() current_header_size = 0 state = MultipartState.HEADER_VALUE_ALMOST_DONE else: @@ -1395,7 +1394,7 @@ def data_callback( self.logger.warning(msg) raise MultipartParseError(msg, offset=i) - on_headers_finished() + self._on_headers_finished() state = MultipartState.PART_DATA_START elif state == MultipartState.PART_DATA_START: @@ -1482,11 +1481,11 @@ def data_callback( flags &= ~FLAG_PART_BOUNDARY # We have identified a boundary, callback for any data before it. - data_callback("part_data", on_part_data, i - index) + data_callback("part_data", self._on_part_data, i - index) # Callback indicating that we've reached the end of # a part, and are starting a new one. - on_part_end() - on_part_begin() + self._on_part_end() + self._on_part_begin() current_header_count = 0 current_header_size = 0 @@ -1507,11 +1506,11 @@ def data_callback( # We need a second hyphen here. if c == HYPHEN: # We have identified a boundary, callback for any data before it. - data_callback("part_data", on_part_data, i - index) + data_callback("part_data", self._on_part_data, i - index) # Callback to end the current part, and then the # message. - on_part_end() - on_end() + self._on_part_end() + self._on_end() state = MultipartState.END else: # No match, so reset index. @@ -1536,7 +1535,7 @@ def data_callback( self.logger.warning(msg) raise MultipartParseError(msg, offset=i) index += 1 - on_end() + self._on_end() state = MultipartState.END elif state == MultipartState.END: @@ -1562,9 +1561,9 @@ def data_callback( # that we haven't yet reached the end of this 'thing'. So, by setting # the mark to 0, we cause any data callbacks that take place in future # calls to this function to start from the beginning of that buffer. - data_callback("header_field", on_header_field, length, True) - data_callback("header_value", on_header_value, length, True) - data_callback("part_data", on_part_data, length - index, True) + data_callback("header_field", self._on_header_field, length, True) + data_callback("header_value", self._on_header_value, length, True) + data_callback("part_data", self._on_part_data, length - index, True) # Save values to locals. self.state = state diff --git a/tests/test_multipart.py b/tests/test_multipart.py index daa3d0a..16c5a5e 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1592,6 +1592,27 @@ def test_invalid_max_size_multipart(self) -> None: with self.assertRaises(ValueError): MultipartParser(b"bound", max_size="foo") # type: ignore[arg-type] + def test_multipart_set_callback(self) -> None: + header_begins = 0 + header_fields: list[bytes] = [] + + def on_header_begin() -> None: + nonlocal header_begins + header_begins += 1 + + def on_header_field(data: bytes, start: int, end: int) -> None: + header_fields.append(data[start:end]) + + parser = MultipartParser(b"boundary") + parser.set_callback("header_begin", on_header_begin) + parser.set_callback("header_field", on_header_field) + parser.set_callback("field_start", None) + data = b"--boundary\r\nX: y\r\n\r\nbody\r\n--boundary--\r\n" + parser.write(data) + + self.assertEqual(header_begins, 1) + self.assertEqual(header_fields, [b"X"]) + def test_multipart_none_callbacks(self) -> None: callbacks: Any = { "on_part_begin": None, From d74fe571fc7a9e3c88a4c88de9b9c5955e257e82 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Sun, 9 Aug 2026 14:29:21 +0200 Subject: [PATCH 4/4] Test disabling a multipart callback --- python_multipart/multipart.py | 5 ++--- tests/test_multipart.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python_multipart/multipart.py b/python_multipart/multipart.py index 2ec4ec7..5f20891 100644 --- a/python_multipart/multipart.py +++ b/python_multipart/multipart.py @@ -1092,11 +1092,10 @@ def set_callback(self, name: CallbackName, new_func: Callable[..., Any] | None) super().set_callback(name, new_func) if name in ("part_data", "header_field", "header_value"): callback = _noop_data if new_func is None else new_func + setattr(self, "_on_" + name, callback) elif name in ("part_begin", "part_end", "header_begin", "header_end", "headers_finished", "end"): callback = _noop_event if new_func is None else new_func - else: - return - setattr(self, "_on_" + name, callback) + setattr(self, "_on_" + name, callback) def write(self, data: bytes) -> int: """Write some data to the parser, which will perform size verification, diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 16c5a5e..4c83334 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1606,7 +1606,7 @@ def on_header_field(data: bytes, start: int, end: int) -> None: parser = MultipartParser(b"boundary") parser.set_callback("header_begin", on_header_begin) parser.set_callback("header_field", on_header_field) - parser.set_callback("field_start", None) + parser.set_callback("part_data", None) data = b"--boundary\r\nX: y\r\n\r\nbody\r\n--boundary--\r\n" parser.write(data)