Skip to content

Instantly share code, notes, and snippets.

@momocow
Last active November 22, 2019 07:55
Show Gist options
  • Select an option

  • Save momocow/5c6766aaa66ef9db9cd26ff762d26239 to your computer and use it in GitHub Desktop.

Select an option

Save momocow/5c6766aaa66ef9db9cd26ff762d26239 to your computer and use it in GitHub Desktop.
Monkey patch for h11. Although HTTP uses CRLF as line-breaks in headers, some implementations may still use LF. The script provides CRLF insensitive patch for ReceiveBuffer from h11.
from h11._receivebuffer import ReceiveBuffer
def monkey_patch():
def _maybe_extract_lines_crlf_insensitive(self):
if self._data[self._start : self._start + 2] == b"\r\n":
self._start += 2
return []
elif self._data[self._start : self._start + 1] == b"\n":
self._start += 1
return []
else:
data = self.maybe_extract_until_next(b"\r\n\r\n") \
or self.maybe_extract_until_next(b"\n\n")
if data is None:
# header not finish
return None
lines = []
lf_lines = data.split(b"\r\n")
for lf_line in lf_lines:
lines += lf_line.split(b"\n")
assert lines[-2] == lines[-1] == b""
del lines[-2:]
return lines
_original_ReceiveBuffer_maybe_extract_lines = ReceiveBuffer.maybe_extract_lines
ReceiveBuffer.maybe_extract_lines = _maybe_extract_lines_crlf_insensitive
def _recover():
ReceiveBuffer.maybe_extract_lines = _original_ReceiveBuffer_maybe_extract_lines
return _recover
@momocow

momocow commented Nov 21, 2019

Copy link
Copy Markdown
Author

The following header may cause error in h11 when trying to decode it since there are CRLFs and LFs.

HTTP/1.1 200 OK\r\nCache-Control: private, max-age=604800, pre-check=604800\nPragma: private\nExpires: Thu, 28 Nov 2019 06:29:17 GMT\nLast-Modified: Thu, 23 May 2019 07:32:40 GMT\nContent-Type: image/jpeg\nContent-Length: 59967\nContent-Transfer-Encoding: binary\r\n\r\n

LocalProtocolError or RemoteProtocolError may be raised with the message malformed data.

File "/.../site-packages/requests_async/api.py", line 6, in request
    return await session.request(method=method, url=url, **kwargs)
  File "/.../site-packages/requests_async/sessions.py", line 79, in request
    resp = await self.send(prep, **send_kwargs)
  File "/.../site-packages/requests_async/sessions.py", line 136, in send
    r = await adapter.send(request, **kwargs)
  File "/.../site-packages/requests_async/adapters.py", line 55, in send
    timeout=timeout,
  File "/.../site-packages/http3/interfaces.py", line 49, in request
    return await self.send(request, verify=verify, cert=cert, timeout=timeout)
  File "/.../site-packages/http3/dispatch/connection_pool.py", line 130, in send
    raise exc
  File "/.../site-packages/http3/dispatch/connection_pool.py", line 121, in send
    request, verify=verify, cert=cert, timeout=timeout
  File "/.../site-packages/http3/dispatch/connection.py", line 59, in send
    response = await self.h11_connection.send(request, timeout=timeout)
  File "/.../site-packages/http3/dispatch/http11.py", line 58, in send
    http_version, status_code, headers = await self._receive_response(timeout)
  File "/.../site-packages/http3/dispatch/http11.py", line 130, in _receive_response
    event = await self._receive_event(timeout)
  File "/.../site-packages/http3/dispatch/http11.py", line 161, in _receive_event
    event = self.h11_state.next_event()
  File "/.../site-packages/h11/_connection.py", line 439, in next_event
    exc._reraise_as_remote_protocol_error()
  File "/.../site-packages/h11/_util.py", line 72, in _reraise_as_remote_protocol_error
    raise self
  File "/.../site-packages/h11/_connection.py", line 420, in next_event
    event = self._extract_next_receive_event()
  File "/.../site-packages/h11/_connection.py", line 361, in _extract_next_receive_event
    event = self._reader(self._receive_buffer)
  File "/.../site-packages/h11/_readers.py", line 88, in maybe_read_from_SEND_RESPONSE_server
    return class_(headers=list(_decode_header_lines(lines[1:])),
  File "/.../site-packages/h11/_readers.py", line 58, in _decode_header_lines
    matches = validate(header_field_re, bytes(line))
  File "/.../site-packages/h11/_util.py", line 96, in validate
    raise LocalProtocolError(msg)
h11._util.RemoteProtocolError: malformed data

Patching the library before any HTTP requests fixes the error.

from monkey_patch_h11 import monkey_patch
recover_h11 = monkey_patch()

# use recover_h11 () to recover

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment