requests: Switch to HTTP/1.1 with Content-Length and raw streaming.

Changes:
- Send HTTP/1.1 requests (was HTTP/1.0).
- Support Content-Length response bodies without buffering in request().
- Preserve .raw as a live BodyStream wrapper; .content remains lazy.

Signed-off-by: Pablo Ventura <pablogventura@gmail.com>

Author: pablogventura

python-ecosys/requests/README.md

@@ -4,7 +4,8 @@ This module provides a lightweight version of the Python
 [requests](https://requests.readthedocs.io/en/latest/) library.
 
 It includes support for all HTTP verbs, https, json decoding of responses,
-redirects, basic authentication.
+redirects, basic authentication, HTTP/1.1 requests, and reading response
+bodies with Content-Length via streaming ``.raw`` or lazy ``.content``.
 
 ### Limitations
 
@@ -14,3 +15,10 @@ redirects, basic authentication.
 * Compressed requests/responses are not currently supported.
 * File upload is not supported.
 * Chunked encoding in responses is not supported.
+* HTTP keep-alive connection reuse is not supported (Connection: close by default).
+
+### Follow-up work
+
+* Chunked response bodies.
+* TLS certificate verification (see micropython-lib issue #838).
+* ``stream=True`` incremental body reads (see issue #777).

python-ecosys/requests/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.11.0", pypi="requests")
+metadata(version="1.0.0", pypi="requests")
 
 package("requests")

python-ecosys/requests/requests/__init__.py

@@ -1,6 +1,37 @@
 import socket
 
 
+class BodyStream:
+    def __init__(self, sock, remaining):
+        self._sock = sock
+        self._remaining = remaining
+
+    def read(self, n=-1):
+        if self._remaining == 0:
+            return b""
+        if n < 0 or n > self._remaining:
+            n = self._remaining
+        data = self._sock.read(n)
+        self._remaining -= len(data)
+        if not data:
+            raise ValueError("Connection closed before Content-Length satisfied")
+        return data
+
+    def readinto(self, buf):
+        if self._remaining == 0:
+            return 0
+        if len(buf) > self._remaining:
+            buf = memoryview(buf)[: self._remaining]
+        got = self._sock.readinto(buf)
+        self._remaining -= got
+        if not got:
+            raise ValueError("Connection closed before Content-Length satisfied")
+        return got
+
+    def close(self):
+        self._sock.close()
+
+
 class Response:
     def __init__(self, f):
         self.raw = f
@@ -104,7 +135,7 @@ def request(
             context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT)
             context.verify_mode = tls.CERT_NONE
             s = context.wrap_socket(s, server_hostname=host)
-        s.write(b"%s /%s HTTP/1.0\r\n" % (method, path))
+        s.write(b"%s /%s HTTP/1.1\r\n" % (method, path))
 
         if "Host" not in headers:
             headers["Host"] = host
@@ -161,6 +192,7 @@ def request(
         reason = ""
         if len(l) > 2:
             reason = l[2].rstrip()
+        remaining = None
         while True:
             l = s.readline()
             if not l or l == b"\r\n":
@@ -179,7 +211,10 @@ def request(
             elif parse_headers is True:
                 l = str(l, "utf-8")
                 k, v = l.split(":", 1)
-                resp_d[k] = v.strip()
+                v = v.strip()
+                resp_d[k] = v
+                if k.lower() == "content-length":
+                    remaining = int(v)
             else:
                 parse_headers(l, resp_d)
     except OSError:
@@ -195,7 +230,10 @@ def request(
         else:
             return request(method, redirect, data, json, headers, stream)
     else:
-        resp = Response(s)
+        if remaining is not None:
+            resp = Response(BodyStream(s, remaining))
+        else:
+            resp = Response(s)
         resp.status_code = status
         resp.reason = reason
         if resp_d is not None:

python-ecosys/requests/test_requests.py

@@ -3,9 +3,9 @@
 
 
 class Socket:
-    def __init__(self):
+    def __init__(self, read_data=b"HTTP/1.1 200 OK\r\n\r\n"):
         self._write_buffer = io.BytesIO()
-        self._read_buffer = io.BytesIO(b"HTTP/1.0 200 OK\r\n\r\n")
+        self._read_buffer = io.BytesIO(read_data)
 
     def connect(self, address):
         pass
@@ -16,6 +16,15 @@ def write(self, buf):
     def readline(self):
         return self._read_buffer.readline()
 
+    def read(self, size=-1):
+        return self._read_buffer.read(size)
+
+    def readinto(self, buf):
+        return self._read_buffer.readinto(buf)
+
+    def close(self):
+        pass
+
 
 class socket:
     AF_INET = 2
@@ -43,7 +52,7 @@ def test_simple_get():
     response = requests.request("GET", "http://example.com")
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n" + b"Connection: close\r\n" + b"Host: example.com\r\n\r\n"
+        b"GET / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Host: example.com\r\n\r\n"
     ), format_message(response)
 
 
@@ -53,7 +62,7 @@ def test_get_auth():
     )
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n"
+        b"GET / HTTP/1.1\r\n"
         + b"Host: example.com\r\n"
         + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n"
         + b"Connection: close\r\n\r\n"
@@ -64,7 +73,7 @@ def test_get_custom_header():
     response = requests.request("GET", "http://example.com", headers={"User-Agent": "test-agent"})
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n"
+        b"GET / HTTP/1.1\r\n"
         + b"User-Agent: test-agent\r\n"
         + b"Host: example.com\r\n"
         + b"Connection: close\r\n\r\n"
@@ -75,7 +84,7 @@ def test_post_json():
     response = requests.request("GET", "http://example.com", json="test")
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n"
+        b"GET / HTTP/1.1\r\n"
         + b"Connection: close\r\n"
         + b"Content-Type: application/json\r\n"
         + b"Host: example.com\r\n"
@@ -91,7 +100,7 @@ def chunks():
     response = requests.request("GET", "http://example.com", data=chunks())
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n"
+        b"GET / HTTP/1.1\r\n"
         + b"Transfer-Encoding: chunked\r\n"
         + b"Host: example.com\r\n"
         + b"Connection: close\r\n\r\n"
@@ -106,7 +115,7 @@ def test_overwrite_get_headers():
     )
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n" + b"Connection: keep-alive\r\n" + b"Host: test.com\r\n\r\n"
+        b"GET / HTTP/1.1\r\n" + b"Connection: keep-alive\r\n" + b"Host: test.com\r\n\r\n"
     ), format_message(response)
 
 
@@ -119,7 +128,7 @@ def test_overwrite_post_json_headers():
     )
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n"
+        b"GET / HTTP/1.1\r\n"
         + b"Connection: close\r\n"
         + b"Content-Length: 10\r\n"
         + b"Content-Type: text/plain\r\n"
@@ -137,7 +146,7 @@ def chunks():
     )
 
     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n"
+        b"GET / HTTP/1.1\r\n"
         + b"Host: example.com\r\n"
         + b"Content-Length: 4\r\n"
         + b"Connection: close\r\n\r\n"
@@ -153,6 +162,70 @@ def test_do_not_modify_headers_argument():
     assert do_not_modify_this_dict == {}, do_not_modify_this_dict
 
 
+def test_content_length_via_content():
+    socket.socket = lambda *a, **k: Socket(
+        read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello"
+    )
+    response = requests.request("GET", "http://example.com")
+    assert response.content == b"hello"
+    assert response.headers["Content-Length"] == "5"
+    socket.socket = lambda *a, **k: Socket()
+
+
+def test_chunked_response_raises():
+    socket.socket = lambda *a, **k: Socket(
+        read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"
+    )
+    raised = False
+    try:
+        requests.request("GET", "http://example.com")
+    except ValueError as e:
+        raised = True
+        if "Unsupported" not in str(e):
+            raise
+    if not raised:
+        raise AssertionError("expected ValueError for chunked response")
+    socket.socket = lambda *a, **k: Socket()
+
+
+def test_raw_open_before_content():
+    socket.socket = lambda *a, **k: Socket(
+        read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello"
+    )
+    response = requests.request("GET", "http://example.com")
+    assert response.raw is not None
+    assert response.raw.read(1) == b"h"
+    socket.socket = lambda *a, **k: Socket()
+
+
+def test_raw_incremental_content_length():
+    socket.socket = lambda *a, **k: Socket(
+        read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nabcdefghij"
+    )
+    response = requests.request("GET", "http://example.com")
+    assert response.raw.read(3) == b"abc"
+    assert response.raw.read(3) == b"def"
+    assert response.content == b"ghij"
+    assert response.raw is None
+    socket.socket = lambda *a, **k: Socket()
+
+
+def test_raw_readinto_content_length():
+    socket.socket = lambda *a, **k: Socket(
+        read_data=b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nabcdefghij"
+    )
+    response = requests.request("GET", "http://example.com")
+    buf = bytearray(3)
+    result = b""
+    while True:
+        n = response.raw.readinto(buf)
+        if n == 0:
+            break
+        result += buf if n == 3 else buf[:n]
+    assert result == b"abcdefghij"
+    socket.socket = lambda *a, **k: Socket()
+
+
 test_simple_get()
 test_get_auth()
 test_get_custom_header()
@@ -162,3 +235,8 @@ def test_do_not_modify_headers_argument():
 test_overwrite_post_json_headers()
 test_overwrite_post_chunked_data_headers()
 test_do_not_modify_headers_argument()
+test_content_length_via_content()
+test_chunked_response_raises()
+test_raw_open_before_content()
+test_raw_incremental_content_length()
+test_raw_readinto_content_length()