Tests: parsing of "Transfer-Encoding: chunked" responses.

6 files changed, 71 insertions, 23 deletions

diff --git a/test/test_node_application.py b/test/test_node_application.py
index a5b4a108..b80d17d3 100644
--- a/test/test_node_application.py
+++ b/test/test_node_application.py
@@ -141,7 +141,7 @@ class TestNodeApplication(TestApplicationNode):
         self.load('write_buffer')
 
         self.assertEqual(
-            self.get()['body'], '6\r\nbuffer\r\n0\r\n\r\n', 'write buffer'
+            self.get()['body'], 'buffer', 'write buffer'
         )
 
     def test_node_application_write_callback(self):
@@ -149,7 +149,7 @@ class TestNodeApplication(TestApplicationNode):
 
         self.assertEqual(
             self.get()['body'],
-            '5\r\nhello\r\n5\r\nworld\r\n0\r\n\r\n',
+            'helloworld',
             'write callback order',
         )
         self.assertTrue(
@@ -173,7 +173,7 @@ class TestNodeApplication(TestApplicationNode):
 
         self.assertEqual(
             self.get()['body'],
-            '4\r\nbody\r\n4\r\ntrue\r\n0\r\n\r\n',
+            'bodytrue',
             'write return',
         )
 
diff --git a/test/test_perl_application.py b/test/test_perl_application.py
index bf3c65d5..a4bac623 100644
--- a/test/test_perl_application.py
+++ b/test/test_perl_application.py
@@ -157,7 +157,7 @@ class TestPerlApplication(TestApplicationPerl):
     def test_perl_application_body_empty(self):
         self.load('body_empty')
 
-        self.assertEqual(self.get()['body'], '0\r\n\r\n', 'body empty')
+        self.assertEqual(self.get()['body'], '', 'body empty')
 
     def test_perl_application_body_array(self):
         self.load('body_array')
diff --git a/test/test_python_application.py b/test/test_python_application.py
index ae8f01ca..5e1ba65a 100644
--- a/test/test_python_application.py
+++ b/test/test_python_application.py
@@ -540,7 +540,7 @@ Connection: close
             }
         )
         self.assertEqual(resp['status'], 200, 'status')
-        self.assertEqual(resp['body'][-5:], '0\r\n\r\n', 'body')
+        self.assertEqual(resp['body'], 'XXXXXXX', 'body')
 
         # Exception before start_response().
 
@@ -607,12 +607,11 @@ Connection: close
                 'X-Skip': '2',
                 'X-Chunked': '1',
                 'Connection': 'close',
-            }
+            },
+            raw_resp=True
         )
-        if 'body' in resp:
-            self.assertNotEqual(
-                resp['body'][-5:], '0\r\n\r\n', 'incomplete body'
-            )
+        if resp:
+            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body')
         self.assertEqual(
             len(self.findall(r'Traceback')), 4, 'traceback count 4'
         )
@@ -646,12 +645,11 @@ Connection: close
                 'X-Skip': '3',
                 'X-Chunked': '1',
                 'Connection': 'close',
-            }
+            },
+            raw_resp=True
         )
-        if 'body' in resp:
-            self.assertNotEqual(
-                resp['body'][-5:], '0\r\n\r\n', 'incomplete body 2'
-            )
+        if resp:
+            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body 2')
         self.assertEqual(
             len(self.findall(r'Traceback')), 6, 'traceback count 6'
         )
diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py
index bbb252d7..83a71f96 100644
--- a/test/test_ruby_application.py
+++ b/test/test_ruby_application.py
@@ -285,7 +285,7 @@ class TestRubyApplication(TestApplicationRuby):
     def test_ruby_application_body_empty(self):
         self.load('body_empty')
 
-        self.assertEqual(self.get()['body'], '0\r\n\r\n', 'body empty')
+        self.assertEqual(self.get()['body'], '', 'body empty')
 
     def test_ruby_application_body_array(self):
         self.load('body_array')
diff --git a/test/unit/feature/isolation.py b/test/unit/feature/isolation.py
index 9b06ab3c..6a429fb1 100644
--- a/test/unit/feature/isolation.py
+++ b/test/unit/feature/isolation.py
@@ -84,4 +84,4 @@ class TestFeatureIsolation(TestApplicationProto):
         return data
 
     def parsejson(self, data):
-        return json.loads(data.split('\n')[1])
+        return json.loads(data)
diff --git a/test/unit/http.py b/test/unit/http.py
index c7e3e36d..d59c7b56 100644
--- a/test/unit/http.py
+++ b/test/unit/http.py
@@ -1,5 +1,6 @@
 import re
 import time
+import json
 import socket
 import select
 from unit.main import TestUnit
@@ -86,23 +87,24 @@ class TestHTTP(TestUnit):
 
         sock.sendall(req)
 
+        encoding = 'utf-8' if 'encoding' not in kwargs else kwargs['encoding']
+
         if TestUnit.detailed:
             print('>>>')
             try:
-                print(req.decode('utf-8', 'ignore'))
+                print(req.decode(encoding, 'ignore'))
             except UnicodeEncodeError:
                 print(req)
 
         resp = ''
 
         if 'no_recv' not in kwargs:
-            enc = 'utf-8' if 'encoding' not in kwargs else kwargs['encoding']
             read_timeout = (
                 30 if 'read_timeout' not in kwargs else kwargs['read_timeout']
             )
             resp = self.recvall(
                 sock, read_timeout=read_timeout, buff_size=read_buffer_size
-            ).decode(enc)
+            ).decode(encoding)
 
         if TestUnit.detailed:
             print('<<<')
@@ -114,6 +116,12 @@ class TestHTTP(TestUnit):
         if 'raw_resp' not in kwargs:
             resp = self._resp_to_dict(resp)
 
+            headers = resp.get('headers')
+            if headers and headers.get('Transfer-Encoding') == 'chunked':
+                resp['body'] = self._parse_chunked_body(resp['body']).decode(
+                    encoding
+                )
+
         if 'start' not in kwargs:
             sock.close()
             return resp
@@ -151,7 +159,7 @@ class TestHTTP(TestUnit):
         return data
 
     def _resp_to_dict(self, resp):
-        m = re.search('(.*?\x0d\x0a?)\x0d\x0a?(.*)', resp, re.M | re.S)
+        m = re.search(r'(.*?\x0d\x0a?)\x0d\x0a?(.*)', resp, re.M | re.S)
 
         if not m:
             return {}
@@ -162,12 +170,12 @@ class TestHTTP(TestUnit):
         headers_lines = p.findall(headers_text)
 
         status = re.search(
-            '^HTTP\/\d\.\d\s(\d+)|$', headers_lines.pop(0)
+            r'^HTTP\/\d\.\d\s(\d+)|$', headers_lines.pop(0)
         ).group(1)
 
         headers = {}
         for line in headers_lines:
-            m = re.search('(.*)\:\s(.*)', line)
+            m = re.search(r'(.*)\:\s(.*)', line)
 
             if m.group(1) not in headers:
                 headers[m.group(1)] = m.group(2)
@@ -180,6 +188,48 @@ class TestHTTP(TestUnit):
 
         return {'status': int(status), 'headers': headers, 'body': body}
 
+    def _parse_chunked_body(self, raw_body):
+        if isinstance(raw_body, str):
+            raw_body = bytes(raw_body.encode())
+
+        crlf = b'\r\n'
+        chunks = raw_body.split(crlf)
+
+        if len(chunks) < 3:
+            self.fail('Invalid chunked body')
+
+        if chunks.pop() != b'':
+            self.fail('No CRLF at the end of the body')
+
+        try:
+            last_size = int(chunks[-2], 16)
+        except:
+            self.fail('Invalid zero size chunk')
+
+        if last_size != 0 or chunks[-1] != b'':
+            self.fail('Incomplete body')
+
+        body = b''
+        while len(chunks) >= 2:
+            try:
+                size = int(chunks.pop(0), 16)
+            except:
+                self.fail('Invalid chunk size %s' % str(size))
+
+            if size == 0:
+                self.assertEqual(len(chunks), 1, 'last zero size')
+                break
+
+            temp_body = crlf.join(chunks)
+
+            body += temp_body[:size]
+
+            temp_body = temp_body[size + len(crlf) :]
+
+            chunks = temp_body.split(crlf)
+
+        return body
+
     def waitforsocket(self, port):
         ret = False