http: Support chunked request bodies

This is a temporary support for chunked request bodies by converting
to Content-Length. This allows for processing of such requests until
a more permanent solution is developed.

A new configuration option "chunked_transform" has been added to enable
this feature. The option can be set as follows:
  {
    "settings": {
      "chunked_transform": true
    }
  }
By default, this option is set to false, which retains the current
behaviour of rejecting chunked requests with a '411 Length Required'
status code.

Please note that this is an experimental implementation.

Reviewed-by: Andrew Clayton <a.clayton@nginx.com>

1 files changed, 135 insertions, 38 deletions

diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c
index fd184eb5..5d1ed790 100644
--- a/src/nxt_h1proto.c
+++ b/src/nxt_h1proto.c
@@ -839,7 +839,12 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data)
     if (field->value_length == 7
         && memcmp(field->value, "chunked", 7) == 0)
     {
+        if (r->chunked_field != NULL) {
+            return NXT_HTTP_BAD_REQUEST;
+        }
+
         te = NXT_HTTP_TE_CHUNKED;
+        r->chunked_field = field;
 
     } else {
         te = NXT_HTTP_TE_UNSUPPORTED;
@@ -856,14 +861,16 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
 {
     size_t             size, body_length, body_buffer_size, body_rest;
     ssize_t            res;
-    nxt_buf_t          *in, *b;
+    nxt_buf_t          *in, *b, *out, *chunk;
     nxt_conn_t         *c;
     nxt_h1proto_t      *h1p;
+    nxt_socket_conf_t  *skcf;
     nxt_http_status_t  status;
 
     static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX");
 
     h1p = r->proto.h1;
+    skcf = r->conf->socket_conf;
 
     nxt_debug(task, "h1p request body read %O te:%d",
               r->content_length_n, h1p->transfer_encoding);
@@ -871,8 +878,19 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
     switch (h1p->transfer_encoding) {
 
     case NXT_HTTP_TE_CHUNKED:
-        status = NXT_HTTP_LENGTH_REQUIRED;
-        goto error;
+        if (!skcf->chunked_transform) {
+            status = NXT_HTTP_LENGTH_REQUIRED;
+            goto error;
+        }
+
+        if (r->content_length != NULL || !nxt_h1p_is_http11(h1p)) {
+            status = NXT_HTTP_BAD_REQUEST;
+            goto error;
+        }
+
+        r->chunked = 1;
+        h1p->chunked_parse.mem_pool = r->mem_pool;
+        break;
 
     case NXT_HTTP_TE_UNSUPPORTED:
         status = NXT_HTTP_NOT_IMPLEMENTED;
@@ -883,19 +901,20 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
         break;
     }
 
-    if (r->content_length_n == -1 || r->content_length_n == 0) {
+    if (!r->chunked &&
+        (r->content_length_n == -1 || r->content_length_n == 0))
+    {
         goto ready;
     }
 
     body_length = (size_t) r->content_length_n;
 
-    body_buffer_size = nxt_min(r->conf->socket_conf->body_buffer_size,
-                               body_length);
+    body_buffer_size = nxt_min(skcf->body_buffer_size, body_length);
 
     if (body_length > body_buffer_size) {
         nxt_str_t  *tmp_path, tmp_name;
 
-        tmp_path = &r->conf->socket_conf->body_temp_path;
+        tmp_path = &skcf->body_temp_path;
 
         tmp_name.length = tmp_path->length + tmp_name_pattern.length;
 
@@ -946,31 +965,69 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
 
     r->body = b;
 
-    body_rest = body_length;
+    body_rest = r->chunked ? 1 : body_length;
 
     in = h1p->conn->read;
 
     size = nxt_buf_mem_used_size(&in->mem);
 
     if (size != 0) {
-        size = nxt_min(size, body_length);
-
         if (nxt_buf_is_file(b)) {
-            res = nxt_fd_write(b->file->fd, in->mem.pos, size);
-            if (nxt_slow_path(res < (ssize_t) size)) {
-                status = NXT_HTTP_INTERNAL_SERVER_ERROR;
-                goto error;
-            }
+            if (r->chunked) {
+                out = nxt_http_chunk_parse(task, &h1p->chunked_parse, in);
 
-            b->file_end += size;
+                if (h1p->chunked_parse.error) {
+                    status = NXT_HTTP_INTERNAL_SERVER_ERROR;
+                    goto error;
+                }
+
+                if (h1p->chunked_parse.chunk_error) {
+                    status = NXT_HTTP_BAD_REQUEST;
+                    goto error;
+                }
+
+                for (chunk = out; chunk != NULL; chunk = chunk->next) {
+                    size = nxt_buf_mem_used_size(&chunk->mem);
+
+                    res = nxt_fd_write(b->file->fd, chunk->mem.pos, size);
+                    if (nxt_slow_path(res < (ssize_t) size)) {
+                        status = NXT_HTTP_INTERNAL_SERVER_ERROR;
+                        goto error;
+                    }
+
+                    b->file_end += size;
+
+                    if ((size_t) b->file_end > skcf->max_body_size) {
+                        status = NXT_HTTP_PAYLOAD_TOO_LARGE;
+                        goto error;
+                    }
+                }
+
+                if (h1p->chunked_parse.last) {
+                    body_rest = 0;
+                }
+
+            } else {
+                size = nxt_min(size, body_length);
+                res = nxt_fd_write(b->file->fd, in->mem.pos, size);
+                if (nxt_slow_path(res < (ssize_t) size)) {
+                    status = NXT_HTTP_INTERNAL_SERVER_ERROR;
+                    goto error;
+                }
+
+                b->file_end += size;
+
+                in->mem.pos += size;
+                body_rest -= size;
+            }
 
         } else {
             size = nxt_min(body_buffer_size, size);
             b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size);
-        }
 
-        in->mem.pos += size;
-        body_rest -= size;
+            in->mem.pos += size;
+            body_rest -= size;
+        }
     }
 
     nxt_debug(task, "h1p body rest: %uz", body_rest);
@@ -1028,9 +1085,10 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data)
 {
     size_t              size, body_rest;
     ssize_t             res;
-    nxt_buf_t           *b;
+    nxt_buf_t           *b, *out, *chunk;
     nxt_conn_t          *c;
     nxt_h1proto_t       *h1p;
+    nxt_socket_conf_t   *skcf;
     nxt_http_request_t  *r;
     nxt_event_engine_t  *engine;
 
@@ -1040,38 +1098,77 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data)
     nxt_debug(task, "h1p conn request body read");
 
     r = h1p->request;
+    skcf = r->conf->socket_conf;
 
     engine = task->thread->engine;
 
     b = c->read;
 
     if (nxt_buf_is_file(b)) {
-        body_rest = b->file->size - b->file_end;
 
-        size = nxt_buf_mem_used_size(&b->mem);
-        size = nxt_min(size, body_rest);
+        if (r->chunked) {
+            body_rest = 1;
 
-        res = nxt_fd_write(b->file->fd, b->mem.pos, size);
-        if (nxt_slow_path(res < (ssize_t) size)) {
-            nxt_h1p_request_error(task, h1p, r);
-            return;
-        }
+            out = nxt_http_chunk_parse(task, &h1p->chunked_parse, b);
 
-        b->file_end += size;
-        body_rest -= res;
+            if (h1p->chunked_parse.error) {
+                nxt_h1p_request_error(task, h1p, r);
+                return;
+            }
 
-        b->mem.pos += size;
+            if (h1p->chunked_parse.chunk_error) {
+                nxt_http_request_error(task, r, NXT_HTTP_BAD_REQUEST);
+                return;
+            }
 
-        if (b->mem.pos == b->mem.free) {
-            if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) {
-                b->mem.free = b->mem.start;
+            for (chunk = out; chunk != NULL; chunk = chunk->next) {
+                size = nxt_buf_mem_used_size(&chunk->mem);
+                res = nxt_fd_write(b->file->fd, chunk->mem.pos, size);
+                if (nxt_slow_path(res < (ssize_t) size)) {
+                    nxt_h1p_request_error(task, h1p, r);
+                    return;
+                }
 
-            } else {
-                /* This required to avoid reading next request. */
-                b->mem.free = b->mem.end - body_rest;
+                b->file_end += size;
+
+                if ((size_t) b->file_end > skcf->max_body_size) {
+                    nxt_h1p_request_error(task, h1p, r);
+                    return;
+                }
             }
 
-            b->mem.pos = b->mem.free;
+            if (h1p->chunked_parse.last) {
+                body_rest = 0;
+            }
+
+        } else {
+            body_rest = b->file->size - b->file_end;
+
+            size = nxt_buf_mem_used_size(&b->mem);
+            size = nxt_min(size, body_rest);
+
+            res = nxt_fd_write(b->file->fd, b->mem.pos, size);
+            if (nxt_slow_path(res < (ssize_t) size)) {
+                nxt_h1p_request_error(task, h1p, r);
+                return;
+            }
+
+            b->file_end += size;
+            body_rest -= res;
+
+            b->mem.pos += size;
+
+            if (b->mem.pos == b->mem.free) {
+                if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) {
+                    b->mem.free = b->mem.start;
+
+                } else {
+                    /* This required to avoid reading next request. */
+                    b->mem.free = b->mem.end - body_rest;
+                }
+
+                b->mem.pos = b->mem.free;
+            }
         }
 
     } else {