http: Support chunked request bodies

This is a temporary support for chunked request bodies by converting
to Content-Length. This allows for processing of such requests until
a more permanent solution is developed.

A new configuration option "chunked_transform" has been added to enable
this feature. The option can be set as follows:
  {
    "settings": {
      "chunked_transform": true
    }
  }
By default, this option is set to false, which retains the current
behaviour of rejecting chunked requests with a '411 Length Required'
status code.

Please note that this is an experimental implementation.

Reviewed-by: Andrew Clayton <a.clayton@nginx.com>

6 files changed, 191 insertions, 38 deletions

diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c
index 4aaa1b9a..f91fc887 100644
--- a/src/nxt_conf_validation.c
+++ b/src/nxt_conf_validation.c
@@ -368,6 +368,9 @@ static nxt_conf_vldt_object_t  nxt_conf_vldt_http_members[] = {
     }, {
         .name       = nxt_string("server_version"),
         .type       = NXT_CONF_VLDT_BOOLEAN,
+    }, {
+        .name       = nxt_string("chunked_transform"),
+        .type       = NXT_CONF_VLDT_BOOLEAN,
     },
 
     NXT_CONF_VLDT_END
diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c
index fd184eb5..5d1ed790 100644
--- a/src/nxt_h1proto.c
+++ b/src/nxt_h1proto.c
@@ -839,7 +839,12 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data)
     if (field->value_length == 7
         && memcmp(field->value, "chunked", 7) == 0)
     {
+        if (r->chunked_field != NULL) {
+            return NXT_HTTP_BAD_REQUEST;
+        }
+
         te = NXT_HTTP_TE_CHUNKED;
+        r->chunked_field = field;
 
     } else {
         te = NXT_HTTP_TE_UNSUPPORTED;
@@ -856,14 +861,16 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
 {
     size_t             size, body_length, body_buffer_size, body_rest;
     ssize_t            res;
-    nxt_buf_t          *in, *b;
+    nxt_buf_t          *in, *b, *out, *chunk;
     nxt_conn_t         *c;
     nxt_h1proto_t      *h1p;
+    nxt_socket_conf_t  *skcf;
     nxt_http_status_t  status;
 
     static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX");
 
     h1p = r->proto.h1;
+    skcf = r->conf->socket_conf;
 
     nxt_debug(task, "h1p request body read %O te:%d",
               r->content_length_n, h1p->transfer_encoding);
@@ -871,8 +878,19 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
     switch (h1p->transfer_encoding) {
 
     case NXT_HTTP_TE_CHUNKED:
-        status = NXT_HTTP_LENGTH_REQUIRED;
-        goto error;
+        if (!skcf->chunked_transform) {
+            status = NXT_HTTP_LENGTH_REQUIRED;
+            goto error;
+        }
+
+        if (r->content_length != NULL || !nxt_h1p_is_http11(h1p)) {
+            status = NXT_HTTP_BAD_REQUEST;
+            goto error;
+        }
+
+        r->chunked = 1;
+        h1p->chunked_parse.mem_pool = r->mem_pool;
+        break;
 
     case NXT_HTTP_TE_UNSUPPORTED:
         status = NXT_HTTP_NOT_IMPLEMENTED;
@@ -883,19 +901,20 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
         break;
     }
 
-    if (r->content_length_n == -1 || r->content_length_n == 0) {
+    if (!r->chunked &&
+        (r->content_length_n == -1 || r->content_length_n == 0))
+    {
         goto ready;
     }
 
     body_length = (size_t) r->content_length_n;
 
-    body_buffer_size = nxt_min(r->conf->socket_conf->body_buffer_size,
-                               body_length);
+    body_buffer_size = nxt_min(skcf->body_buffer_size, body_length);
 
     if (body_length > body_buffer_size) {
         nxt_str_t  *tmp_path, tmp_name;
 
-        tmp_path = &r->conf->socket_conf->body_temp_path;
+        tmp_path = &skcf->body_temp_path;
 
         tmp_name.length = tmp_path->length + tmp_name_pattern.length;
 
@@ -946,31 +965,69 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
 
     r->body = b;
 
-    body_rest = body_length;
+    body_rest = r->chunked ? 1 : body_length;
 
     in = h1p->conn->read;
 
     size = nxt_buf_mem_used_size(&in->mem);
 
     if (size != 0) {
-        size = nxt_min(size, body_length);
-
         if (nxt_buf_is_file(b)) {
-            res = nxt_fd_write(b->file->fd, in->mem.pos, size);
-            if (nxt_slow_path(res < (ssize_t) size)) {
-                status = NXT_HTTP_INTERNAL_SERVER_ERROR;
-                goto error;
-            }
+            if (r->chunked) {
+                out = nxt_http_chunk_parse(task, &h1p->chunked_parse, in);
 
-            b->file_end += size;
+                if (h1p->chunked_parse.error) {
+                    status = NXT_HTTP_INTERNAL_SERVER_ERROR;
+                    goto error;
+                }
+
+                if (h1p->chunked_parse.chunk_error) {
+                    status = NXT_HTTP_BAD_REQUEST;
+                    goto error;
+                }
+
+                for (chunk = out; chunk != NULL; chunk = chunk->next) {
+                    size = nxt_buf_mem_used_size(&chunk->mem);
+
+                    res = nxt_fd_write(b->file->fd, chunk->mem.pos, size);
+                    if (nxt_slow_path(res < (ssize_t) size)) {
+                        status = NXT_HTTP_INTERNAL_SERVER_ERROR;
+                        goto error;
+                    }
+
+                    b->file_end += size;
+
+                    if ((size_t) b->file_end > skcf->max_body_size) {
+                        status = NXT_HTTP_PAYLOAD_TOO_LARGE;
+                        goto error;
+                    }
+                }
+
+                if (h1p->chunked_parse.last) {
+                    body_rest = 0;
+                }
+
+            } else {
+                size = nxt_min(size, body_length);
+                res = nxt_fd_write(b->file->fd, in->mem.pos, size);
+                if (nxt_slow_path(res < (ssize_t) size)) {
+                    status = NXT_HTTP_INTERNAL_SERVER_ERROR;
+                    goto error;
+                }
+
+                b->file_end += size;
+
+                in->mem.pos += size;
+                body_rest -= size;
+            }
 
         } else {
             size = nxt_min(body_buffer_size, size);
             b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size);
-        }
 
-        in->mem.pos += size;
-        body_rest -= size;
+            in->mem.pos += size;
+            body_rest -= size;
+        }
     }
 
     nxt_debug(task, "h1p body rest: %uz", body_rest);
@@ -1028,9 +1085,10 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data)
 {
     size_t              size, body_rest;
     ssize_t             res;
-    nxt_buf_t           *b;
+    nxt_buf_t           *b, *out, *chunk;
     nxt_conn_t          *c;
     nxt_h1proto_t       *h1p;
+    nxt_socket_conf_t   *skcf;
     nxt_http_request_t  *r;
     nxt_event_engine_t  *engine;
 
@@ -1040,38 +1098,77 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data)
     nxt_debug(task, "h1p conn request body read");
 
     r = h1p->request;
+    skcf = r->conf->socket_conf;
 
     engine = task->thread->engine;
 
     b = c->read;
 
     if (nxt_buf_is_file(b)) {
-        body_rest = b->file->size - b->file_end;
 
-        size = nxt_buf_mem_used_size(&b->mem);
-        size = nxt_min(size, body_rest);
+        if (r->chunked) {
+            body_rest = 1;
 
-        res = nxt_fd_write(b->file->fd, b->mem.pos, size);
-        if (nxt_slow_path(res < (ssize_t) size)) {
-            nxt_h1p_request_error(task, h1p, r);
-            return;
-        }
+            out = nxt_http_chunk_parse(task, &h1p->chunked_parse, b);
 
-        b->file_end += size;
-        body_rest -= res;
+            if (h1p->chunked_parse.error) {
+                nxt_h1p_request_error(task, h1p, r);
+                return;
+            }
 
-        b->mem.pos += size;
+            if (h1p->chunked_parse.chunk_error) {
+                nxt_http_request_error(task, r, NXT_HTTP_BAD_REQUEST);
+                return;
+            }
 
-        if (b->mem.pos == b->mem.free) {
-            if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) {
-                b->mem.free = b->mem.start;
+            for (chunk = out; chunk != NULL; chunk = chunk->next) {
+                size = nxt_buf_mem_used_size(&chunk->mem);
+                res = nxt_fd_write(b->file->fd, chunk->mem.pos, size);
+                if (nxt_slow_path(res < (ssize_t) size)) {
+                    nxt_h1p_request_error(task, h1p, r);
+                    return;
+                }
 
-            } else {
-                /* This required to avoid reading next request. */
-                b->mem.free = b->mem.end - body_rest;
+                b->file_end += size;
+
+                if ((size_t) b->file_end > skcf->max_body_size) {
+                    nxt_h1p_request_error(task, h1p, r);
+                    return;
+                }
             }
 
-            b->mem.pos = b->mem.free;
+            if (h1p->chunked_parse.last) {
+                body_rest = 0;
+            }
+
+        } else {
+            body_rest = b->file->size - b->file_end;
+
+            size = nxt_buf_mem_used_size(&b->mem);
+            size = nxt_min(size, body_rest);
+
+            res = nxt_fd_write(b->file->fd, b->mem.pos, size);
+            if (nxt_slow_path(res < (ssize_t) size)) {
+                nxt_h1p_request_error(task, h1p, r);
+                return;
+            }
+
+            b->file_end += size;
+            body_rest -= res;
+
+            b->mem.pos += size;
+
+            if (b->mem.pos == b->mem.free) {
+                if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) {
+                    b->mem.free = b->mem.start;
+
+                } else {
+                    /* This required to avoid reading next request. */
+                    b->mem.free = b->mem.end - body_rest;
+                }
+
+                b->mem.pos = b->mem.free;
+            }
         }
 
     } else {
diff --git a/src/nxt_http.h b/src/nxt_http.h
index 5fab5c67..fe5e72a8 100644
--- a/src/nxt_http.h
+++ b/src/nxt_http.h
@@ -157,6 +157,7 @@ struct nxt_http_request_s {
     nxt_list_t                      *fields;
     nxt_http_field_t                *content_type;
     nxt_http_field_t                *content_length;
+    nxt_http_field_t                *chunked_field;
     nxt_http_field_t                *cookie;
     nxt_http_field_t                *referer;
     nxt_http_field_t                *user_agent;
@@ -204,6 +205,7 @@ struct nxt_http_request_s {
     uint8_t                         inconsistent; /* 1 bit  */
     uint8_t                         error;        /* 1 bit  */
     uint8_t                         websocket_handshake;  /* 1 bit */
+    uint8_t                         chunked;  /* 1 bit */
 };
 
 
diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c
index 425a4607..54d1bd27 100644
--- a/src/nxt_http_request.c
+++ b/src/nxt_http_request.c
@@ -540,15 +540,58 @@ static const nxt_http_request_state_t  nxt_http_request_body_state
 };
 
 
+static nxt_int_t
+nxt_http_request_chunked_transform(nxt_http_request_t *r)
+{
+    size_t            size;
+    u_char            *p, *end;
+    nxt_http_field_t  *f;
+
+    r->chunked_field->skip = 1;
+
+    size = r->body->file_end;
+
+    f = nxt_list_zero_add(r->fields);
+    if (nxt_slow_path(f == NULL)) {
+        return NXT_ERROR;
+    }
+
+    nxt_http_field_name_set(f, "Content-Length");
+
+    p = nxt_mp_nget(r->mem_pool, NXT_OFF_T_LEN);
+    if (nxt_slow_path(p == NULL)) {
+        return NXT_ERROR;
+    }
+
+    f->value = p;
+    end = nxt_sprintf(p, p + NXT_OFF_T_LEN, "%uz", size);
+    f->value_length = end - p;
+
+    r->content_length = f;
+    r->content_length_n = size;
+
+    return NXT_OK;
+}
+
+
 static void
 nxt_http_request_ready(nxt_task_t *task, void *obj, void *data)
 {
+    nxt_int_t           ret;
     nxt_http_action_t   *action;
     nxt_http_request_t  *r;
 
     r = obj;
     action = r->conf->socket_conf->action;
 
+    if (r->chunked) {
+        ret = nxt_http_request_chunked_transform(r);
+        if (nxt_slow_path(ret != NXT_OK)) {
+            nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR);
+            return;
+        }
+    }
+
     nxt_http_request_action(task, r, action);
 }
 
diff --git a/src/nxt_router.c b/src/nxt_router.c
index 48870d20..43209451 100644
--- a/src/nxt_router.c
+++ b/src/nxt_router.c
@@ -1575,6 +1575,12 @@ static nxt_conf_map_t  nxt_router_http_conf[] = {
         NXT_CONF_MAP_INT8,
         offsetof(nxt_socket_conf_t, server_version),
     },
+
+    {
+        nxt_string("chunked_transform"),
+        NXT_CONF_MAP_INT8,
+        offsetof(nxt_socket_conf_t, chunked_transform),
+    },
 };
 
 
@@ -1994,6 +2000,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
             skcf->proxy_read_timeout = 30 * 1000;
 
             skcf->server_version = 1;
+            skcf->chunked_transform = 0;
 
             skcf->websocket_conf.max_frame_size = 1024 * 1024;
             skcf->websocket_conf.read_timeout = 60 * 1000;
diff --git a/src/nxt_router.h b/src/nxt_router.h
index 3e523001..cfc7258c 100644
--- a/src/nxt_router.h
+++ b/src/nxt_router.h
@@ -208,6 +208,7 @@ typedef struct {
     uint8_t                discard_unsafe_fields;  /* 1 bit */
 
     uint8_t                server_version;         /* 1 bit */
+    uint8_t                chunked_transform;      /* 1 bit */
 
     nxt_http_forward_t     *forwarded;
     nxt_http_forward_t     *client_ip;