diff options
| author | Andrei Belov <defan@nginx.com> | 2020-08-13 19:28:27 +0300 |
|---|---|---|
| committer | Andrei Belov <defan@nginx.com> | 2020-08-13 19:28:27 +0300 |
| commit | aff76e4f90b4e948c327ce2b021dc3203c33cbcd (patch) | |
| tree | 5bd6ac3aa92683777548472984c209bf26d8a971 /src | |
| parent | 04ce9f997e0e49e57ce4b5fc4aa98134232a1974 (diff) | |
| parent | 6473d4b65a99aa10d509220fb99d8c4f65631ed0 (diff) | |
| download | unit-1.19.0-1.tar.gz unit-1.19.0-1.tar.bz2 | |
Merged with the default branch.1.19.0-1
Diffstat (limited to '')
57 files changed, 7073 insertions, 3071 deletions
diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 975174d4..1ee5b742 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -13,11 +13,14 @@ #include <nxt_unit_websocket.h> +static void delete_port_data(uv_handle_t* handle); + napi_ref Unit::constructor_; -struct nxt_nodejs_ctx_t { - nxt_unit_port_id_t port_id; +struct port_data_t { + nxt_unit_ctx_t *ctx; + nxt_unit_port_t *port; uv_poll_t poll; }; @@ -348,7 +351,11 @@ Unit::shm_ack_handler(nxt_unit_ctx_t *ctx) static void nxt_uv_read_callback(uv_poll_t *handle, int status, int events) { - nxt_unit_run_once((nxt_unit_ctx_t *) handle->data); + port_data_t *data; + + data = (port_data_t *) handle->data; + + nxt_unit_process_port_msg(data->ctx, data->port); } @@ -358,8 +365,8 @@ Unit::add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) int err; Unit *obj; uv_loop_t *loop; + port_data_t *data; napi_status status; - nxt_nodejs_ctx_t *node_ctx; if (port->in_fd != -1) { obj = reinterpret_cast<Unit *>(ctx->unit->data); @@ -376,55 +383,56 @@ Unit::add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) return NXT_UNIT_ERROR; } - node_ctx = new nxt_nodejs_ctx_t; + data = new port_data_t; - err = uv_poll_init(loop, &node_ctx->poll, port->in_fd); + err = uv_poll_init(loop, &data->poll, port->in_fd); if (err < 0) { nxt_unit_warn(ctx, "Failed to init uv.poll"); return NXT_UNIT_ERROR; } - err = uv_poll_start(&node_ctx->poll, UV_READABLE, nxt_uv_read_callback); + err = uv_poll_start(&data->poll, UV_READABLE, nxt_uv_read_callback); if (err < 0) { nxt_unit_warn(ctx, "Failed to start uv.poll"); return NXT_UNIT_ERROR; } - ctx->data = node_ctx; + port->data = data; - node_ctx->port_id = port->id; - node_ctx->poll.data = ctx; + data->ctx = ctx; + data->port = port; + data->poll.data = data; } - return nxt_unit_add_port(ctx, port); -} - - -inline bool -operator == (const nxt_unit_port_id_t &p1, const nxt_unit_port_id_t &p2) -{ - return p1.pid == p2.pid && p1.id == p2.id; + return NXT_UNIT_OK; } void -Unit::remove_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id) +Unit::remove_port(nxt_unit_t *unit, nxt_unit_port_t *port) { - nxt_nodejs_ctx_t *node_ctx; + port_data_t *data; - if (ctx->data != NULL) { - node_ctx = (nxt_nodejs_ctx_t *) ctx->data; + if (port->data != NULL) { + data = (port_data_t *) port->data; - if (node_ctx->port_id == *port_id) { - uv_poll_stop(&node_ctx->poll); + if (data->port == port) { + uv_poll_stop(&data->poll); - delete node_ctx; - - ctx->data = NULL; + uv_close((uv_handle_t *) &data->poll, delete_port_data); } } +} + + +static void +delete_port_data(uv_handle_t* handle) +{ + port_data_t *data; + + data = (port_data_t *) handle->data; - nxt_unit_remove_port(ctx, port_id); + delete data; } diff --git a/src/nodejs/unit-http/unit.h b/src/nodejs/unit-http/unit.h index 18359118..07823c26 100644 --- a/src/nodejs/unit-http/unit.h +++ b/src/nodejs/unit-http/unit.h @@ -40,7 +40,7 @@ private: void shm_ack_handler(nxt_unit_ctx_t *ctx); static int add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); - static void remove_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id); + static void remove_port(nxt_unit_t *unit, nxt_unit_port_t *port); static void quit_cb(nxt_unit_ctx_t *ctx); void quit(nxt_unit_ctx_t *ctx); diff --git a/src/nxt_app_nncq.h b/src/nxt_app_nncq.h new file mode 100644 index 00000000..f9b8ce0c --- /dev/null +++ b/src/nxt_app_nncq.h @@ -0,0 +1,165 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_APP_NNCQ_H_INCLUDED_ +#define _NXT_APP_NNCQ_H_INCLUDED_ + + +/* Appilcation Numeric Naive Circular Queue */ + +#define NXT_APP_NNCQ_SIZE 131072 + +typedef uint32_t nxt_app_nncq_atomic_t; +typedef uint16_t nxt_app_nncq_cycle_t; + +typedef struct { + nxt_app_nncq_atomic_t head; + nxt_app_nncq_atomic_t entries[NXT_APP_NNCQ_SIZE]; + nxt_app_nncq_atomic_t tail; +} nxt_app_nncq_t; + + +static inline nxt_app_nncq_atomic_t +nxt_app_nncq_head(nxt_app_nncq_t const volatile *q) +{ + return q->head; +} + + +static inline nxt_app_nncq_atomic_t +nxt_app_nncq_tail(nxt_app_nncq_t const volatile *q) +{ + return q->tail; +} + + +static inline void +nxt_app_nncq_tail_cmp_inc(nxt_app_nncq_t volatile *q, nxt_app_nncq_atomic_t t) +{ + nxt_atomic_cmp_set(&q->tail, t, t + 1); +} + + +static inline nxt_app_nncq_atomic_t +nxt_app_nncq_index(nxt_app_nncq_t const volatile *q, nxt_app_nncq_atomic_t i) +{ + return i % NXT_APP_NNCQ_SIZE; +} + + +static inline nxt_app_nncq_atomic_t +nxt_app_nncq_map(nxt_app_nncq_t const volatile *q, nxt_app_nncq_atomic_t i) +{ + return i % NXT_APP_NNCQ_SIZE; +} + + +static inline nxt_app_nncq_cycle_t +nxt_app_nncq_cycle(nxt_app_nncq_t const volatile *q, nxt_app_nncq_atomic_t i) +{ + return i / NXT_APP_NNCQ_SIZE; +} + + +static inline nxt_app_nncq_cycle_t +nxt_app_nncq_next_cycle(nxt_app_nncq_t const volatile *q, + nxt_app_nncq_cycle_t i) +{ + return i + 1; +} + + +static inline nxt_app_nncq_atomic_t +nxt_app_nncq_new_entry(nxt_app_nncq_t const volatile *q, + nxt_app_nncq_cycle_t cycle, + nxt_app_nncq_atomic_t i) +{ + return cycle * NXT_APP_NNCQ_SIZE + (i % NXT_APP_NNCQ_SIZE); +} + + +static inline nxt_app_nncq_atomic_t +nxt_app_nncq_empty(nxt_app_nncq_t const volatile *q) +{ + return NXT_APP_NNCQ_SIZE; +} + + +static void +nxt_app_nncq_init(nxt_app_nncq_t volatile *q) +{ + q->head = NXT_APP_NNCQ_SIZE; + nxt_memzero((void *) q->entries, + NXT_APP_NNCQ_SIZE * sizeof(nxt_app_nncq_atomic_t)); + q->tail = NXT_APP_NNCQ_SIZE; +} + + +static void +nxt_app_nncq_enqueue(nxt_app_nncq_t volatile *q, nxt_app_nncq_atomic_t val) +{ + nxt_app_nncq_cycle_t e_cycle, t_cycle; + nxt_app_nncq_atomic_t n, t, e, j; + + for ( ;; ) { + t = nxt_app_nncq_tail(q); + j = nxt_app_nncq_map(q, t); + e = q->entries[j]; + + e_cycle = nxt_app_nncq_cycle(q, e); + t_cycle = nxt_app_nncq_cycle(q, t); + + if (e_cycle == t_cycle) { + nxt_app_nncq_tail_cmp_inc(q, t); + continue; + } + + if (nxt_app_nncq_next_cycle(q, e_cycle) != t_cycle) { + continue; + } + + n = nxt_app_nncq_new_entry(q, t_cycle, val); + + if (nxt_atomic_cmp_set(&q->entries[j], e, n)) { + break; + } + } + + nxt_app_nncq_tail_cmp_inc(q, t); +} + + +static nxt_app_nncq_atomic_t +nxt_app_nncq_dequeue(nxt_app_nncq_t volatile *q) +{ + nxt_app_nncq_cycle_t e_cycle, h_cycle; + nxt_app_nncq_atomic_t h, j, e; + + for ( ;; ) { + h = nxt_app_nncq_head(q); + j = nxt_app_nncq_map(q, h); + e = q->entries[j]; + + e_cycle = nxt_app_nncq_cycle(q, e); + h_cycle = nxt_app_nncq_cycle(q, h); + + if (e_cycle != h_cycle) { + if (nxt_app_nncq_next_cycle(q, e_cycle) == h_cycle) { + return nxt_app_nncq_empty(q); + } + + continue; + } + + if (nxt_atomic_cmp_set(&q->head, h, h + 1)) { + break; + } + } + + return nxt_app_nncq_index(q, e); +} + + +#endif /* _NXT_APP_NNCQ_H_INCLUDED_ */ diff --git a/src/nxt_app_queue.h b/src/nxt_app_queue.h new file mode 100644 index 00000000..127cb8f3 --- /dev/null +++ b/src/nxt_app_queue.h @@ -0,0 +1,119 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_APP_QUEUE_H_INCLUDED_ +#define _NXT_APP_QUEUE_H_INCLUDED_ + + +#include <nxt_app_nncq.h> + + +/* Using Numeric Naive Circular Queue as a backend. */ + +#define NXT_APP_QUEUE_SIZE NXT_APP_NNCQ_SIZE +#define NXT_APP_QUEUE_MSG_SIZE 31 + +typedef struct { + uint8_t size; + uint8_t data[NXT_APP_QUEUE_MSG_SIZE]; + uint32_t tracking; +} nxt_app_queue_item_t; + + +typedef struct { + nxt_app_nncq_atomic_t nitems; + nxt_app_nncq_t free_items; + nxt_app_nncq_t queue; + nxt_app_queue_item_t items[NXT_APP_QUEUE_SIZE]; +} nxt_app_queue_t; + + +nxt_inline void +nxt_app_queue_init(nxt_app_queue_t volatile *q) +{ + nxt_app_nncq_atomic_t i; + + nxt_app_nncq_init(&q->free_items); + nxt_app_nncq_init(&q->queue); + + for (i = 0; i < NXT_APP_QUEUE_SIZE; i++) { + nxt_app_nncq_enqueue(&q->free_items, i); + } + + q->nitems = 0; +} + + +nxt_inline nxt_int_t +nxt_app_queue_send(nxt_app_queue_t volatile *q, const void *p, + uint8_t size, uint32_t tracking, int *notify, uint32_t *cookie) +{ + nxt_app_queue_item_t *qi; + nxt_app_nncq_atomic_t i; + + i = nxt_app_nncq_dequeue(&q->free_items); + if (i == nxt_app_nncq_empty(&q->free_items)) { + return NXT_AGAIN; + } + + qi = (nxt_app_queue_item_t *) &q->items[i]; + + qi->size = size; + nxt_memcpy(qi->data, p, size); + qi->tracking = tracking; + *cookie = i; + + nxt_app_nncq_enqueue(&q->queue, i); + + i = nxt_atomic_fetch_add(&q->nitems, 1); + + if (notify != NULL) { + *notify = (i == 0); + } + + return NXT_OK; +} + + +nxt_inline nxt_bool_t +nxt_app_queue_cancel(nxt_app_queue_t volatile *q, uint32_t cookie, + uint32_t tracking) +{ + nxt_app_queue_item_t *qi; + + qi = (nxt_app_queue_item_t *) &q->items[cookie]; + + return nxt_atomic_cmp_set(&qi->tracking, tracking, 0); +} + + +nxt_inline ssize_t +nxt_app_queue_recv(nxt_app_queue_t volatile *q, void *p, uint32_t *cookie) +{ + ssize_t res; + nxt_app_queue_item_t *qi; + nxt_app_nncq_atomic_t i; + + i = nxt_app_nncq_dequeue(&q->queue); + if (i == nxt_app_nncq_empty(&q->queue)) { + *cookie = 0; + return -1; + } + + qi = (nxt_app_queue_item_t *) &q->items[i]; + + res = qi->size; + nxt_memcpy(p, qi->data, qi->size); + *cookie = i; + + nxt_app_nncq_enqueue(&q->free_items, i); + + nxt_atomic_fetch_add(&q->nitems, -1); + + return res; +} + + +#endif /* _NXT_APP_QUEUE_H_INCLUDED_ */ diff --git a/src/nxt_application.c b/src/nxt_application.c index 566bf256..57e4615e 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -41,19 +41,21 @@ static void nxt_discovery_quit(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static nxt_app_module_t *nxt_app_module_load(nxt_task_t *task, const char *name); -static nxt_int_t nxt_app_prefork(nxt_task_t *task, nxt_process_t *process, +static nxt_int_t nxt_app_main_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp); static nxt_int_t nxt_app_setup(nxt_task_t *task, nxt_process_t *process); static nxt_int_t nxt_app_set_environment(nxt_conf_value_t *environment); static u_char *nxt_cstr_dup(nxt_mp_t *mp, u_char *dst, u_char *src); #if (NXT_HAVE_ISOLATION_ROOTFS) -static nxt_int_t nxt_app_prepare_rootfs(nxt_task_t *task, - nxt_process_t *process); -static nxt_int_t nxt_app_prepare_lang_mounts(nxt_task_t *task, +static nxt_int_t nxt_app_set_isolation_mounts(nxt_task_t *task, + nxt_process_t *process, nxt_str_t *app_type); +static nxt_int_t nxt_app_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, nxt_array_t *syspaths); static nxt_int_t nxt_app_set_isolation_rootfs(nxt_task_t *task, nxt_conf_value_t *isolation, nxt_process_t *process); +static nxt_int_t nxt_app_prepare_rootfs(nxt_task_t *task, + nxt_process_t *process); #endif static nxt_int_t nxt_app_set_isolation(nxt_task_t *task, @@ -124,7 +126,7 @@ const nxt_process_init_t nxt_discovery_process = { const nxt_process_init_t nxt_app_process = { .type = NXT_PROCESS_APP, .setup = nxt_app_setup, - .prefork = nxt_app_prefork, + .prefork = nxt_app_main_prefork, .restart = 0, .start = NULL, /* set to module->start */ .port_handlers = &nxt_app_process_port_handlers, @@ -287,7 +289,7 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) *p++ = ']'; - if (nxt_slow_path(p >= end)) { + if (nxt_slow_path(p > end)) { nxt_alert(task, "discovery write past the buffer"); goto fail; } @@ -472,22 +474,16 @@ nxt_discovery_quit(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) static nxt_int_t -nxt_app_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) +nxt_app_main_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) { - nxt_int_t cap_setid, cap_chroot; + nxt_int_t cap_setid; nxt_int_t ret; nxt_runtime_t *rt; nxt_common_app_conf_t *app_conf; - nxt_app_lang_module_t *lang; rt = task->thread->runtime; app_conf = process->data.app; cap_setid = rt->capabilities.setid; - cap_chroot = rt->capabilities.chroot; - - lang = nxt_app_lang_module(rt, &app_conf->type); - - nxt_assert(lang != NULL); if (app_conf->isolation != NULL) { ret = nxt_app_set_isolation(task, app_conf->isolation, process); @@ -499,24 +495,14 @@ nxt_app_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) #if (NXT_HAVE_CLONE_NEWUSER) if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWUSER)) { cap_setid = 1; - cap_chroot = 1; } #endif #if (NXT_HAVE_ISOLATION_ROOTFS) if (process->isolation.rootfs != NULL) { - if (!cap_chroot) { - nxt_log(task, NXT_LOG_ERR, - "The \"rootfs\" field requires privileges"); - - return NXT_ERROR; - } - - if (lang->mounts != NULL && lang->mounts->nelts > 0) { - ret = nxt_app_prepare_lang_mounts(task, process, lang->mounts); - if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; - } + ret = nxt_app_set_isolation_mounts(task, process, &app_conf->type); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; } } #endif @@ -765,71 +751,6 @@ nxt_app_set_isolation_namespaces(nxt_task_t *task, nxt_conf_value_t *isolation, #endif -#if (NXT_HAVE_ISOLATION_ROOTFS) - -static nxt_int_t -nxt_app_set_isolation_rootfs(nxt_task_t *task, nxt_conf_value_t *isolation, - nxt_process_t *process) -{ - nxt_str_t str; - nxt_conf_value_t *obj; - - static nxt_str_t rootfs_name = nxt_string("rootfs"); - - obj = nxt_conf_get_object_member(isolation, &rootfs_name, NULL); - if (obj != NULL) { - nxt_conf_get_string(obj, &str); - - if (nxt_slow_path(str.length <= 1 || str.start[0] != '/')) { - nxt_log(task, NXT_LOG_ERR, "rootfs requires an absolute path other " - "than \"/\" but given \"%V\"", &str); - - return NXT_ERROR; - } - - if (str.start[str.length - 1] == '/') { - str.length--; - } - - process->isolation.rootfs = nxt_mp_alloc(process->mem_pool, - str.length + 1); - - if (nxt_slow_path(process->isolation.rootfs == NULL)) { - return NXT_ERROR; - } - - nxt_memcpy(process->isolation.rootfs, str.start, str.length); - - process->isolation.rootfs[str.length] = '\0'; - } - - return NXT_OK; -} - -#endif - - -#if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) - -static nxt_int_t -nxt_app_set_isolation_new_privs(nxt_task_t *task, nxt_conf_value_t *isolation, - nxt_process_t *process) -{ - nxt_conf_value_t *obj; - - static nxt_str_t new_privs_name = nxt_string("new_privs"); - - obj = nxt_conf_get_object_member(isolation, &new_privs_name, NULL); - if (obj != NULL) { - process->isolation.new_privs = nxt_conf_get_boolean(obj); - } - - return NXT_OK; -} - -#endif - - #if (NXT_HAVE_CLONE_NEWUSER) static nxt_int_t @@ -1002,7 +923,83 @@ nxt_app_clone_flags(nxt_task_t *task, nxt_conf_value_t *namespaces, #if (NXT_HAVE_ISOLATION_ROOTFS) static nxt_int_t -nxt_app_prepare_lang_mounts(nxt_task_t *task, nxt_process_t *process, +nxt_app_set_isolation_rootfs(nxt_task_t *task, nxt_conf_value_t *isolation, + nxt_process_t *process) +{ + nxt_str_t str; + nxt_conf_value_t *obj; + + static nxt_str_t rootfs_name = nxt_string("rootfs"); + + obj = nxt_conf_get_object_member(isolation, &rootfs_name, NULL); + if (obj != NULL) { + nxt_conf_get_string(obj, &str); + + if (nxt_slow_path(str.length <= 1 || str.start[0] != '/')) { + nxt_log(task, NXT_LOG_ERR, "rootfs requires an absolute path other " + "than \"/\" but given \"%V\"", &str); + + return NXT_ERROR; + } + + if (str.start[str.length - 1] == '/') { + str.length--; + } + + process->isolation.rootfs = nxt_mp_alloc(process->mem_pool, + str.length + 1); + + if (nxt_slow_path(process->isolation.rootfs == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(process->isolation.rootfs, str.start, str.length); + + process->isolation.rootfs[str.length] = '\0'; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_app_set_isolation_mounts(nxt_task_t *task, nxt_process_t *process, + nxt_str_t *app_type) +{ + nxt_int_t ret, cap_chroot; + nxt_runtime_t *rt; + nxt_app_lang_module_t *lang; + + rt = task->thread->runtime; + cap_chroot = rt->capabilities.chroot; + lang = nxt_app_lang_module(rt, app_type); + + nxt_assert(lang != NULL); + +#if (NXT_HAVE_CLONE_NEWUSER) + if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWUSER)) { + cap_chroot = 1; + } +#endif + + if (!cap_chroot) { + nxt_log(task, NXT_LOG_ERR, "The \"rootfs\" field requires privileges"); + return NXT_ERROR; + } + + if (lang->mounts != NULL && lang->mounts->nelts > 0) { + ret = nxt_app_set_lang_mounts(task, process, lang->mounts); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_app_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, nxt_array_t *lang_mounts) { u_char *p; @@ -1045,7 +1042,6 @@ nxt_app_prepare_lang_mounts(nxt_task_t *task, nxt_process_t *process, } - static nxt_int_t nxt_app_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) { @@ -1137,6 +1133,27 @@ undo: #endif +#if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) + +static nxt_int_t +nxt_app_set_isolation_new_privs(nxt_task_t *task, nxt_conf_value_t *isolation, + nxt_process_t *process) +{ + nxt_conf_value_t *obj; + + static nxt_str_t new_privs_name = nxt_string("new_privs"); + + obj = nxt_conf_get_object_member(isolation, &new_privs_name, NULL); + if (obj != NULL) { + process->isolation.new_privs = nxt_conf_get_boolean(obj); + } + + return NXT_OK; +} + +#endif + + static u_char * nxt_cstr_dup(nxt_mp_t *mp, u_char *dst, u_char *src) { @@ -1246,7 +1263,7 @@ nxt_app_parse_type(u_char *p, size_t length) nxt_int_t nxt_unit_default_init(nxt_task_t *task, nxt_unit_init_t *init) { - nxt_port_t *my_port, *main_port; + nxt_port_t *my_port, *main_port, *router_port; nxt_runtime_t *rt; nxt_memzero(init, sizeof(nxt_unit_init_t)); @@ -1258,6 +1275,11 @@ nxt_unit_default_init(nxt_task_t *task, nxt_unit_init_t *init) return NXT_ERROR; } + router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; + if (nxt_slow_path(router_port == NULL)) { + return NXT_ERROR; + } + my_port = nxt_runtime_port_find(rt, nxt_pid, 0); if (nxt_slow_path(my_port == NULL)) { return NXT_ERROR; @@ -1265,17 +1287,20 @@ nxt_unit_default_init(nxt_task_t *task, nxt_unit_init_t *init) init->ready_port.id.pid = main_port->pid; init->ready_port.id.id = main_port->id; + init->ready_port.in_fd = -1; init->ready_port.out_fd = main_port->pair[1]; - nxt_fd_blocking(task, main_port->pair[1]); - init->ready_stream = my_port->process->stream; + init->router_port.id.pid = router_port->pid; + init->router_port.id.id = router_port->id; + init->router_port.in_fd = -1; + init->router_port.out_fd = router_port->pair[1]; + init->read_port.id.pid = my_port->pid; init->read_port.id.id = my_port->id; init->read_port.in_fd = my_port->pair[0]; - - nxt_fd_blocking(task, my_port->pair[0]); + init->read_port.out_fd = -1; init->log_fd = 2; diff --git a/src/nxt_array.c b/src/nxt_array.c index 6fe9ad6a..1e13c22a 100644 --- a/src/nxt_array.c +++ b/src/nxt_array.c @@ -51,7 +51,7 @@ nxt_array_add(nxt_array_t *array) if (nalloc < 16) { /* Allocate new array twice larger than current. */ - new_alloc = nalloc * 2; + new_alloc = (nalloc == 0) ? 4 : nalloc * 2; } else { /* Allocate new array 1.5 times larger than current. */ diff --git a/src/nxt_array.h b/src/nxt_array.h index 5762ec27..8318fccd 100644 --- a/src/nxt_array.h +++ b/src/nxt_array.h @@ -18,6 +18,14 @@ typedef struct { } nxt_array_t; +nxt_inline void +nxt_array_init(nxt_array_t *array, nxt_mp_t *mp, size_t size) +{ + array->elts = nxt_pointer_to(array, sizeof(nxt_array_t)); + array->size = size; + array->mem_pool = mp; +} + NXT_EXPORT nxt_array_t *nxt_array_create(nxt_mp_t *mp, nxt_uint_t n, size_t size); NXT_EXPORT void nxt_array_destroy(nxt_array_t *array); diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index c4f78608..b5530b85 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -31,6 +31,11 @@ typedef enum { |NXT_CONF_VLDT_OBJECT) +typedef enum { + NXT_CONF_VLDT_REQUIRED = 1, +} nxt_conf_vldt_flags_t; + + typedef nxt_int_t (*nxt_conf_vldt_handler_t)(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); @@ -38,14 +43,15 @@ typedef nxt_int_t (*nxt_conf_vldt_handler_t)(nxt_conf_validation_t *vldt, typedef struct { nxt_str_t name; - nxt_conf_vldt_type_t type; + nxt_conf_vldt_type_t type:32; + nxt_conf_vldt_flags_t flags:32; nxt_conf_vldt_handler_t validator; void *data; } nxt_conf_vldt_object_t; -#define NXT_CONF_VLDT_NEXT(f) { nxt_null_string, 0, NULL, (f) } -#define NXT_CONF_VLDT_END { nxt_null_string, 0, NULL, NULL } +#define NXT_CONF_VLDT_NEXT(f) { nxt_null_string, 0, 0, NULL, (f) } +#define NXT_CONF_VLDT_END { nxt_null_string, 0, 0, NULL, NULL } typedef nxt_int_t (*nxt_conf_vldt_member_t)(nxt_conf_validation_t *vldt, @@ -58,6 +64,8 @@ static nxt_int_t nxt_conf_vldt_type(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type); static nxt_int_t nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...); +static nxt_int_t nxt_conf_vldt_var(nxt_conf_validation_t *vldt, + const char *option, nxt_str_t *value); static nxt_int_t nxt_conf_vldt_mtypes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); @@ -165,16 +173,19 @@ static nxt_int_t nxt_conf_vldt_clone_gidmap(nxt_conf_validation_t *vldt, static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { { nxt_string("read_timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("keepalive_interval"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("max_frame_size"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, @@ -185,6 +196,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { { nxt_string("mime_types"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_mtypes, NULL }, @@ -195,46 +207,55 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { { nxt_string("header_read_timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("body_read_timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("send_timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("idle_timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("body_buffer_size"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("max_body_size"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("body_temp_path"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("websocket"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_websocket_members }, { nxt_string("static"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_static_members }, @@ -245,6 +266,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { { nxt_string("http"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_http_members }, @@ -255,31 +277,37 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { { nxt_string("settings"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_setting_members }, { nxt_string("listeners"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_listener }, { nxt_string("routes"), NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_routes, NULL }, { nxt_string("applications"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_app }, { nxt_string("upstreams"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_upstream }, { nxt_string("access_log"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, @@ -292,6 +320,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { { nxt_string("certificate"), NXT_CONF_VLDT_STRING, + 0, &nxt_conf_vldt_certificate, NULL }, @@ -304,11 +333,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { { nxt_string("pass"), NXT_CONF_VLDT_STRING, + 0, &nxt_conf_vldt_pass, NULL }, { nxt_string("application"), NXT_CONF_VLDT_STRING, + 0, &nxt_conf_vldt_app_name, NULL }, @@ -316,6 +347,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { { nxt_string("tls"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_tls_members }, @@ -328,46 +360,55 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { { nxt_string("method"), NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_patterns, NULL }, { nxt_string("scheme"), NXT_CONF_VLDT_STRING, + 0, &nxt_conf_vldt_match_scheme_pattern, NULL }, { nxt_string("host"), NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_patterns, NULL }, { nxt_string("source"), NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_addrs, NULL }, { nxt_string("destination"), NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_addrs, NULL }, { nxt_string("uri"), NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_encoded_patterns, NULL }, { nxt_string("arguments"), NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_encoded_patterns_sets, NULL }, { nxt_string("headers"), NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_patterns_sets, NULL }, { nxt_string("cookies"), NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_match_patterns_sets, NULL }, @@ -378,6 +419,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { { nxt_string("pass"), NXT_CONF_VLDT_STRING, + 0, &nxt_conf_vldt_pass, NULL }, @@ -388,11 +430,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_return_action_members[] = { { nxt_string("return"), NXT_CONF_VLDT_INTEGER, + 0, &nxt_conf_vldt_return, NULL }, { nxt_string("location"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, @@ -403,11 +447,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_return_action_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { { nxt_string("share"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("fallback"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_action, NULL }, @@ -418,6 +464,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { { nxt_string("proxy"), NXT_CONF_VLDT_STRING, + 0, &nxt_conf_vldt_proxy, NULL }, @@ -428,11 +475,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { { nxt_string("match"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_match_members }, { nxt_string("action"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_action, NULL }, @@ -443,21 +492,25 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { { nxt_string("timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("reschedule_timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("requests"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("shm"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, @@ -468,16 +521,19 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { { nxt_string("spare"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("max"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("idle_timeout"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, @@ -490,6 +546,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_CLONE_NEWUSER) { nxt_string("credential"), NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, #endif @@ -497,6 +554,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_CLONE_NEWPID) { nxt_string("pid"), NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, #endif @@ -504,6 +562,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_CLONE_NEWNET) { nxt_string("network"), NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, #endif @@ -511,6 +570,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_CLONE_NEWNS) { nxt_string("mount"), NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, #endif @@ -518,6 +578,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_CLONE_NEWUTS) { nxt_string("uname"), NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, #endif @@ -525,6 +586,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_CLONE_NEWCGROUP) { nxt_string("cgroup"), NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, #endif @@ -538,18 +600,23 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { { nxt_string("container"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("host"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, { nxt_string("size"), NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, + + NXT_CONF_VLDT_END }; #endif @@ -558,6 +625,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { { nxt_string("namespaces"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_clone_namespaces, (void *) &nxt_conf_vldt_app_namespaces_members }, @@ -565,11 +633,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { { nxt_string("uidmap"), NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_array_iterator, (void *) &nxt_conf_vldt_clone_uidmap }, { nxt_string("gidmap"), NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_array_iterator, (void *) &nxt_conf_vldt_clone_gidmap }, @@ -579,6 +649,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { { nxt_string("rootfs"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, @@ -588,6 +659,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { { nxt_string("new_privs"), NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, @@ -600,41 +672,49 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { { nxt_string("type"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("limits"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_app_limits_members }, { nxt_string("processes"), NXT_CONF_VLDT_INTEGER | NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_processes, (void *) &nxt_conf_vldt_app_processes_members }, { nxt_string("user"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("group"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("working_directory"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("environment"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_environment }, { nxt_string("isolation"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_isolation, (void *) &nxt_conf_vldt_app_isolation_members }, @@ -645,11 +725,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { { nxt_string("executable"), NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, { nxt_string("arguments"), NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_array_iterator, (void *) &nxt_conf_vldt_argument }, @@ -660,16 +742,19 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { { nxt_string("home"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("path"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("module"), NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, @@ -680,34 +765,42 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { { nxt_string("root"), NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, { nxt_string("script"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("index"), NXT_CONF_VLDT_STRING, + 0, NULL, - NULL } + NULL }, + + NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { { nxt_string("file"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("admin"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_php_option }, { nxt_string("user"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_php_option }, @@ -718,6 +811,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { { nxt_string("options"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object, (void *) &nxt_conf_vldt_php_options_members }, @@ -728,16 +822,19 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { { nxt_string("root"), NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, { nxt_string("script"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, { nxt_string("index"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, @@ -748,21 +845,25 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { { nxt_string("root"), NXT_CONF_VLDT_ANY_TYPE, + 0, &nxt_conf_vldt_php_targets_exclusive, (void *) "root" }, { nxt_string("script"), NXT_CONF_VLDT_ANY_TYPE, + 0, &nxt_conf_vldt_php_targets_exclusive, (void *) "script" }, { nxt_string("index"), NXT_CONF_VLDT_ANY_TYPE, + 0, &nxt_conf_vldt_php_targets_exclusive, (void *) "index" }, { nxt_string("targets"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_php_targets, NULL }, @@ -773,6 +874,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { { nxt_string("script"), NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, @@ -783,6 +885,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { { nxt_string("script"), NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, @@ -793,21 +896,25 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { { nxt_string("classpath"), NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_array_iterator, (void *) &nxt_conf_vldt_java_classpath }, { nxt_string("webapp"), NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, { nxt_string("options"), NXT_CONF_VLDT_ARRAY, + 0, &nxt_conf_vldt_array_iterator, (void *) &nxt_conf_vldt_java_option }, { nxt_string("unit_jars"), NXT_CONF_VLDT_STRING, + 0, NULL, NULL }, @@ -818,6 +925,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_members[] = { { nxt_string("servers"), NXT_CONF_VLDT_OBJECT, + 0, &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_server }, @@ -828,6 +936,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_server_members[] = { { nxt_string("weight"), NXT_CONF_VLDT_NUMBER, + 0, &nxt_conf_vldt_server_weight, NULL }, @@ -861,7 +970,7 @@ nxt_conf_vldt_type(nxt_conf_validation_t *vldt, nxt_str_t *name, { u_char *p; nxt_str_t expected; - nxt_bool_t serial; + nxt_bool_t comma; nxt_uint_t value_type, n, t; u_char buf[nxt_length(NXT_CONF_VLDT_ANY_TYPE_STR)]; @@ -889,7 +998,7 @@ nxt_conf_vldt_type(nxt_conf_validation_t *vldt, nxt_str_t *name, p = nxt_cpymem(p, "either ", 7); } - serial = (n > 2); + comma = (n > 2); for ( ;; ) { t = __builtin_ffs(type) - 1; @@ -902,7 +1011,7 @@ nxt_conf_vldt_type(nxt_conf_validation_t *vldt, nxt_str_t *name, break; } - if (n > 1 || serial) { + if (comma) { *p++ = ','; } @@ -958,6 +1067,21 @@ nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...) } +static nxt_int_t +nxt_conf_vldt_var(nxt_conf_validation_t *vldt, const char *option, + nxt_str_t *value) +{ + u_char error[NXT_MAX_ERROR_STR]; + + if (nxt_var_test(value, error) != NXT_OK) { + return nxt_conf_vldt_error(vldt, "%s in the \"%s\" value.", + error, option); + } + + return NXT_OK; +} + + typedef struct { nxt_mp_t *pool; nxt_str_t *type; @@ -1135,6 +1259,10 @@ nxt_conf_vldt_pass(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_conf_get_string(value, &pass); + if (nxt_is_var(&pass)) { + return nxt_conf_vldt_var(vldt, "pass", &pass); + } + ret = nxt_http_pass_segments(vldt->pool, &pass, segments, 3); if (ret != NXT_OK) { @@ -1346,16 +1474,9 @@ static nxt_int_t nxt_conf_vldt_match_pattern(nxt_conf_validation_t *vldt, nxt_conf_value_t *value) { - u_char ch; nxt_str_t pattern; nxt_uint_t i, first, last; - enum { - sw_none, - sw_side, - sw_middle - } state; - if (nxt_conf_type(value) != NXT_CONF_STRING) { return nxt_conf_vldt_error(vldt, "The \"match\" patterns for \"host\", " "\"uri\", and \"method\" must be strings."); @@ -1369,37 +1490,11 @@ nxt_conf_vldt_match_pattern(nxt_conf_validation_t *vldt, first = (pattern.start[0] == '!'); last = pattern.length - 1; - state = sw_none; - - for (i = first; i != pattern.length; i++) { - ch = pattern.start[i]; - - if (ch != '*') { - continue; - } - - switch (state) { - case sw_none: - state = (i == first) ? sw_side : sw_middle; - break; - - case sw_side: - if (i == last) { - if (last - first != 1) { - break; - } - - return nxt_conf_vldt_error(vldt, "The \"match\" pattern must " - "not contain double \"*\" markers."); - } - - /* Fall through. */ - - case sw_middle: - return nxt_conf_vldt_error(vldt, "The \"match\" patterns can " - "either contain \"*\" markers at " - "the sides or only one in the middle."); + for (i = first; i < last; i++) { + if (pattern.start[i] == '*' && pattern.start[i + 1] == '*') { + return nxt_conf_vldt_error(vldt, "The \"match\" pattern must " + "not contain double \"*\" markers."); } } @@ -1759,6 +1854,31 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_conf_value_t *member; nxt_conf_vldt_object_t *vals; + vals = data; + + for ( ;; ) { + if (vals->name.length == 0) { + + if (vals->data != NULL) { + vals = vals->data; + continue; + } + + break; + } + + if (vals->flags & NXT_CONF_VLDT_REQUIRED) { + member = nxt_conf_get_object_member(value, &vals->name, NULL); + + if (member == NULL) { + return nxt_conf_vldt_error(vldt, "Required parameter \"%V\" " + "is missing.", &vals->name); + } + } + + vals++; + } + index = 0; for ( ;; ) { diff --git a/src/nxt_conn_accept.c b/src/nxt_conn_accept.c index 6a89840c..77c44c58 100644 --- a/src/nxt_conn_accept.c +++ b/src/nxt_conn_accept.c @@ -98,7 +98,9 @@ nxt_conn_accept_alloc(nxt_task_t *task, nxt_listen_event_t *lev) if (nxt_fast_path(mp != NULL)) { c = nxt_conn_create(mp, lev->socket.task); if (nxt_slow_path(c == NULL)) { - goto fail; + nxt_mp_destroy(mp); + + return NULL; } c->socket.read_work_queue = lev->socket.read_work_queue; @@ -109,11 +111,9 @@ nxt_conn_accept_alloc(nxt_task_t *task, nxt_listen_event_t *lev) lev->next = c; return c; } - } - fail: - - nxt_mp_destroy(mp); + nxt_conn_free(task, c); + } } return NULL; diff --git a/src/nxt_controller.c b/src/nxt_controller.c index a61c127d..9a34a877 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -54,7 +54,7 @@ static nxt_int_t nxt_controller_conf_default(void); static void nxt_controller_conf_init_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static void nxt_controller_flush_requests(nxt_task_t *task); -static nxt_int_t nxt_controller_conf_send(nxt_task_t *task, +static nxt_int_t nxt_controller_conf_send(nxt_task_t *task, nxt_mp_t *mp, nxt_conf_value_t *conf, nxt_port_rpc_handler_t handler, void *data); static void nxt_controller_conn_init(nxt_task_t *task, void *obj, void *data); @@ -344,7 +344,7 @@ nxt_controller_send_current_conf(nxt_task_t *task) conf = nxt_controller_conf.root; if (conf != NULL) { - rc = nxt_controller_conf_send(task, conf, + rc = nxt_controller_conf_send(task, nxt_controller_conf.pool, conf, nxt_controller_conf_init_handler, NULL); if (nxt_fast_path(rc == NXT_OK)) { @@ -497,11 +497,14 @@ nxt_controller_flush_requests(nxt_task_t *task) static nxt_int_t -nxt_controller_conf_send(nxt_task_t *task, nxt_conf_value_t *conf, +nxt_controller_conf_send(nxt_task_t *task, nxt_mp_t *mp, nxt_conf_value_t *conf, nxt_port_rpc_handler_t handler, void *data) { + void *mem; + u_char *end; size_t size; uint32_t stream; + nxt_fd_t fd; nxt_int_t rc; nxt_buf_t *b; nxt_port_t *router_port, *controller_port; @@ -518,30 +521,53 @@ nxt_controller_conf_send(nxt_task_t *task, nxt_conf_value_t *conf, size = nxt_conf_json_length(conf, NULL); - b = nxt_port_mmap_get_buf(task, router_port, size); + b = nxt_buf_mem_alloc(mp, sizeof(size_t), 0); if (nxt_slow_path(b == NULL)) { return NXT_ERROR; } - b->mem.free = nxt_conf_json_print(b->mem.free, conf, NULL); + fd = nxt_shm_open(task, size); + if (nxt_slow_path(fd == -1)) { + return NXT_ERROR; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + goto fail; + } + + end = nxt_conf_json_print(mem, conf, NULL); + + nxt_mem_munmap(mem, size); + + size = end - (u_char *) mem; + + b->mem.free = nxt_cpymem(b->mem.pos, &size, sizeof(size_t)); stream = nxt_port_rpc_register_handler(task, controller_port, handler, handler, router_port->pid, data); - if (nxt_slow_path(stream == 0)) { - return NXT_ERROR; + goto fail; } - rc = nxt_port_socket_write(task, router_port, NXT_PORT_MSG_DATA_LAST, -1, - stream, controller_port->id, b); + rc = nxt_port_socket_write(task, router_port, + NXT_PORT_MSG_DATA_LAST | NXT_PORT_MSG_CLOSE_FD, + fd, stream, controller_port->id, b); if (nxt_slow_path(rc != NXT_OK)) { nxt_port_rpc_cancel(task, controller_port, stream); - return NXT_ERROR; + + goto fail; } return NXT_OK; + +fail: + + nxt_fd_close(fd); + + return NXT_ERROR; } @@ -1201,7 +1227,7 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, goto alloc_fail; } - rc = nxt_controller_conf_send(task, value, + rc = nxt_controller_conf_send(task, mp, value, nxt_controller_conf_handler, req); if (nxt_slow_path(rc != NXT_OK)) { @@ -1282,7 +1308,7 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, goto alloc_fail; } - rc = nxt_controller_conf_send(task, value, + rc = nxt_controller_conf_send(task, mp, value, nxt_controller_conf_handler, req); if (nxt_slow_path(rc != NXT_OK)) { @@ -1564,9 +1590,9 @@ nxt_controller_process_cert_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, mbuf = &c->read->mem; - nxt_fd_write(msg->fd, mbuf->pos, nxt_buf_mem_used_size(mbuf)); + nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); - nxt_fd_close(msg->fd); + nxt_fd_close(msg->fd[0]); nxt_memzero(&resp, sizeof(nxt_controller_response_t)); diff --git a/src/nxt_external.c b/src/nxt_external.c index 6370a9c4..1adb839c 100644 --- a/src/nxt_external.c +++ b/src/nxt_external.c @@ -52,8 +52,6 @@ nxt_external_fd_no_cloexec(nxt_task_t *task, nxt_socket_t fd) return NXT_ERROR; } - nxt_fd_blocking(task, fd); - return NXT_OK; } @@ -69,7 +67,7 @@ nxt_external_start(nxt_task_t *task, nxt_process_data_t *data) nxt_str_t str; nxt_int_t rc; nxt_uint_t i, argc; - nxt_port_t *my_port, *main_port; + nxt_port_t *my_port, *main_port, *router_port; nxt_runtime_t *rt; nxt_conf_value_t *value; nxt_common_app_conf_t *conf; @@ -79,9 +77,12 @@ nxt_external_start(nxt_task_t *task, nxt_process_data_t *data) conf = data->app; main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; my_port = nxt_runtime_port_find(rt, nxt_pid, 0); - if (nxt_slow_path(main_port == NULL || my_port == NULL)) { + if (nxt_slow_path(main_port == NULL || my_port == NULL + || router_port == NULL)) + { return NXT_ERROR; } @@ -90,6 +91,11 @@ nxt_external_start(nxt_task_t *task, nxt_process_data_t *data) return NXT_ERROR; } + rc = nxt_external_fd_no_cloexec(task, router_port->pair[1]); + if (nxt_slow_path(rc != NXT_OK)) { + return NXT_ERROR; + } + rc = nxt_external_fd_no_cloexec(task, my_port->pair[0]); if (nxt_slow_path(rc != NXT_OK)) { return NXT_ERROR; @@ -101,9 +107,11 @@ nxt_external_start(nxt_task_t *task, nxt_process_data_t *data) "%s;%uD;" "%PI,%ud,%d;" "%PI,%ud,%d;" + "%PI,%ud,%d;" "%d,%z,%Z", NXT_VERSION, my_port->process->stream, main_port->pid, main_port->id, main_port->pair[1], + router_port->pid, router_port->id, router_port->pair[1], my_port->pid, my_port->id, my_port->pair[0], 2, conf->shm_limit); diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index a139f611..b34be019 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -99,6 +99,7 @@ static nxt_int_t nxt_h1p_peer_header_parse(nxt_http_peer_t *peer, nxt_buf_mem_t *bm); static void nxt_h1p_peer_read(nxt_task_t *task, nxt_http_peer_t *peer); static void nxt_h1p_peer_read_done(nxt_task_t *task, void *obj, void *data); +static void nxt_h1p_peer_body_process(nxt_task_t *task, nxt_http_peer_t *peer, nxt_buf_t *out); static void nxt_h1p_peer_closed(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_peer_error(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_peer_send_timeout(nxt_task_t *task, void *obj, void *data); @@ -106,6 +107,8 @@ static void nxt_h1p_peer_read_timeout(nxt_task_t *task, void *obj, void *data); static nxt_msec_t nxt_h1p_peer_timer_value(nxt_conn_t *c, uintptr_t data); static void nxt_h1p_peer_close(nxt_task_t *task, nxt_http_peer_t *peer); static void nxt_h1p_peer_free(nxt_task_t *task, void *obj, void *data); +static nxt_int_t nxt_h1p_peer_transfer_encoding(void *ctx, + nxt_http_field_t *field, uintptr_t data); #if (NXT_TLS) static const nxt_conn_state_t nxt_http_idle_state; @@ -178,7 +181,7 @@ static nxt_lvlhsh_t nxt_h1p_peer_fields_hash; static nxt_http_field_proc_t nxt_h1p_peer_fields[] = { { nxt_string("Connection"), &nxt_http_proxy_skip, 0 }, - { nxt_string("Transfer-Encoding"), &nxt_http_proxy_skip, 0 }, + { nxt_string("Transfer-Encoding"), &nxt_h1p_peer_transfer_encoding, 0 }, { nxt_string("Server"), &nxt_http_proxy_skip, 0 }, { nxt_string("Date"), &nxt_http_proxy_date, 0 }, { nxt_string("Content-Length"), &nxt_http_proxy_content_length, 0 }, @@ -903,7 +906,7 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) b->file->fd = mkstemp((char *) tmp_name.start); if (nxt_slow_path(b->file->fd == -1)) { - nxt_log(task, NXT_LOG_ERR, "mkstemp() failed %E", nxt_errno); + nxt_alert(task, "mkstemp(%s) failed %E", tmp_name.start, nxt_errno); status = NXT_HTTP_INTERNAL_SERVER_ERROR; goto error; @@ -2139,9 +2142,6 @@ nxt_h1p_peer_connect(nxt_task_t *task, nxt_http_peer_t *peer) peer->proto.h1 = h1p; h1p->request = r; - c->socket.task = task; - c->read_timer.task = task; - c->write_timer.task = task; c->socket.data = peer; c->remote = peer->server->sockaddr; @@ -2238,7 +2238,8 @@ nxt_h1p_peer_header_send(nxt_task_t *task, nxt_http_peer_t *peer) r = peer->request; size = r->method->length + sizeof(" ") + r->target.length - + sizeof(" HTTP/1.0\r\n") + + sizeof(" HTTP/1.1\r\n") + + sizeof("Connection: close\r\n") + sizeof("\r\n"); nxt_list_each(field, r->fields) { @@ -2261,7 +2262,8 @@ nxt_h1p_peer_header_send(nxt_task_t *task, nxt_http_peer_t *peer) p = nxt_cpymem(p, r->method->start, r->method->length); *p++ = ' '; p = nxt_cpymem(p, r->target.start, r->target.length); - p = nxt_cpymem(p, " HTTP/1.0\r\n", 11); + p = nxt_cpymem(p, " HTTP/1.1\r\n", 11); + p = nxt_cpymem(p, "Connection: close\r\n", 19); nxt_list_each(field, r->fields) { @@ -2466,6 +2468,7 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) nxt_int_t ret; nxt_buf_t *b; nxt_conn_t *c; + nxt_h1proto_t *h1p; nxt_http_peer_t *peer; nxt_http_request_t *r; nxt_event_engine_t *engine; @@ -2503,11 +2506,26 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) c->read = NULL; - if (nxt_buf_mem_used_size(&b->mem) != 0) { - peer->body = b; + peer->header_received = 1; + + h1p = peer->proto.h1; + + if (h1p->chunked) { + if (r->resp.content_length != NULL) { + peer->status = NXT_HTTP_BAD_GATEWAY; + break; + } + + h1p->chunked_parse.mem_pool = c->mem_pool; + + } else if (r->resp.content_length_n > 0) { + h1p->remainder = r->resp.content_length_n; } - peer->header_received = 1; + if (nxt_buf_mem_used_size(&b->mem) != 0) { + nxt_h1p_peer_body_process(task, peer, b); + return; + } r->state->ready_handler(task, r, peer); return; @@ -2613,18 +2631,54 @@ static const nxt_conn_state_t nxt_h1p_peer_read_state static void nxt_h1p_peer_read_done(nxt_task_t *task, void *obj, void *data) { - nxt_conn_t *c; - nxt_http_peer_t *peer; - nxt_http_request_t *r; + nxt_buf_t *out; + nxt_conn_t *c; + nxt_http_peer_t *peer; c = obj; peer = data; nxt_debug(task, "h1p peer read done"); - peer->body = c->read; + out = c->read; c->read = NULL; + nxt_h1p_peer_body_process(task, peer, out); +} + + +static void +nxt_h1p_peer_body_process(nxt_task_t *task, nxt_http_peer_t *peer, + nxt_buf_t *out) +{ + size_t length; + nxt_h1proto_t *h1p; + nxt_http_request_t *r; + + h1p = peer->proto.h1; + + if (h1p->chunked) { + out = nxt_http_chunk_parse(task, &h1p->chunked_parse, out); + + if (h1p->chunked_parse.chunk_error || h1p->chunked_parse.error) { + peer->status = NXT_HTTP_BAD_GATEWAY; + r = peer->request; + r->state->error_handler(task, r, peer); + return; + } + + if (h1p->chunked_parse.last) { + nxt_buf_chain_add(&out, nxt_http_buf_last(peer->request)); + peer->closed = 1; + } + + } else if (h1p->remainder > 0) { + length = nxt_buf_chain_length(out); + h1p->remainder -= length; + } + + peer->body = out; + r = peer->request; r->state->ready_handler(task, r, peer); } @@ -2644,8 +2698,8 @@ nxt_h1p_peer_closed(nxt_task_t *task, void *obj, void *data) if (peer->header_received) { peer->body = nxt_http_buf_last(r); - peer->closed = 1; + r->inconsistent = (peer->proto.h1->remainder != 0); r->state->ready_handler(task, r, peer); @@ -2777,3 +2831,22 @@ nxt_h1p_peer_free(nxt_task_t *task, void *obj, void *data) nxt_conn_free(task, c); } + + +static nxt_int_t +nxt_h1p_peer_transfer_encoding(void *ctx, nxt_http_field_t *field, + uintptr_t data) +{ + nxt_http_request_t *r; + + r = ctx; + field->skip = 1; + + if (field->value_length == 7 + && nxt_memcmp(field->value, "chunked", 7) == 0) + { + r->peer->proto.h1->chunked = 1; + } + + return NXT_OK; +} diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index 3294713f..f8500963 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -18,6 +18,8 @@ typedef struct nxt_h1p_websocket_timer_s nxt_h1p_websocket_timer_t; struct nxt_h1proto_s { nxt_http_request_parse_t parser; + nxt_http_chunk_parse_t chunked_parse; + nxt_off_t remainder; uint8_t nbuffers; uint8_t header_buffer_slot; diff --git a/src/nxt_http.h b/src/nxt_http.h index 68051e69..08181520 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -119,7 +119,6 @@ typedef struct { nxt_upstream_server_t *server; nxt_list_t *fields; nxt_buf_t *body; - nxt_off_t remainder; nxt_http_status_t status:16; nxt_http_protocol_t protocol:8; /* 2 bits */ @@ -165,11 +164,17 @@ struct nxt_http_request_s { nxt_timer_t timer; void *timer_data; + nxt_var_query_t *var_query; + void *req_rpc_data; nxt_http_peer_t *peer; nxt_buf_t *last; + nxt_queue_link_t app_link; /* nxt_app_t.ack_waiting_req */ + nxt_event_engine_t *engine; + nxt_work_t err_work; + nxt_http_response_t resp; nxt_http_status_t status:16; @@ -199,6 +204,7 @@ struct nxt_http_action_s { nxt_upstream_t *upstream; uint32_t upstream_number; nxt_http_status_t return_code; + nxt_var_t *var; } u; nxt_str_t name; @@ -284,15 +290,16 @@ nxt_int_t nxt_http_routes_resolve(nxt_task_t *task, nxt_int_t nxt_http_pass_segments(nxt_mp_t *mp, nxt_str_t *pass, nxt_str_t *segments, nxt_uint_t n); nxt_http_action_t *nxt_http_pass_application(nxt_task_t *task, - nxt_router_temp_conf_t *tmcf, nxt_str_t *name); -void nxt_http_routes_cleanup(nxt_task_t *task, nxt_http_routes_t *routes); -void nxt_http_action_cleanup(nxt_task_t *task, nxt_http_action_t *action); + nxt_router_conf_t *rtcf, nxt_str_t *name); nxt_int_t nxt_upstreams_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_conf_value_t *conf); nxt_int_t nxt_upstreams_joint_create(nxt_router_temp_conf_t *tmcf, nxt_upstream_t ***upstream_joint); +void nxt_http_request_action(nxt_task_t *task, nxt_http_request_t *r, + nxt_http_action_t *action); + nxt_http_action_t *nxt_http_return_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action); @@ -306,7 +313,7 @@ nxt_str_t *nxt_http_static_mtypes_hash_find(nxt_lvlhsh_t *hash, nxt_http_action_t *nxt_http_application_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action); -void nxt_upstream_find(nxt_upstreams_t *upstreams, nxt_str_t *name, +nxt_int_t nxt_upstream_find(nxt_upstreams_t *upstreams, nxt_str_t *name, nxt_http_action_t *action); nxt_http_action_t *nxt_upstream_proxy_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_upstream_t *upstream); diff --git a/src/nxt_http_chunk_parse.c b/src/nxt_http_chunk_parse.c index 644b9805..2164524b 100644 --- a/src/nxt_http_chunk_parse.c +++ b/src/nxt_http_chunk_parse.c @@ -21,13 +21,17 @@ static nxt_int_t nxt_http_chunk_buffer(nxt_http_chunk_parse_t *hcp, nxt_buf_t ***tail, nxt_buf_t *in); +static void nxt_http_chunk_buf_completion(nxt_task_t *task, void *obj, + void *data); + + nxt_buf_t * nxt_http_chunk_parse(nxt_task_t *task, nxt_http_chunk_parse_t *hcp, nxt_buf_t *in) { u_char c, ch; nxt_int_t ret; - nxt_buf_t *b, *out, *nb, **tail; + nxt_buf_t *b, *out, *next, **tail; enum { sw_start = 0, sw_chunk_size, @@ -37,12 +41,13 @@ nxt_http_chunk_parse(nxt_task_t *task, nxt_http_chunk_parse_t *hcp, sw_chunk, } state; + next = NULL; out = NULL; tail = &out; state = hcp->state; - for (b = in; b != NULL; b = b->next) { + for (b = in; b != NULL; b = next) { hcp->pos = b->mem.pos; @@ -60,7 +65,7 @@ nxt_http_chunk_parse(nxt_task_t *task, nxt_http_chunk_parse_t *hcp, if (nxt_slow_path(ret == NXT_ERROR)) { hcp->error = 1; - goto done; + return out; } state = sw_chunk_end_newline; @@ -152,7 +157,7 @@ nxt_http_chunk_parse(nxt_task_t *task, nxt_http_chunk_parse_t *hcp, continue; } - goto done; + return out; } goto chunk_error; @@ -168,15 +173,15 @@ nxt_http_chunk_parse(nxt_task_t *task, nxt_http_chunk_parse_t *hcp, if (b->retain == 0) { /* No chunk data was found in a buffer. */ - nxt_thread_current_work_queue_add(task->thread, - b->completion_handler, - task, b, b->parent); + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); } next: - continue; + next = b->next; + b->next = NULL; } hcp->state = state; @@ -187,20 +192,6 @@ chunk_error: hcp->chunk_error = 1; -done: - - nb = nxt_buf_sync_alloc(hcp->mem_pool, NXT_BUF_SYNC_LAST); - - if (nxt_fast_path(nb != NULL)) { - *tail = nb; - - } else { - hcp->error = 1; - } - - // STUB: hcp->chunk_error = 1; - // STUB: hcp->error = 1; - return out; } @@ -216,43 +207,35 @@ nxt_http_chunk_buffer(nxt_http_chunk_parse_t *hcp, nxt_buf_t ***tail, p = hcp->pos; size = in->mem.free - p; - if (hcp->chunk_size >= size && in->retain == 0) { - /* - * Use original buffer if the buffer is lesser than or equal - * to a chunk size and this is the first chunk in the buffer. - */ - in->mem.pos = p; - **tail = in; - *tail = &in->next; - - } else { - b = nxt_buf_mem_alloc(hcp->mem_pool, 0, 0); - if (nxt_slow_path(b == NULL)) { - return NXT_ERROR; - } + b = nxt_buf_mem_alloc(hcp->mem_pool, 0, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } - **tail = b; - *tail = &b->next; + **tail = b; + *tail = &b->next; - b->parent = in; - in->retain++; - b->mem.pos = p; - b->mem.start = p; + nxt_mp_retain(hcp->mem_pool); + b->completion_handler = nxt_http_chunk_buf_completion; - if (hcp->chunk_size < size) { - p += hcp->chunk_size; - hcp->pos = p; + b->parent = in; + in->retain++; + b->mem.pos = p; + b->mem.start = p; - b->mem.free = p; - b->mem.end = p; + if (hcp->chunk_size < size) { + p += hcp->chunk_size; + hcp->pos = p; - return NXT_HTTP_CHUNK_END; - } + b->mem.free = p; + b->mem.end = p; - b->mem.free = in->mem.free; - b->mem.end = in->mem.free; + return NXT_HTTP_CHUNK_END; } + b->mem.free = in->mem.free; + b->mem.end = in->mem.free; + hcp->chunk_size -= size; if (hcp->chunk_size == 0) { @@ -261,3 +244,31 @@ nxt_http_chunk_buffer(nxt_http_chunk_parse_t *hcp, nxt_buf_t ***tail, return NXT_HTTP_CHUNK_MIDDLE; } + + +static void +nxt_http_chunk_buf_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_mp_t *mp; + nxt_buf_t *b, *next, *parent; + + b = obj; + parent = data; + + nxt_debug(task, "buf completion: %p %p", b, b->mem.start); + + nxt_assert(data == b->parent); + + do { + next = b->next; + parent = b->parent; + mp = b->data; + + nxt_mp_free(mp, b); + nxt_mp_release(mp); + + nxt_buf_parent_completion(task, parent); + + b = next; + } while (b != NULL); +} diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h index 0f888949..cbfc8433 100644 --- a/src/nxt_http_parse.h +++ b/src/nxt_http_parse.h @@ -90,6 +90,19 @@ struct nxt_http_field_s { }; +typedef struct { + u_char *pos; + nxt_mp_t *mem_pool; + + uint64_t chunk_size; + + uint8_t state; + uint8_t last; /* 1 bit */ + uint8_t chunk_error; /* 1 bit */ + uint8_t error; /* 1 bit */ +} nxt_http_chunk_parse_t; + + #define NXT_HTTP_FIELD_HASH_INIT 159406U #define nxt_http_field_hash_char(h, c) (((h) << 4) + (h) + (c)) #define nxt_http_field_hash_end(h) (((h) >> 16) ^ (h)) @@ -109,6 +122,9 @@ nxt_uint_t nxt_http_fields_hash_collisions(nxt_lvlhsh_t *hash, nxt_int_t nxt_http_fields_process(nxt_list_t *fields, nxt_lvlhsh_t *hash, void *ctx); +nxt_buf_t *nxt_http_chunk_parse(nxt_task_t *task, nxt_http_chunk_parse_t *hcp, + nxt_buf_t *in); + extern const nxt_lvlhsh_proto_t nxt_http_fields_hash_proto; diff --git a/src/nxt_http_proxy.c b/src/nxt_http_proxy.c index 893e9303..34d0f36e 100644 --- a/src/nxt_http_proxy.c +++ b/src/nxt_http_proxy.c @@ -27,8 +27,6 @@ static void nxt_http_proxy_header_send(nxt_task_t *task, void *obj, void *data); static void nxt_http_proxy_header_sent(nxt_task_t *task, void *obj, void *data); static void nxt_http_proxy_header_read(nxt_task_t *task, void *obj, void *data); static void nxt_http_proxy_send_body(nxt_task_t *task, void *obj, void *data); -static void nxt_http_proxy_request_send(nxt_task_t *task, - nxt_http_request_t *r, nxt_buf_t *out); static void nxt_http_proxy_read(nxt_task_t *task, void *obj, void *data); static void nxt_http_proxy_buf_mem_completion(nxt_task_t *task, void *obj, void *data); @@ -253,10 +251,6 @@ nxt_http_proxy_header_read(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "http proxy status: %d", peer->status); - if (r->resp.content_length_n > 0) { - peer->remainder = r->resp.content_length_n; - } - nxt_list_each(field, peer->fields) { nxt_debug(task, "http proxy header: \"%*s: %*s\"", @@ -275,6 +269,8 @@ nxt_http_proxy_header_read(nxt_task_t *task, void *obj, void *data) } nxt_list_loop; + r->state = &nxt_http_proxy_read_state; + nxt_http_request_header_send(task, r, nxt_http_proxy_send_body, peer); } @@ -292,27 +288,13 @@ nxt_http_proxy_send_body(nxt_task_t *task, void *obj, void *data) if (out != NULL) { peer->body = NULL; - nxt_http_proxy_request_send(task, r, out); - } - - r->state = &nxt_http_proxy_read_state; - - nxt_http_proto[peer->protocol].peer_read(task, peer); -} - + nxt_http_request_send(task, r, out); -static void -nxt_http_proxy_request_send(nxt_task_t *task, nxt_http_request_t *r, - nxt_buf_t *out) -{ - size_t length; - - if (r->peer->remainder > 0) { - length = nxt_buf_chain_length(out); - r->peer->remainder -= length; } - nxt_http_request_send(task, r, out); + if (!peer->closed) { + nxt_http_proto[peer->protocol].peer_read(task, peer); + } } @@ -328,7 +310,6 @@ static void nxt_http_proxy_read(nxt_task_t *task, void *obj, void *data) { nxt_buf_t *out; - nxt_bool_t last; nxt_http_peer_t *peer; nxt_http_request_t *r; @@ -336,16 +317,15 @@ nxt_http_proxy_read(nxt_task_t *task, void *obj, void *data) peer = data; out = peer->body; peer->body = NULL; - last = nxt_buf_is_last(out); - nxt_http_proxy_request_send(task, r, out); + if (out != NULL) { + nxt_http_request_send(task, r, out); + } - if (!last) { + if (!peer->closed) { nxt_http_proto[peer->protocol].peer_read(task, peer); } else { - r->inconsistent = (peer->remainder != 0); - nxt_http_proto[peer->protocol].peer_close(task, peer); nxt_mp_release(r->mem_pool); @@ -422,7 +402,7 @@ nxt_http_proxy_error(nxt_task_t *task, void *obj, void *data) nxt_mp_release(r->mem_pool); - nxt_http_request_error(task, r, peer->status); + nxt_http_request_error(&r->task, r, peer->status); } diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index cc1ae17d..76fb3427 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -10,7 +10,7 @@ static nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp); static void nxt_http_request_start(nxt_task_t *task, void *obj, void *data); -static void nxt_http_request_action(nxt_task_t *task, void *obj, void *data); +static void nxt_http_request_ready(nxt_task_t *task, void *obj, void *data); static void nxt_http_request_proto_info(nxt_task_t *task, nxt_http_request_t *r); static void nxt_http_request_mem_buf_completion(nxt_task_t *task, void *obj, @@ -285,21 +285,28 @@ nxt_http_request_start(nxt_task_t *task, void *obj, void *data) static const nxt_http_request_state_t nxt_http_request_body_state nxt_aligned(64) = { - .ready_handler = nxt_http_request_action, + .ready_handler = nxt_http_request_ready, .error_handler = nxt_http_request_close_handler, }; static void -nxt_http_request_action(nxt_task_t *task, void *obj, void *data) +nxt_http_request_ready(nxt_task_t *task, void *obj, void *data) { nxt_http_action_t *action; nxt_http_request_t *r; r = obj; - action = r->conf->socket_conf->action; + nxt_http_request_action(task, r, action); +} + + +void +nxt_http_request_action(nxt_task_t *task, nxt_http_request_t *r, + nxt_http_action_t *action) +{ if (nxt_fast_path(action != NULL)) { do { diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index a8a6b181..0b2103cd 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -26,7 +26,6 @@ typedef enum { typedef enum { NXT_HTTP_ROUTE_PATTERN_EXACT = 0, NXT_HTTP_ROUTE_PATTERN_BEGIN, - NXT_HTTP_ROUTE_PATTERN_MIDDLE, NXT_HTTP_ROUTE_PATTERN_END, NXT_HTTP_ROUTE_PATTERN_SUBSTRING, } nxt_http_route_pattern_type_t; @@ -70,13 +69,16 @@ typedef struct { typedef struct { - u_char *start1; - u_char *start2; - uint32_t length1; - uint32_t length2; + u_char *start; + uint32_t length; + nxt_http_route_pattern_type_t type:8; +} nxt_http_route_pattern_slice_t; + + +typedef struct { uint32_t min_length; + nxt_array_t *pattern_slices; - nxt_http_route_pattern_type_t type:8; uint8_t case_sensitive; /* 1 bit */ uint8_t negative; /* 1 bit */ uint8_t any; /* 1 bit */ @@ -169,7 +171,7 @@ struct nxt_http_routes_s { }; -#define NJS_COOKIE_HASH \ +#define NXT_COOKIE_HASH \ (nxt_http_field_hash_end( \ nxt_http_field_hash_char( \ nxt_http_field_hash_char( \ @@ -209,16 +211,26 @@ static nxt_int_t nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, nxt_http_route_encoding_t encoding); static nxt_int_t nxt_http_route_decode_str(nxt_str_t *str, nxt_http_route_encoding_t encoding); -static u_char *nxt_http_route_pattern_copy(nxt_mp_t *mp, nxt_str_t *test, +static nxt_int_t nxt_http_route_pattern_slice(nxt_array_t *slices, + nxt_str_t *test, + nxt_http_route_pattern_type_t type, + nxt_http_route_encoding_t encoding, nxt_http_route_pattern_case_t pattern_case); static nxt_int_t nxt_http_route_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_http_route_t *route); static nxt_int_t nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_http_action_t *action); -static void nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name, +static nxt_http_action_t *nxt_http_action_pass_var(nxt_task_t *task, + nxt_http_request_t *r, nxt_http_action_t *action); +static void nxt_http_action_pass_var_ready(nxt_task_t *task, void *obj, + void *data); +static void nxt_http_action_pass_var_error(nxt_task_t *task, void *obj, + void *data); +static nxt_int_t nxt_http_pass_find(nxt_task_t *task, nxt_mp_t *mp, + nxt_router_conf_t *rtcf, nxt_http_action_t *action); +static nxt_int_t nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name, nxt_http_action_t *action); -static void nxt_http_route_cleanup(nxt_task_t *task, nxt_http_route_t *routes); static nxt_http_action_t *nxt_http_route_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *start); @@ -1044,103 +1056,163 @@ nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, nxt_http_route_pattern_case_t pattern_case, nxt_http_route_encoding_t encoding) { - u_char *start; - nxt_str_t test, test2; - nxt_int_t ret; - nxt_uint_t n, length; - nxt_http_route_pattern_type_t type; + u_char c, *p, *end; + nxt_str_t test, tmp; + nxt_int_t ret; + nxt_array_t *slices; + nxt_http_route_pattern_type_t type; + + nxt_http_route_pattern_slice_t *slice; type = NXT_HTTP_ROUTE_PATTERN_EXACT; nxt_conf_get_string(cv, &test); + slices = nxt_array_create(mp, 1, sizeof(nxt_http_route_pattern_slice_t)); + if (nxt_slow_path(slices == NULL)) { + return NXT_ERROR; + } + + pattern->pattern_slices = slices; + pattern->negative = 0; pattern->any = 1; pattern->min_length = 0; - if (test.length != 0) { + if (test.length != 0 && test.start[0] == '!') { + test.start++; + test.length--; - if (test.start[0] == '!') { - test.start++; - test.length--; + pattern->negative = 1; + pattern->any = 0; - pattern->negative = 1; - pattern->any = 0; + if (test.length == 0) { + return NXT_OK; } + } - if (test.length != 0) { - if (test.start[0] == '*') { - test.start++; - test.length--; + if (test.length == 0) { + slice = nxt_array_add(slices); + if (nxt_slow_path(slice == NULL)) { + return NXT_ERROR; + } - if (test.length != 0) { - if (test.start[test.length - 1] == '*') { - test.length--; - type = NXT_HTTP_ROUTE_PATTERN_SUBSTRING; + slice->type = NXT_HTTP_ROUTE_PATTERN_EXACT; + slice->start = NULL; + slice->length = 0; - } else { - type = NXT_HTTP_ROUTE_PATTERN_END; - } + return NXT_OK; + } - } else { - type = NXT_HTTP_ROUTE_PATTERN_BEGIN; - } + if (test.start[0] == '*') { + /* 'type' is no longer 'EXACT', assume 'END'. */ + type = NXT_HTTP_ROUTE_PATTERN_END; + test.start++; + test.length--; + } - } else if (test.start[test.length - 1] == '*') { - test.length--; - type = NXT_HTTP_ROUTE_PATTERN_BEGIN; + if (type == NXT_HTTP_ROUTE_PATTERN_EXACT) { + tmp.start = test.start; - } else { - length = test.length - 1; + p = nxt_memchr(test.start, '*', test.length); - for (n = 1; n < length; n++) { - if (test.start[n] != '*') { - continue; - } + if (p == NULL) { + /* No '*' found - EXACT pattern. */ + tmp.length = test.length; + type = NXT_HTTP_ROUTE_PATTERN_EXACT; - test.length = n; + test.start += test.length; + test.length = 0; - test2.start = &test.start[n + 1]; - test2.length = length - n; + } else { + /* '*' found - BEGIN pattern. */ + tmp.length = p - test.start; + type = NXT_HTTP_ROUTE_PATTERN_BEGIN; + + test.start = p + 1; + test.length -= tmp.length + 1; + } + + ret = nxt_http_route_pattern_slice(slices, &tmp, type, encoding, + pattern_case); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } - ret = nxt_http_route_decode_str(&test2, encoding); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } + pattern->min_length += tmp.length; + } - type = NXT_HTTP_ROUTE_PATTERN_MIDDLE; + end = test.start + test.length; - pattern->length2 = test2.length; - pattern->min_length += test2.length; + if (test.length != 0 && end[-1] != '*') { + p = end - 1; - start = nxt_http_route_pattern_copy(mp, &test2, - pattern_case); - if (nxt_slow_path(start == NULL)) { - return NXT_ERROR; - } + while (p != test.start) { + c = *p--; - pattern->start2 = start; - break; - } + if (c == '*') { + p += 2; + break; } + } - ret = nxt_http_route_decode_str(&test, encoding); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } + tmp.start = p; + tmp.length = end - p; + + test.length -= tmp.length; + end = p; + + ret = nxt_http_route_pattern_slice(slices, &tmp, + NXT_HTTP_ROUTE_PATTERN_END, + encoding, pattern_case); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; } + + pattern->min_length += tmp.length; } - pattern->type = type; - pattern->min_length += test.length; - pattern->length1 = test.length; + tmp.start = test.start; + tmp.length = 0; - start = nxt_http_route_pattern_copy(mp, &test, pattern_case); - if (nxt_slow_path(start == NULL)) { - return NXT_ERROR; + p = tmp.start; + + while (p != end) { + c = *p++; + + if (c != '*') { + tmp.length++; + continue; + } + + if (tmp.length == 0) { + tmp.start = p; + continue; + } + + ret = nxt_http_route_pattern_slice(slices, &tmp, + NXT_HTTP_ROUTE_PATTERN_SUBSTRING, + encoding, pattern_case); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + pattern->min_length += tmp.length; + + tmp.start = p; + tmp.length = 0; } - pattern->start1 = start; + if (tmp.length != 0) { + ret = nxt_http_route_pattern_slice(slices, &tmp, + NXT_HTTP_ROUTE_PATTERN_SUBSTRING, + encoding, pattern_case); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + pattern->min_length += tmp.length; + } return NXT_OK; } @@ -1185,15 +1257,25 @@ nxt_http_route_decode_str(nxt_str_t *str, nxt_http_route_encoding_t encoding) } -static u_char * -nxt_http_route_pattern_copy(nxt_mp_t *mp, nxt_str_t *test, +static nxt_int_t +nxt_http_route_pattern_slice(nxt_array_t *slices, + nxt_str_t *test, + nxt_http_route_pattern_type_t type, + nxt_http_route_encoding_t encoding, nxt_http_route_pattern_case_t pattern_case) { - u_char *start; + u_char *start; + nxt_int_t ret; + nxt_http_route_pattern_slice_t *slice; - start = nxt_mp_nget(mp, test->length); + ret = nxt_http_route_decode_str(test, encoding); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + start = nxt_mp_nget(slices->mem_pool, test->length); if (nxt_slow_path(start == NULL)) { - return start; + return NXT_ERROR; } switch (pattern_case) { @@ -1211,7 +1293,16 @@ nxt_http_route_pattern_copy(nxt_mp_t *mp, nxt_str_t *test, break; } - return start; + slice = nxt_array_add(slices); + if (nxt_slow_path(slice == NULL)) { + return NXT_ERROR; + } + + slice->type = type; + slice->start = start; + slice->length = test->length; + + return NXT_OK; } @@ -1269,10 +1360,8 @@ static nxt_int_t nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_http_action_t *action) { - nxt_str_t *targets; - nxt_int_t ret; - nxt_uint_t i; - nxt_str_t segments[3]; + nxt_var_t *var; + nxt_int_t ret; if (action->handler != NULL) { if (action->handler == nxt_http_static_handler @@ -1284,14 +1373,118 @@ nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, return NXT_OK; } - ret = nxt_http_pass_segments(tmcf->mem_pool, &action->name, segments, 3); + if (nxt_is_var(&action->name)) { + var = nxt_var_compile(&action->name, tmcf->router_conf->mem_pool); + if (nxt_slow_path(var == NULL)) { + return NXT_ERROR; + } + + action->u.var = var; + action->handler = nxt_http_action_pass_var; + return NXT_OK; + } + + ret = nxt_http_pass_find(task, tmcf->mem_pool, tmcf->router_conf, action); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } + return NXT_OK; +} + + +static nxt_http_action_t * +nxt_http_action_pass_var(nxt_task_t *task, nxt_http_request_t *r, + nxt_http_action_t *action) +{ + nxt_var_t *var; + nxt_int_t ret; + + ret = nxt_var_query_init(&r->var_query, r, r->mem_pool); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + + var = action->u.var; + + action = nxt_mp_get(r->mem_pool, sizeof(nxt_http_action_t)); + if (nxt_slow_path(action == NULL)) { + goto fail; + } + + nxt_var_query(task, r->var_query, var, &action->name); + nxt_var_query_resolve(task, r->var_query, action, + nxt_http_action_pass_var_ready, + nxt_http_action_pass_var_error); + return NULL; + +fail: + + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); + return NULL; +} + + +static void +nxt_http_action_pass_var_ready(nxt_task_t *task, void *obj, void *data) +{ + nxt_int_t ret; + nxt_router_conf_t *rtcf; + nxt_http_action_t *action; + nxt_http_status_t status; + nxt_http_request_t *r; + + r = obj; + action = data; + rtcf = r->conf->socket_conf->router_conf; + + nxt_debug(task, "http pass lookup: %V", &action->name); + + ret = nxt_http_pass_find(task, r->mem_pool, rtcf, action); + + if (ret != NXT_OK) { + status = (ret == NXT_DECLINED) ? NXT_HTTP_NOT_FOUND + : NXT_HTTP_INTERNAL_SERVER_ERROR; + + nxt_http_request_error(task, r, status); + return; + } + + nxt_http_request_action(task, r, action); +} + + +static void +nxt_http_action_pass_var_error(nxt_task_t *task, void *obj, void *data) +{ + nxt_http_request_t *r; + + r = obj; + + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); +} + + +static nxt_int_t +nxt_http_pass_find(nxt_task_t *task, nxt_mp_t *mp, nxt_router_conf_t *rtcf, + nxt_http_action_t *action) +{ + nxt_str_t *targets; + nxt_int_t ret; + nxt_uint_t i; + nxt_str_t segments[3]; + + ret = nxt_http_pass_segments(mp, &action->name, segments, 3); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + if (nxt_str_eq(&segments[0], "applications", 12)) { - nxt_router_listener_application(tmcf, &segments[1], action); - nxt_router_app_use(task, action->u.application, 1); + ret = nxt_router_listener_application(rtcf, &segments[1], action); + + if (ret != NXT_OK) { + return ret; + } if (segments[2].length != 0) { targets = action->u.application->targets; @@ -1304,14 +1497,20 @@ nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, action->target = 0; } - } else if (nxt_str_eq(&segments[0], "upstreams", 9)) { - nxt_upstream_find(tmcf->router_conf->upstreams, &segments[1], action); + return NXT_OK; + } + + if (segments[2].length == 0) { + if (nxt_str_eq(&segments[0], "upstreams", 9)) { + return nxt_upstream_find(rtcf->upstreams, &segments[1], action); + } - } else if (nxt_str_eq(&segments[0], "routes", 6)) { - nxt_http_route_find(tmcf->router_conf->routes, &segments[1], action); + if (nxt_str_eq(&segments[0], "routes", 6)) { + return nxt_http_route_find(rtcf->routes, &segments[1], action); + } } - return NXT_OK; + return NXT_DECLINED; } @@ -1367,7 +1566,7 @@ nxt_http_pass_segments(nxt_mp_t *mp, nxt_str_t *pass, nxt_str_t *segments, } -static void +static nxt_int_t nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name, nxt_http_action_t *action) { @@ -1381,11 +1580,13 @@ nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name, action->u.route = *route; action->handler = nxt_http_route_handler; - return; + return NXT_OK; } route++; } + + return NXT_DECLINED; } @@ -1413,21 +1614,19 @@ nxt_http_action_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, /* COMPATIBILITY: listener application. */ nxt_http_action_t * -nxt_http_pass_application(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, +nxt_http_pass_application(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_str_t *name) { nxt_http_action_t *action; - action = nxt_mp_alloc(tmcf->router_conf->mem_pool, - sizeof(nxt_http_action_t)); + action = nxt_mp_alloc(rtcf->mem_pool, sizeof(nxt_http_action_t)); if (nxt_slow_path(action == NULL)) { return NULL; } action->name = *name; - nxt_router_listener_application(tmcf, name, action); - nxt_router_app_use(task, action->u.application, 1); + (void) nxt_router_listener_application(rtcf, name, action); action->target = 0; @@ -1435,56 +1634,6 @@ nxt_http_pass_application(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, } -void -nxt_http_routes_cleanup(nxt_task_t *task, nxt_http_routes_t *routes) -{ - nxt_http_route_t **route, **end; - - if (routes != NULL) { - route = &routes->route[0]; - end = route + routes->items; - - while (route < end) { - nxt_http_route_cleanup(task, *route); - - route++; - } - } -} - - -static void -nxt_http_route_cleanup(nxt_task_t *task, nxt_http_route_t *route) -{ - nxt_http_route_match_t **match, **end; - - match = &route->match[0]; - end = match + route->items; - - while (match < end) { - nxt_http_action_cleanup(task, &(*match)->action); - - match++; - } -} - - -void -nxt_http_action_cleanup(nxt_task_t *task, nxt_http_action_t *action) -{ - if (action->handler == nxt_http_application_handler) { - nxt_router_app_use(task, action->u.application, -1); - return; - } - - if (action->handler == nxt_http_static_handler - && action->u.fallback != NULL) - { - nxt_http_action_cleanup(task, action->u.fallback); - } -} - - static nxt_http_action_t * nxt_http_route_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *start) @@ -2037,9 +2186,11 @@ nxt_http_route_test_argument(nxt_http_request_t *r, static nxt_int_t nxt_http_route_scheme(nxt_http_request_t *r, nxt_http_route_rule_t *rule) { - nxt_bool_t tls, https; + nxt_bool_t tls, https; + nxt_http_route_pattern_slice_t *pattern_slice; - https = (rule->pattern[0].length1 == nxt_length("https")); + pattern_slice = rule->pattern[0].pattern_slices->elts; + https = (pattern_slice->length == nxt_length("https")); tls = (r->tls != NULL); return (tls == https); @@ -2078,7 +2229,7 @@ nxt_http_route_cookies_parse(nxt_http_request_t *r) nxt_list_each(f, r->fields) { - if (f->hash != NJS_COOKIE_HASH + if (f->hash != NXT_COOKIE_HASH || f->name_length != 6 || nxt_strncasecmp(f->name, (u_char *) "Cookie", 6) != 0) { @@ -2246,60 +2397,69 @@ static nxt_int_t nxt_http_route_pattern(nxt_http_request_t *r, nxt_http_route_pattern_t *pattern, u_char *start, size_t length) { - u_char *p, *end, *test; - size_t test_length; - nxt_int_t ret; + u_char *p, *end, *test; + size_t test_length; + uint32_t i; + nxt_array_t *pattern_slices; + nxt_http_route_pattern_slice_t *pattern_slice; if (length < pattern->min_length) { return 0; } - test = pattern->start1; - test_length = pattern->length1; + pattern_slices = pattern->pattern_slices; + pattern_slice = pattern_slices->elts; + end = start + length; - switch (pattern->type) { + for (i = 0; i < pattern_slices->nelts; i++, pattern_slice++) { + test = pattern_slice->start; + test_length = pattern_slice->length; - case NXT_HTTP_ROUTE_PATTERN_EXACT: - if (length != test_length) { - return 0; - } + switch (pattern_slice->type) { + case NXT_HTTP_ROUTE_PATTERN_EXACT: + return ((length == pattern->min_length) && + nxt_http_route_memcmp(start, test, test_length, + pattern->case_sensitive)); - break; + case NXT_HTTP_ROUTE_PATTERN_BEGIN: + if (nxt_http_route_memcmp(start, test, test_length, + pattern->case_sensitive)) + { + start += test_length; + break; + } - case NXT_HTTP_ROUTE_PATTERN_BEGIN: - break; + return 0; - case NXT_HTTP_ROUTE_PATTERN_MIDDLE: - ret = nxt_http_route_memcmp(start, test, test_length, - pattern->case_sensitive); - if (!ret) { - return ret; - } + case NXT_HTTP_ROUTE_PATTERN_END: + p = end - test_length; - test = pattern->start2; - test_length = pattern->length2; + if (nxt_http_route_memcmp(p, test, test_length, + pattern->case_sensitive)) + { + end = p; + break; + } - /* Fall through. */ + return 0; - case NXT_HTTP_ROUTE_PATTERN_END: - start += length - test_length; - break; + case NXT_HTTP_ROUTE_PATTERN_SUBSTRING: + if (pattern->case_sensitive) { + p = nxt_memstrn(start, end, (char *) test, test_length); - case NXT_HTTP_ROUTE_PATTERN_SUBSTRING: - end = start + length; + } else { + p = nxt_memcasestrn(start, end, (char *) test, test_length); + } - if (pattern->case_sensitive) { - p = nxt_memstrn(start, end, (char *) test, test_length); + if (p == NULL) { + return 0; + } - } else { - p = nxt_memcasestrn(start, end, (char *) test, test_length); + start = p + test_length; } - - return (p != NULL); } - return nxt_http_route_memcmp(start, test, test_length, - pattern->case_sensitive); + return 1; } diff --git a/src/nxt_http_variables.c b/src/nxt_http_variables.c new file mode 100644 index 00000000..222d717c --- /dev/null +++ b/src/nxt_http_variables.c @@ -0,0 +1,59 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_router.h> +#include <nxt_http.h> + + +static nxt_int_t nxt_http_var_method(nxt_task_t *task, nxt_var_query_t *query, + nxt_str_t *str, void *ctx); +static nxt_int_t nxt_http_var_uri(nxt_task_t *task, nxt_var_query_t *query, + nxt_str_t *str, void *ctx); + + +static nxt_var_decl_t nxt_http_vars[] = { + { nxt_string("method"), + &nxt_http_var_method, + 0 }, + + { nxt_string("uri"), + &nxt_http_var_uri, + 0 }, +}; + + +nxt_int_t +nxt_http_register_variables(void) +{ + return nxt_var_register(nxt_http_vars, nxt_nitems(nxt_http_vars)); +} + + +static nxt_int_t +nxt_http_var_method(nxt_task_t *task, nxt_var_query_t *query, nxt_str_t *str, + void *ctx) +{ + nxt_http_request_t *r; + + r = ctx; + + *str = *r->method; + + return NXT_OK; +} + + +static nxt_int_t +nxt_http_var_uri(nxt_task_t *task, nxt_var_query_t *query, nxt_str_t *str, + void *ctx) +{ + nxt_http_request_t *r; + + r = ctx; + + *str = *r->path; + + return NXT_OK; +} diff --git a/src/nxt_http_websocket.c b/src/nxt_http_websocket.c index fb888f5d..1968633e 100644 --- a/src/nxt_http_websocket.c +++ b/src/nxt_http_websocket.c @@ -33,15 +33,13 @@ nxt_http_websocket_client(nxt_task_t *task, void *obj, void *data) nxt_buf_t *out, *buf, **out_tail, *b, *next; nxt_int_t res; nxt_http_request_t *r; - nxt_request_app_link_t *req_app_link; nxt_request_rpc_data_t *req_rpc_data; nxt_websocket_header_t *wsh; r = obj; + req_rpc_data = r->req_rpc_data; - if (nxt_slow_path((req_rpc_data = r->req_rpc_data) == NULL - || (req_app_link = req_rpc_data->req_app_link) == NULL)) - { + if (nxt_slow_path(req_rpc_data == NULL)) { nxt_debug(task, "websocket client frame for destroyed request"); return; @@ -69,7 +67,7 @@ nxt_http_websocket_client(nxt_task_t *task, void *obj, void *data) if (buf == NULL || buf_free_size == 0) { buf_free_size = nxt_min(frame_size, PORT_MMAP_DATA_SIZE); - buf = nxt_port_mmap_get_buf(task, req_app_link->app_port, + buf = nxt_port_mmap_get_buf(task, &req_rpc_data->app->outgoing, buf_free_size); *out_tail = buf; @@ -100,10 +98,10 @@ nxt_http_websocket_client(nxt_task_t *task, void *obj, void *data) b = next; } - res = nxt_port_socket_twrite(task, req_app_link->app_port, - NXT_PORT_MSG_WEBSOCKET, -1, - req_app_link->stream, - req_app_link->reply_port->id, out, NULL); + res = nxt_port_socket_write(task, req_rpc_data->app_port, + NXT_PORT_MSG_WEBSOCKET, -1, + req_rpc_data->stream, + task->thread->engine->port->id, out); if (nxt_slow_path(res != NXT_OK)) { // TODO: handle } @@ -129,32 +127,27 @@ static void nxt_http_websocket_error_handler(nxt_task_t *task, void *obj, void *data) { nxt_http_request_t *r; - nxt_request_app_link_t *req_app_link; nxt_request_rpc_data_t *req_rpc_data; nxt_debug(task, "http websocket error handler"); r = obj; + req_rpc_data = r->req_rpc_data; - if ((req_rpc_data = r->req_rpc_data) == NULL) { + if (req_rpc_data == NULL) { nxt_debug(task, " req_rpc_data is NULL"); goto close_handler; } - if ((req_app_link = req_rpc_data->req_app_link) == NULL) { - nxt_debug(task, " req_app_link is NULL"); - goto close_handler; - } - - if (req_app_link->app_port == NULL) { + if (req_rpc_data->app_port == NULL) { nxt_debug(task, " app_port is NULL"); goto close_handler; } - (void) nxt_port_socket_twrite(task, req_app_link->app_port, - NXT_PORT_MSG_WEBSOCKET_LAST, - -1, req_app_link->stream, - req_app_link->reply_port->id, NULL, NULL); + (void) nxt_port_socket_write(task, req_rpc_data->app_port, + NXT_PORT_MSG_WEBSOCKET_LAST, + -1, req_rpc_data->stream, + task->thread->engine->port->id, NULL); close_handler: diff --git a/src/nxt_java.c b/src/nxt_java.c index c7471509..1f8864bd 100644 --- a/src/nxt_java.c +++ b/src/nxt_java.c @@ -26,7 +26,8 @@ #include "java/nxt_jni_URLClassLoader.h" #include "nxt_jars.h" -#include "nxt_java_mounts.h" + +#include NXT_JAVA_MOUNTS_H static nxt_int_t nxt_java_setup(nxt_task_t *task, nxt_process_t *process, nxt_common_app_conf_t *conf); diff --git a/src/nxt_main.h b/src/nxt_main.h index 5914fbd1..7f812568 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -66,6 +66,7 @@ typedef uint16_t nxt_port_id_t; #include <nxt_sprintf.h> #include <nxt_parse.h> +#include <nxt_var.h> /* TODO: remove unused */ diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index a16e44d3..48eb2abb 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -605,25 +605,22 @@ nxt_main_start_process(nxt_task_t *task, nxt_process_t *process) nxt_process_port_add(task, process, port); - nxt_process_use(task, process, -1); - - ret = NXT_ERROR; - tmp_mp = NULL; - ret = nxt_port_socket_init(task, port, 0); if (nxt_slow_path(ret != NXT_OK)) { - goto fail; + goto free_port; } tmp_mp = nxt_mp_create(1024, 128, 256, 32); - if (tmp_mp == NULL) { - goto fail; + if (nxt_slow_path(tmp_mp == NULL)) { + ret = NXT_ERROR; + + goto close_port; } if (init->prefork) { ret = init->prefork(task, process, tmp_mp); if (nxt_slow_path(ret != NXT_OK)) { - goto fail; + goto free_mempool; } } @@ -632,18 +629,22 @@ nxt_main_start_process(nxt_task_t *task, nxt_process_t *process) switch (pid) { case -1: - nxt_port_close(task, port); + ret = NXT_ERROR; break; case 0: /* The child process: return to the event engine work queue loop. */ + nxt_process_use(task, process, -1); + ret = NXT_AGAIN; break; default: /* The main process created a new process. */ + nxt_process_use(task, process, -1); + nxt_port_read_close(port); nxt_port_write_enable(task, port); @@ -651,14 +652,20 @@ nxt_main_start_process(nxt_task_t *task, nxt_process_t *process) break; } -fail: +free_mempool: - nxt_port_use(task, port, -1); + nxt_mp_destroy(tmp_mp); + +close_port: - if (nxt_fast_path(tmp_mp != NULL)) { - nxt_mp_destroy(tmp_mp); + if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_port_close(task, port); } +free_port: + + nxt_port_use(task, port, -1); + return ret; } diff --git a/src/nxt_mp.c b/src/nxt_mp.c index 5c1a4d00..4eaa16d0 100644 --- a/src/nxt_mp.c +++ b/src/nxt_mp.c @@ -1059,3 +1059,17 @@ nxt_mp_cleanup(nxt_mp_t *mp, nxt_work_handler_t handler, return NXT_OK; } + + +void * +nxt_mp_lvlhsh_alloc(void *pool, size_t size) +{ + return nxt_mp_align(pool, size, size); +} + + +void +nxt_mp_lvlhsh_free(void *pool, void *p) +{ + nxt_mp_free(pool, p); +} diff --git a/src/nxt_mp.h b/src/nxt_mp.h index 53d1f011..a5aaabd1 100644 --- a/src/nxt_mp.h +++ b/src/nxt_mp.h @@ -112,4 +112,8 @@ NXT_EXPORT nxt_int_t nxt_mp_cleanup(nxt_mp_t *mp, nxt_work_handler_t handler, NXT_EXPORT void nxt_mp_thread_adopt(nxt_mp_t *mp); + +NXT_EXPORT void *nxt_mp_lvlhsh_alloc(void *pool, size_t size); +NXT_EXPORT void nxt_mp_lvlhsh_free(void *pool, void *p); + #endif /* _NXT_MP_H_INCLUDED_ */ diff --git a/src/nxt_nncq.h b/src/nxt_nncq.h new file mode 100644 index 00000000..20e7ecff --- /dev/null +++ b/src/nxt_nncq.h @@ -0,0 +1,162 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_NNCQ_H_INCLUDED_ +#define _NXT_NNCQ_H_INCLUDED_ + + +/* Numeric Naive Circular Queue */ + +#define NXT_NNCQ_SIZE 16384 + +typedef uint32_t nxt_nncq_atomic_t; +typedef uint16_t nxt_nncq_cycle_t; + +typedef struct { + nxt_nncq_atomic_t head; + nxt_nncq_atomic_t entries[NXT_NNCQ_SIZE]; + nxt_nncq_atomic_t tail; +} nxt_nncq_t; + + +static inline nxt_nncq_atomic_t +nxt_nncq_head(nxt_nncq_t const volatile *q) +{ + return q->head; +} + + +static inline nxt_nncq_atomic_t +nxt_nncq_tail(nxt_nncq_t const volatile *q) +{ + return q->tail; +} + + +static inline void +nxt_nncq_tail_cmp_inc(nxt_nncq_t volatile *q, nxt_nncq_atomic_t t) +{ + nxt_atomic_cmp_set(&q->tail, t, t + 1); +} + + +static inline nxt_nncq_atomic_t +nxt_nncq_index(nxt_nncq_t const volatile *q, nxt_nncq_atomic_t i) +{ + return i % NXT_NNCQ_SIZE; +} + + +static inline nxt_nncq_atomic_t +nxt_nncq_map(nxt_nncq_t const volatile *q, nxt_nncq_atomic_t i) +{ + return i % NXT_NNCQ_SIZE; +} + + +static inline nxt_nncq_cycle_t +nxt_nncq_cycle(nxt_nncq_t const volatile *q, nxt_nncq_atomic_t i) +{ + return i / NXT_NNCQ_SIZE; +} + + +static inline nxt_nncq_cycle_t +nxt_nncq_next_cycle(nxt_nncq_t const volatile *q, nxt_nncq_cycle_t i) +{ + return i + 1; +} + + +static inline nxt_nncq_atomic_t +nxt_nncq_new_entry(nxt_nncq_t const volatile *q, nxt_nncq_cycle_t cycle, + nxt_nncq_atomic_t i) +{ + return cycle * NXT_NNCQ_SIZE + (i % NXT_NNCQ_SIZE); +} + + +static inline nxt_nncq_atomic_t +nxt_nncq_empty(nxt_nncq_t const volatile *q) +{ + return NXT_NNCQ_SIZE; +} + + +static void +nxt_nncq_init(nxt_nncq_t volatile *q) +{ + q->head = NXT_NNCQ_SIZE; + nxt_memzero((void *) q->entries, NXT_NNCQ_SIZE * sizeof(nxt_nncq_atomic_t)); + q->tail = NXT_NNCQ_SIZE; +} + + +static void +nxt_nncq_enqueue(nxt_nncq_t volatile *q, nxt_nncq_atomic_t val) +{ + nxt_nncq_cycle_t e_cycle, t_cycle; + nxt_nncq_atomic_t n, t, e, j; + + for ( ;; ) { + t = nxt_nncq_tail(q); + j = nxt_nncq_map(q, t); + e = q->entries[j]; + + e_cycle = nxt_nncq_cycle(q, e); + t_cycle = nxt_nncq_cycle(q, t); + + if (e_cycle == t_cycle) { + nxt_nncq_tail_cmp_inc(q, t); + continue; + } + + if (nxt_nncq_next_cycle(q, e_cycle) != t_cycle) { + continue; + } + + n = nxt_nncq_new_entry(q, t_cycle, val); + + if (nxt_atomic_cmp_set(&q->entries[j], e, n)) { + break; + } + } + + nxt_nncq_tail_cmp_inc(q, t); +} + + +static nxt_nncq_atomic_t +nxt_nncq_dequeue(nxt_nncq_t volatile *q) +{ + nxt_nncq_cycle_t e_cycle, h_cycle; + nxt_nncq_atomic_t h, j, e; + + for ( ;; ) { + h = nxt_nncq_head(q); + j = nxt_nncq_map(q, h); + e = q->entries[j]; + + e_cycle = nxt_nncq_cycle(q, e); + h_cycle = nxt_nncq_cycle(q, h); + + if (e_cycle != h_cycle) { + if (nxt_nncq_next_cycle(q, e_cycle) == h_cycle) { + return nxt_nncq_empty(q); + } + + continue; + } + + if (nxt_atomic_cmp_set(&q->head, h, h + 1)) { + break; + } + } + + return nxt_nncq_index(q, e); +} + + +#endif /* _NXT_NNCQ_H_INCLUDED_ */ diff --git a/src/nxt_nvbcq.h b/src/nxt_nvbcq.h new file mode 100644 index 00000000..2b019dcc --- /dev/null +++ b/src/nxt_nvbcq.h @@ -0,0 +1,146 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_NVBCQ_H_INCLUDED_ +#define _NXT_NVBCQ_H_INCLUDED_ + + +/* Numeric VBart Circular Queue */ + +#define NXT_NVBCQ_SIZE 16384 + +typedef uint32_t nxt_nvbcq_atomic_t; + +struct nxt_nvbcq_s { + nxt_nvbcq_atomic_t head; + nxt_nvbcq_atomic_t entries[NXT_NVBCQ_SIZE]; + nxt_nvbcq_atomic_t tail; +}; + +typedef struct nxt_nvbcq_s nxt_nvbcq_t; + + +static inline nxt_nvbcq_atomic_t +nxt_nvbcq_head(nxt_nvbcq_t const volatile *q) +{ + return q->head; +} + + +static inline nxt_nvbcq_atomic_t +nxt_nvbcq_tail(nxt_nvbcq_t const volatile *q) +{ + return q->tail; +} + + +static inline void +nxt_nvbcq_tail_cmp_inc(nxt_nvbcq_t volatile *q, nxt_nvbcq_atomic_t t) +{ + nxt_atomic_cmp_set(&q->tail, t, t + 1); +} + + +static inline nxt_nvbcq_atomic_t +nxt_nvbcq_index(nxt_nvbcq_t const volatile *q, nxt_nvbcq_atomic_t i) +{ + return i % NXT_NVBCQ_SIZE; +} + + +static inline nxt_nvbcq_atomic_t +nxt_nvbcq_map(nxt_nvbcq_t const volatile *q, nxt_nvbcq_atomic_t i) +{ + return i % NXT_NVBCQ_SIZE; +} + + +static inline nxt_nvbcq_atomic_t +nxt_nvbcq_empty(nxt_nvbcq_t const volatile *q) +{ + return NXT_NVBCQ_SIZE; +} + + +static void +nxt_nvbcq_init(nxt_nvbcq_t volatile *q) +{ + nxt_nvbcq_atomic_t i; + + q->head = 0; + + for (i = 0; i < NXT_NVBCQ_SIZE; i++) { + q->entries[i] = NXT_NVBCQ_SIZE; + } + + q->tail = NXT_NVBCQ_SIZE; +} + + +static void +nxt_nvbcq_enqueue(nxt_nvbcq_t volatile *q, nxt_nvbcq_atomic_t val) +{ + nxt_nvbcq_atomic_t t, h, i; + + t = nxt_nvbcq_tail(q); + h = t - NXT_NVBCQ_SIZE; + + for ( ;; ) { + i = nxt_nvbcq_map(q, t); + + if (q->entries[i] == NXT_NVBCQ_SIZE + && nxt_atomic_cmp_set(&q->entries[i], NXT_NVBCQ_SIZE, val)) + { + nxt_nvbcq_tail_cmp_inc(q, t); + return; + } + + if ((t - h) == NXT_NVBCQ_SIZE) { + h = nxt_nvbcq_head(q); + + if ((t - h) == NXT_NVBCQ_SIZE) { + return; + } + } + + t++; + } +} + + +static nxt_nvbcq_atomic_t +nxt_nvbcq_dequeue(nxt_nvbcq_t volatile *q) +{ + nxt_nvbcq_atomic_t h, t, i, e; + + h = nxt_nvbcq_head(q); + t = h + NXT_NVBCQ_SIZE; + + for ( ;; ) { + i = nxt_nvbcq_map(q, h); + e = q->entries[i]; + + if (e < NXT_NVBCQ_SIZE + && nxt_atomic_cmp_set(&q->entries[i], e, NXT_NVBCQ_SIZE)) + { + nxt_atomic_cmp_set(&q->head, h, h + 1); + + return e; + } + + if ((t - h) == NXT_NVBCQ_SIZE) { + t = nxt_nvbcq_tail(q); + + if ((t - h) == NXT_NVBCQ_SIZE) { + return NXT_NVBCQ_SIZE; + } + } + + h++; + } +} + + +#endif /* _NXT_NVBCQ_H_INCLUDED_ */ diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 7ae8484d..bc8341f4 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -28,6 +28,9 @@ #if PHP_VERSION_ID >= 70000 #define NXT_PHP7 1 #endif +#if PHP_VERSION_ID >= 80000 +#define NXT_PHP8 1 +#endif /* PHP 8 */ #ifndef TSRMLS_CC @@ -61,7 +64,9 @@ typedef struct { } nxt_php_run_ctx_t; -#ifdef NXT_PHP7 +#if NXT_PHP8 +typedef int (*nxt_php_disable_t)(const char *p, size_t size); +#elif NXT_PHP7 typedef int (*nxt_php_disable_t)(char *p, size_t size); #else typedef int (*nxt_php_disable_t)(char *p, uint TSRMLS_DC); @@ -105,11 +110,15 @@ nxt_inline void nxt_php_set_str(nxt_unit_request_info_t *req, const char *name, static void nxt_php_set_cstr(nxt_unit_request_info_t *req, const char *name, const char *str, uint32_t len, zval *track_vars_array TSRMLS_DC); static void nxt_php_register_variables(zval *track_vars_array TSRMLS_DC); +#if NXT_PHP8 +static void nxt_php_log_message(const char *message, int syslog_type_int); +#else #ifdef NXT_HAVE_PHP_LOG_MESSAGE_WITH_SYSLOG_TYPE static void nxt_php_log_message(char *message, int syslog_type_int); #else static void nxt_php_log_message(char *message TSRMLS_DC); #endif +#endif #ifdef NXT_PHP7 static size_t nxt_php_unbuffered_write(const char *str, @@ -251,9 +260,9 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { static nxt_php_target_t *nxt_php_targets; static nxt_int_t nxt_php_last_target = -1; -static nxt_task_t *nxt_php_task; +static nxt_unit_ctx_t *nxt_php_unit_ctx; #if defined(ZTS) && PHP_VERSION_ID < 70400 -static void ***tsrm_ls; +static void ***tsrm_ls; #endif @@ -265,8 +274,6 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) nxt_str_t ini_path, name; nxt_int_t ret; nxt_uint_t n; - nxt_port_t *my_port, *main_port; - nxt_runtime_t *rt; nxt_unit_ctx_t *unit_ctx; nxt_unit_init_t php_init; nxt_conf_value_t *value; @@ -277,8 +284,6 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) static nxt_str_t user_str = nxt_string("user"); static nxt_str_t admin_str = nxt_string("admin"); - nxt_php_task = task; - conf = data->app; c = &conf->u.php; @@ -365,39 +370,13 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) nxt_php_set_options(task, value, ZEND_INI_USER); } - rt = task->thread->runtime; - - main_port = rt->port_by_type[NXT_PROCESS_MAIN]; - if (nxt_slow_path(main_port == NULL)) { - nxt_alert(task, "main process not found"); - return NXT_ERROR; - } - - my_port = nxt_runtime_port_find(rt, nxt_pid, 0); - if (nxt_slow_path(my_port == NULL)) { - nxt_alert(task, "my_port not found"); - return NXT_ERROR; + ret = nxt_unit_default_init(task, &php_init); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_alert(task, "nxt_unit_default_init() failed"); + return ret; } - nxt_memzero(&php_init, sizeof(nxt_unit_init_t)); - php_init.callbacks.request_handler = nxt_php_request_handler; - - php_init.ready_port.id.pid = main_port->pid; - php_init.ready_port.id.id = main_port->id; - php_init.ready_port.out_fd = main_port->pair[1]; - - nxt_fd_blocking(task, main_port->pair[1]); - - php_init.ready_stream = my_port->process->stream; - - php_init.read_port.id.pid = my_port->pid; - php_init.read_port.id.id = my_port->id; - php_init.read_port.in_fd = my_port->pair[0]; - - nxt_fd_blocking(task, my_port->pair[0]); - - php_init.log_fd = 2; php_init.shm_limit = conf->shm_limit; unit_ctx = nxt_unit_init(&php_init); @@ -405,6 +384,8 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) return NXT_ERROR; } + nxt_php_unit_ctx = unit_ctx; + nxt_unit_run(unit_ctx); nxt_unit_done(unit_ctx); @@ -430,11 +411,6 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, value = nxt_conf_get_object_member(conf, &root_str, NULL); - if (value == NULL) { - nxt_alert(task, "no php root specified"); - return NXT_ERROR; - } - nxt_conf_get_string(value, &str); tmp = nxt_malloc(str.length + 1); @@ -707,7 +683,11 @@ nxt_php_dirname(const nxt_str_t *file, nxt_str_t *dir) { size_t length; - nxt_assert(file->length > 0 && file->start[0] == '/'); + if (file->length == 0 || file->start[0] != '/') { + nxt_unit_alert(NULL, "php_dirname: invalid file name " + "(not starts from '/')"); + return NXT_ERROR; + } length = file->length; @@ -1269,6 +1249,10 @@ nxt_php_set_cstr(nxt_unit_request_info_t *req, const char *name, } +#if NXT_PHP8 +static void +nxt_php_log_message(const char *message, int syslog_type_int) +#else #ifdef NXT_HAVE_PHP_LOG_MESSAGE_WITH_SYSLOG_TYPE static void nxt_php_log_message(char *message, int syslog_type_int) @@ -1276,6 +1260,18 @@ nxt_php_log_message(char *message, int syslog_type_int) static void nxt_php_log_message(char *message TSRMLS_DC) #endif +#endif { - nxt_log(nxt_php_task, NXT_LOG_NOTICE, "php message: %s", message); + nxt_php_run_ctx_t *ctx; + + ctx = SG(server_context); + + if (ctx != NULL) { + nxt_unit_req_log(ctx->req, NXT_UNIT_LOG_NOTICE, + "php message: %s", message); + + } else { + nxt_unit_log(nxt_php_unit_ctx, NXT_UNIT_LOG_NOTICE, + "php message: %s", message); + } } diff --git a/src/nxt_port.c b/src/nxt_port.c index 7232c465..dbcdec11 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -8,6 +8,7 @@ #include <nxt_runtime.h> #include <nxt_port.h> #include <nxt_router.h> +#include <nxt_port_queue.h> static void nxt_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); @@ -67,8 +68,8 @@ nxt_port_new(nxt_task_t *task, nxt_port_id_t id, nxt_pid_t pid, nxt_queue_init(&port->messages); nxt_thread_mutex_create(&port->write_mutex); - nxt_queue_init(&port->pending_requests); - nxt_queue_init(&port->active_websockets); + + port->queue_fd = -1; } else { nxt_mp_destroy(mp); @@ -101,6 +102,16 @@ nxt_port_close(nxt_task_t *task, nxt_port_t *port) nxt_router_app_port_close(task, port); } } + + if (port->queue_fd != -1) { + nxt_fd_close(port->queue_fd); + port->queue_fd = -1; + } + + if (port->queue != NULL) { + nxt_mem_munmap(port->queue, sizeof(nxt_port_queue_t)); + port->queue = NULL; + } } @@ -178,6 +189,7 @@ nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* TODO join with process_ready and move to nxt_main_process.c */ nxt_inline void nxt_port_send_new_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_port_t *new_port, uint32_t stream) @@ -229,8 +241,9 @@ nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, nxt_port_t *new_port, msg->max_share = port->max_share; msg->type = new_port->type; - return nxt_port_socket_write(task, port, NXT_PORT_MSG_NEW_PORT, - new_port->pair[1], stream, 0, b); + return nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, + new_port->pair[1], new_port->queue_fd, + stream, 0, b); } @@ -248,15 +261,15 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* TODO check b size and make plain */ nxt_debug(task, "new port %d received for process %PI:%d", - msg->fd, new_port_msg->pid, new_port_msg->id); + msg->fd[0], new_port_msg->pid, new_port_msg->id); port = nxt_runtime_port_find(rt, new_port_msg->pid, new_port_msg->id); if (port != NULL) { nxt_debug(task, "port %PI:%d already exists", new_port_msg->pid, new_port_msg->id); - nxt_fd_close(msg->fd); - msg->fd = -1; + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; return; } @@ -267,10 +280,10 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - nxt_fd_nonblocking(task, msg->fd); + nxt_fd_nonblocking(task, msg->fd[0]); port->pair[0] = -1; - port->pair[1] = msg->fd; + port->pair[1] = msg->fd[0]; port->max_size = new_port_msg->max_size; port->max_share = new_port_msg->max_share; @@ -281,7 +294,7 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) msg->u.new_port = port; } - +/* TODO move to nxt_main_process.c */ void nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -306,6 +319,13 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "process %PI ready", msg->port_msg.pid); + if (msg->fd[0] != -1) { + port->queue_fd = msg->fd[0]; + port->queue = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, MAP_SHARED, + msg->fd[0], 0); + } + nxt_port_send_new_port(task, rt, port, msg->port_msg.stream); } @@ -318,7 +338,7 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; - if (nxt_slow_path(msg->fd == -1)) { + if (nxt_slow_path(msg->fd[0] == -1)) { nxt_log(task, NXT_LOG_WARN, "invalid fd passed with mmap message"); return; @@ -332,11 +352,11 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) goto fail_close; } - nxt_port_incoming_port_mmap(task, process, msg->fd); + nxt_port_incoming_port_mmap(task, process, msg->fd[0]); fail_close: - nxt_fd_close(msg->fd); + nxt_fd_close(msg->fd[0]); } @@ -389,14 +409,14 @@ nxt_port_change_log_file_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) log_file = nxt_list_elt(rt->log_files, slot); - nxt_debug(task, "change log file %FD:%FD", msg->fd, log_file->fd); + nxt_debug(task, "change log file %FD:%FD", msg->fd[0], log_file->fd); /* * The old log file descriptor must be closed at the moment when no * other threads use it. dup2() allows to use the old file descriptor * for new log file. This change is performed atomically in the kernel. */ - if (nxt_file_redirect(log_file, msg->fd) == NXT_OK) { + if (nxt_file_redirect(log_file, msg->fd[0]) == NXT_OK) { if (slot == 0) { (void) nxt_file_stderr(log_file); } diff --git a/src/nxt_port.h b/src/nxt_port.h index 0e8707f3..3ac8c735 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -25,7 +25,9 @@ struct nxt_port_handlers_s { /* File descriptor exchange. */ nxt_port_handler_t change_file; nxt_port_handler_t new_port; + nxt_port_handler_t get_port; nxt_port_handler_t mmap; + nxt_port_handler_t get_mmap; /* New process */ nxt_port_handler_t process_created; @@ -39,6 +41,8 @@ struct nxt_port_handlers_s { /* Request headers. */ nxt_port_handler_t req_headers; + nxt_port_handler_t req_headers_ack; + nxt_port_handler_t req_body; /* Websocket frame. */ nxt_port_handler_t websocket_frame; @@ -48,6 +52,8 @@ struct nxt_port_handlers_s { nxt_port_handler_t oosm; nxt_port_handler_t shm_ack; + nxt_port_handler_t read_queue; + nxt_port_handler_t read_socket; }; @@ -77,7 +83,9 @@ typedef enum { _NXT_PORT_MSG_CHANGE_FILE = nxt_port_handler_idx(change_file), _NXT_PORT_MSG_NEW_PORT = nxt_port_handler_idx(new_port), + _NXT_PORT_MSG_GET_PORT = nxt_port_handler_idx(get_port), _NXT_PORT_MSG_MMAP = nxt_port_handler_idx(mmap), + _NXT_PORT_MSG_GET_MMAP = nxt_port_handler_idx(get_mmap), _NXT_PORT_MSG_PROCESS_CREATED = nxt_port_handler_idx(process_created), _NXT_PORT_MSG_PROCESS_READY = nxt_port_handler_idx(process_ready), @@ -85,12 +93,16 @@ typedef enum { _NXT_PORT_MSG_QUIT = nxt_port_handler_idx(quit), _NXT_PORT_MSG_REQ_HEADERS = nxt_port_handler_idx(req_headers), + _NXT_PORT_MSG_REQ_HEADERS_ACK = nxt_port_handler_idx(req_headers_ack), + _NXT_PORT_MSG_REQ_BODY = nxt_port_handler_idx(req_body), _NXT_PORT_MSG_WEBSOCKET = nxt_port_handler_idx(websocket_frame), _NXT_PORT_MSG_DATA = nxt_port_handler_idx(data), _NXT_PORT_MSG_OOSM = nxt_port_handler_idx(oosm), _NXT_PORT_MSG_SHM_ACK = nxt_port_handler_idx(shm_ack), + _NXT_PORT_MSG_READ_QUEUE = nxt_port_handler_idx(read_queue), + _NXT_PORT_MSG_READ_SOCKET = nxt_port_handler_idx(read_socket), NXT_PORT_MSG_MAX = sizeof(nxt_port_handlers_t) / sizeof(nxt_port_handler_t), @@ -107,8 +119,10 @@ typedef enum { NXT_PORT_MSG_ACCESS_LOG = nxt_msg_last(_NXT_PORT_MSG_ACCESS_LOG), NXT_PORT_MSG_CHANGE_FILE = nxt_msg_last(_NXT_PORT_MSG_CHANGE_FILE), NXT_PORT_MSG_NEW_PORT = nxt_msg_last(_NXT_PORT_MSG_NEW_PORT), + NXT_PORT_MSG_GET_PORT = nxt_msg_last(_NXT_PORT_MSG_GET_PORT), NXT_PORT_MSG_MMAP = nxt_msg_last(_NXT_PORT_MSG_MMAP) - | NXT_PORT_MSG_CLOSE_FD | NXT_PORT_MSG_SYNC, + | NXT_PORT_MSG_SYNC, + NXT_PORT_MSG_GET_MMAP = nxt_msg_last(_NXT_PORT_MSG_GET_MMAP), NXT_PORT_MSG_PROCESS_CREATED = nxt_msg_last(_NXT_PORT_MSG_PROCESS_CREATED), NXT_PORT_MSG_PROCESS_READY = nxt_msg_last(_NXT_PORT_MSG_PROCESS_READY), @@ -116,6 +130,7 @@ typedef enum { NXT_PORT_MSG_REMOVE_PID = nxt_msg_last(_NXT_PORT_MSG_REMOVE_PID), NXT_PORT_MSG_REQ_HEADERS = _NXT_PORT_MSG_REQ_HEADERS, + NXT_PORT_MSG_REQ_BODY = _NXT_PORT_MSG_REQ_BODY, NXT_PORT_MSG_WEBSOCKET = _NXT_PORT_MSG_WEBSOCKET, NXT_PORT_MSG_WEBSOCKET_LAST = nxt_msg_last(_NXT_PORT_MSG_WEBSOCKET), @@ -124,6 +139,8 @@ typedef enum { NXT_PORT_MSG_OOSM = nxt_msg_last(_NXT_PORT_MSG_OOSM), NXT_PORT_MSG_SHM_ACK = nxt_msg_last(_NXT_PORT_MSG_SHM_ACK), + NXT_PORT_MSG_READ_QUEUE = _NXT_PORT_MSG_READ_QUEUE, + NXT_PORT_MSG_READ_SOCKET = _NXT_PORT_MSG_READ_SOCKET, } nxt_port_msg_type_t; @@ -156,7 +173,7 @@ typedef struct { nxt_queue_link_t link; nxt_buf_t *buf; size_t share; - nxt_fd_t fd; + nxt_fd_t fd[2]; nxt_port_msg_t port_msg; uint32_t tracking_msg[2]; uint8_t close_fd; /* 1 bit */ @@ -165,7 +182,7 @@ typedef struct { struct nxt_port_recv_msg_s { - nxt_fd_t fd; + nxt_fd_t fd[2]; nxt_buf_t *buf; nxt_port_t *port; nxt_port_msg_t port_msg; @@ -188,6 +205,7 @@ struct nxt_port_s { nxt_queue_link_t app_link; /* for nxt_app_t.ports */ nxt_app_t *app; + nxt_port_t *main_app_port; nxt_queue_link_t idle_link; /* for nxt_app_t.idle_ports */ nxt_msec_t idle_start; @@ -200,11 +218,10 @@ struct nxt_port_s { /* Maximum interleave of message parts. */ uint32_t max_share; - uint32_t app_pending_responses; uint32_t app_responses; - nxt_queue_t pending_requests; - nxt_queue_t active_websockets; + uint32_t active_websockets; + uint32_t active_requests; nxt_port_handler_t handler; nxt_port_handler_t *data; @@ -226,6 +243,12 @@ struct nxt_port_s { nxt_atomic_t use_count; nxt_process_type_t type; + + nxt_fd_t queue_fd; + void *queue; + + void *socket_msg; + int from_socket; }; @@ -238,6 +261,17 @@ typedef struct { } nxt_port_msg_new_port_t; +typedef struct { + nxt_port_id_t id; + nxt_pid_t pid; +} nxt_port_msg_get_port_t; + + +typedef struct { + uint32_t id; +} nxt_port_msg_get_mmap_t; + + /* * nxt_port_data_t size is allocation size * which enables effective reuse of memory pool cache. @@ -265,17 +299,17 @@ void nxt_port_write_enable(nxt_task_t *task, nxt_port_t *port); void nxt_port_write_close(nxt_port_t *port); void nxt_port_read_enable(nxt_task_t *task, nxt_port_t *port); void nxt_port_read_close(nxt_port_t *port); -nxt_int_t nxt_port_socket_twrite(nxt_task_t *task, nxt_port_t *port, - nxt_uint_t type, nxt_fd_t fd, uint32_t stream, nxt_port_id_t reply_port, - nxt_buf_t *b, void *tracking); +nxt_int_t nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, + nxt_uint_t type, nxt_fd_t fd, nxt_fd_t fd2, uint32_t stream, + nxt_port_id_t reply_port, nxt_buf_t *b); nxt_inline nxt_int_t nxt_port_socket_write(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, nxt_fd_t fd, uint32_t stream, nxt_port_id_t reply_port, nxt_buf_t *b) { - return nxt_port_socket_twrite(task, port, type, fd, stream, reply_port, b, - NULL); + return nxt_port_socket_write2(task, port, type, fd, -1, stream, reply_port, + b); } void nxt_port_enable(nxt_task_t *task, nxt_port_t *port, diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index f4d2125c..1e01629e 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -282,11 +282,10 @@ fail: static nxt_port_mmap_handler_t * -nxt_port_new_port_mmap(nxt_task_t *task, nxt_process_t *process, - nxt_port_t *port, nxt_bool_t tracking, nxt_int_t n) +nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, + nxt_bool_t tracking, nxt_int_t n) { void *mem; - u_char *p, name[64]; nxt_fd_t fd; nxt_int_t i; nxt_free_map_t *free_map; @@ -296,24 +295,87 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_process_t *process, mmap_handler = nxt_zalloc(sizeof(nxt_port_mmap_handler_t)); if (nxt_slow_path(mmap_handler == NULL)) { - nxt_log(task, NXT_LOG_WARN, "failed to allocate mmap_handler"); + nxt_alert(task, "failed to allocate mmap_handler"); return NULL; } - port_mmap = nxt_port_mmap_at(&process->outgoing, process->outgoing.size); + port_mmap = nxt_port_mmap_at(mmaps, mmaps->size); if (nxt_slow_path(port_mmap == NULL)) { - nxt_log(task, NXT_LOG_WARN, - "failed to add port mmap to outgoing array"); + nxt_alert(task, "failed to add port mmap to mmaps array"); nxt_free(mmap_handler); return NULL; } + fd = nxt_shm_open(task, PORT_MMAP_SIZE); + if (nxt_slow_path(fd == -1)) { + goto remove_fail; + } + + mem = nxt_mem_mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + + if (nxt_slow_path(mem == MAP_FAILED)) { + goto remove_fail; + } + + mmap_handler->hdr = mem; + mmap_handler->fd = fd; + port_mmap->mmap_handler = mmap_handler; + nxt_port_mmap_handler_use(mmap_handler, 1); + + /* Init segment header. */ + hdr = mmap_handler->hdr; + + nxt_memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); + nxt_memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); + + hdr->id = mmaps->size - 1; + hdr->src_pid = nxt_pid; + hdr->sent_over = 0xFFFFu; + + /* Mark first chunk as busy */ + free_map = tracking ? hdr->free_tracking_map : hdr->free_map; + + for (i = 0; i < n; i++) { + nxt_port_mmap_set_chunk_busy(free_map, i); + } + + /* Mark as busy chunk followed the last available chunk. */ + nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); + nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); + + nxt_log(task, NXT_LOG_DEBUG, "new mmap #%D created for %PI -> ...", + hdr->id, nxt_pid); + + return mmap_handler; + +remove_fail: + + nxt_free(mmap_handler); + + mmaps->size--; + + return NULL; +} + + +nxt_int_t +nxt_shm_open(nxt_task_t *task, size_t size) +{ + nxt_fd_t fd; + +#if (NXT_HAVE_MEMFD_CREATE || NXT_HAVE_SHM_OPEN) + + u_char *p, name[64]; + p = nxt_sprintf(name, name + sizeof(name), NXT_SHM_PREFIX "unit.%PI.%uxD", nxt_pid, nxt_random(&task->thread->random)); *p = '\0'; +#endif + #if (NXT_HAVE_MEMFD_CREATE) fd = syscall(SYS_memfd_create, name, MFD_CLOEXEC); @@ -321,7 +383,7 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_process_t *process, if (nxt_slow_path(fd == -1)) { nxt_alert(task, "memfd_create(%s) failed %E", name, nxt_errno); - goto remove_fail; + return -1; } nxt_debug(task, "memfd_create(%s): %FD", name, fd); @@ -330,14 +392,14 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_process_t *process, fd = shm_open(SHM_ANON, O_RDWR, S_IRUSR | S_IWUSR); - nxt_debug(task, "shm_open(SHM_ANON): %FD", fd); - if (nxt_slow_path(fd == -1)) { nxt_alert(task, "shm_open(SHM_ANON) failed %E", nxt_errno); - goto remove_fail; + return -1; } + nxt_debug(task, "shm_open(SHM_ANON): %FD", fd); + #elif (NXT_HAVE_SHM_OPEN) /* Just in case. */ @@ -345,14 +407,14 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_process_t *process, fd = shm_open((char *) name, O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); - nxt_debug(task, "shm_open(%s): %FD", name, fd); - if (nxt_slow_path(fd == -1)) { nxt_alert(task, "shm_open(%s) failed %E", name, nxt_errno); - goto remove_fail; + return -1; } + nxt_debug(task, "shm_open(%s): %FD", name, fd); + if (nxt_slow_path(shm_unlink((char *) name) == -1)) { nxt_log(task, NXT_LOG_WARN, "shm_unlink(%s) failed %E", name, nxt_errno); @@ -364,94 +426,41 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_process_t *process, #endif - if (nxt_slow_path(ftruncate(fd, PORT_MMAP_SIZE) == -1)) { - nxt_log(task, NXT_LOG_WARN, "ftruncate() failed %E", nxt_errno); - - goto remove_fail; - } - - mem = nxt_mem_mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, - MAP_SHARED, fd, 0); + if (nxt_slow_path(ftruncate(fd, size) == -1)) { + nxt_alert(task, "ftruncate() failed %E", nxt_errno); - if (nxt_slow_path(mem == MAP_FAILED)) { - goto remove_fail; - } + nxt_fd_close(fd); - mmap_handler->hdr = mem; - port_mmap->mmap_handler = mmap_handler; - nxt_port_mmap_handler_use(mmap_handler, 1); - - /* Init segment header. */ - hdr = mmap_handler->hdr; - - nxt_memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); - nxt_memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); - - hdr->id = process->outgoing.size - 1; - hdr->src_pid = nxt_pid; - hdr->dst_pid = process->pid; - hdr->sent_over = port->id; - - /* Mark first chunk as busy */ - free_map = tracking ? hdr->free_tracking_map : hdr->free_map; - - for (i = 0; i < n; i++) { - nxt_port_mmap_set_chunk_busy(free_map, i); + return -1; } - /* Mark as busy chunk followed the last available chunk. */ - nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); - nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); - - nxt_debug(task, "send mmap fd %FD to process %PI", fd, port->pid); - - /* TODO handle error */ - (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_MMAP, fd, 0, 0, NULL); - - nxt_log(task, NXT_LOG_DEBUG, "new mmap #%D created for %PI -> %PI", - hdr->id, nxt_pid, process->pid); - - return mmap_handler; - -remove_fail: - - nxt_free(mmap_handler); - - process->outgoing.size--; - - return NULL; + return fd; } static nxt_port_mmap_handler_t * -nxt_port_mmap_get(nxt_task_t *task, nxt_port_t *port, nxt_chunk_id_t *c, +nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, nxt_int_t n, nxt_bool_t tracking) { nxt_int_t i, res, nchunks; - nxt_process_t *process; nxt_free_map_t *free_map; nxt_port_mmap_t *port_mmap; nxt_port_mmap_t *end_port_mmap; nxt_port_mmap_header_t *hdr; nxt_port_mmap_handler_t *mmap_handler; - process = port->process; - if (nxt_slow_path(process == NULL)) { - return NULL; - } - - nxt_thread_mutex_lock(&process->outgoing.mutex); + nxt_thread_mutex_lock(&mmaps->mutex); - end_port_mmap = process->outgoing.elts + process->outgoing.size; + end_port_mmap = mmaps->elts + mmaps->size; - for (port_mmap = process->outgoing.elts; + for (port_mmap = mmaps->elts; port_mmap < end_port_mmap; port_mmap++) { mmap_handler = port_mmap->mmap_handler; hdr = mmap_handler->hdr; - if (hdr->sent_over != 0xFFFFu && hdr->sent_over != port->id) { + if (hdr->sent_over != 0xFFFFu) { continue; } @@ -489,11 +498,11 @@ nxt_port_mmap_get(nxt_task_t *task, nxt_port_t *port, nxt_chunk_id_t *c, /* TODO introduce port_mmap limit and release wait. */ *c = 0; - mmap_handler = nxt_port_new_port_mmap(task, process, port, tracking, n); + mmap_handler = nxt_port_new_port_mmap(task, mmaps, tracking, n); unlock_return: - nxt_thread_mutex_unlock(&process->outgoing.mutex); + nxt_thread_mutex_unlock(&mmaps->mutex); return mmap_handler; } @@ -528,7 +537,7 @@ nxt_port_get_port_incoming_mmap(nxt_task_t *task, nxt_pid_t spid, uint32_t id) nxt_int_t -nxt_port_mmap_get_tracking(nxt_task_t *task, nxt_port_t *port, +nxt_port_mmap_get_tracking(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_port_mmap_tracking_t *tracking, uint32_t stream) { nxt_chunk_id_t c; @@ -537,7 +546,7 @@ nxt_port_mmap_get_tracking(nxt_task_t *task, nxt_port_t *port, nxt_debug(task, "request tracking for stream #%uD", stream); - mmap_handler = nxt_port_mmap_get(task, port, &c, 1, 1); + mmap_handler = nxt_port_mmap_get(task, mmaps, &c, 1, 1); if (nxt_slow_path(mmap_handler == NULL)) { return NXT_ERROR; } @@ -659,7 +668,7 @@ nxt_port_mmap_tracking_read(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_buf_t * -nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_t *port, size_t size) +nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_mmaps_t *mmaps, size_t size) { nxt_mp_t *mp; nxt_buf_t *b; @@ -686,7 +695,7 @@ nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_t *port, size_t size) b->completion_handler = nxt_port_mmap_buf_completion; nxt_buf_set_port_mmap(b); - mmap_handler = nxt_port_mmap_get(task, port, &c, nchunks, 0); + mmap_handler = nxt_port_mmap_get(task, mmaps, &c, nchunks, 0); if (nxt_slow_path(mmap_handler == NULL)) { mp = task->thread->engine->mem_pool; nxt_mp_free(mp, b); @@ -922,9 +931,7 @@ nxt_port_mmap_read(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_method_t nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) { - nxt_port_method_t m; - nxt_port_mmap_header_t *hdr; - nxt_port_mmap_handler_t *mmap_handler; + nxt_port_method_t m; m = NXT_PORT_METHOD_ANY; @@ -935,9 +942,6 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) } if (nxt_buf_is_port_mmap(b)) { - mmap_handler = b->parent; - hdr = mmap_handler->hdr; - if (m == NXT_PORT_METHOD_PLAIN) { nxt_log_error(NXT_LOG_ERR, task->log, "mixing plain and mmap buffers, " @@ -946,16 +950,6 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) break; } - if (port->pid != hdr->dst_pid) { - nxt_log_error(NXT_LOG_ERR, task->log, - "send mmap buffer for %PI to %PI, " - "using plain mode", hdr->dst_pid, port->pid); - - m = NXT_PORT_METHOD_PLAIN; - - break; - } - if (m == NXT_PORT_METHOD_ANY) { nxt_debug(task, "using mmap mode"); diff --git a/src/nxt_port_memory.h b/src/nxt_port_memory.h index 748549b1..8e71af3d 100644 --- a/src/nxt_port_memory.h +++ b/src/nxt_port_memory.h @@ -23,7 +23,7 @@ struct nxt_port_mmap_tracking_s { }; nxt_int_t -nxt_port_mmap_get_tracking(nxt_task_t *task, nxt_port_t *port, +nxt_port_mmap_get_tracking(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_port_mmap_tracking_t *tracking, uint32_t stream); nxt_bool_t @@ -37,14 +37,12 @@ nxt_bool_t nxt_port_mmap_tracking_read(nxt_task_t *task, nxt_port_recv_msg_t *msg); /* - * Allocates nxt_but_t structure from port's mem_pool, assigns this buf 'mem' - * pointers to first available shared mem bucket(s). 'size' used as a hint to - * acquire several successive buckets if possible. - * - * This function assumes that current thread operates the 'port' exclusively. + * Allocates nxt_but_t structure from task's thread engine mem_pool, assigns + * this buf 'mem' pointers to first available shared mem bucket(s). 'size' + * used as a hint to acquire several successive buckets if possible. */ nxt_buf_t * -nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_t *port, size_t size); +nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_mmaps_t *mmaps, size_t size); nxt_int_t nxt_port_mmap_increase_buf(nxt_task_t *task, nxt_buf_t *b, size_t size, size_t min_size); @@ -71,5 +69,6 @@ typedef enum nxt_port_method_e nxt_port_method_t; nxt_port_method_t nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b); +nxt_int_t nxt_shm_open(nxt_task_t *task, size_t size); #endif /* _NXT_PORT_MEMORY_H_INCLUDED_ */ diff --git a/src/nxt_port_memory_int.h b/src/nxt_port_memory_int.h index 87c3d833..d2524ee4 100644 --- a/src/nxt_port_memory_int.h +++ b/src/nxt_port_memory_int.h @@ -63,6 +63,7 @@ struct nxt_port_mmap_header_s { struct nxt_port_mmap_handler_s { nxt_port_mmap_header_t *hdr; nxt_atomic_t use_count; + nxt_fd_t fd; }; /* diff --git a/src/nxt_port_queue.h b/src/nxt_port_queue.h new file mode 100644 index 00000000..d2b2326b --- /dev/null +++ b/src/nxt_port_queue.h @@ -0,0 +1,102 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_PORT_QUEUE_H_INCLUDED_ +#define _NXT_PORT_QUEUE_H_INCLUDED_ + + +#include <nxt_nncq.h> + + +/* Using Numeric Naive Circular Queue as a backend. */ + +#define NXT_PORT_QUEUE_SIZE NXT_NNCQ_SIZE +#define NXT_PORT_QUEUE_MSG_SIZE 31 + + +typedef struct { + uint8_t size; + uint8_t data[NXT_PORT_QUEUE_MSG_SIZE]; +} nxt_port_queue_item_t; + + +typedef struct { + nxt_nncq_atomic_t nitems; + nxt_nncq_t free_items; + nxt_nncq_t queue; + nxt_port_queue_item_t items[NXT_PORT_QUEUE_SIZE]; +} nxt_port_queue_t; + + +nxt_inline void +nxt_port_queue_init(nxt_port_queue_t volatile *q) +{ + nxt_nncq_atomic_t i; + + nxt_nncq_init(&q->free_items); + nxt_nncq_init(&q->queue); + + for (i = 0; i < NXT_PORT_QUEUE_SIZE; i++) { + nxt_nncq_enqueue(&q->free_items, i); + } + + q->nitems = 0; +} + + +nxt_inline nxt_int_t +nxt_port_queue_send(nxt_port_queue_t volatile *q, const void *p, uint8_t size, + int *notify) +{ + nxt_nncq_atomic_t i; + nxt_port_queue_item_t *qi; + + i = nxt_nncq_dequeue(&q->free_items); + if (i == nxt_nncq_empty(&q->free_items)) { + *notify = 0; + return NXT_AGAIN; + } + + qi = (nxt_port_queue_item_t *) &q->items[i]; + + qi->size = size; + nxt_memcpy(qi->data, p, size); + + nxt_nncq_enqueue(&q->queue, i); + + i = nxt_atomic_fetch_add(&q->nitems, 1); + + *notify = (i == 0); + + return NXT_OK; +} + + +nxt_inline ssize_t +nxt_port_queue_recv(nxt_port_queue_t volatile *q, void *p) +{ + ssize_t res; + nxt_nncq_atomic_t i; + nxt_port_queue_item_t *qi; + + i = nxt_nncq_dequeue(&q->queue); + if (i == nxt_nncq_empty(&q->queue)) { + return -1; + } + + qi = (nxt_port_queue_item_t *) &q->items[i]; + + res = qi->size; + nxt_memcpy(p, qi->data, qi->size); + + nxt_nncq_enqueue(&q->free_items, i); + + nxt_atomic_fetch_add(&q->nitems, -1); + + return res; +} + + +#endif /* _NXT_PORT_QUEUE_H_INCLUDED_ */ diff --git a/src/nxt_port_rpc.c b/src/nxt_port_rpc.c index 37f2d902..f4008a18 100644 --- a/src/nxt_port_rpc.c +++ b/src/nxt_port_rpc.c @@ -389,7 +389,8 @@ nxt_port_rpc_remove_peer(nxt_task_t *task, nxt_port_t *port, nxt_pid_t peer) nxt_memzero(&msg, sizeof(msg)); nxt_memzero(&buf, sizeof(buf)); - msg.fd = -1; + msg.fd[0] = -1; + msg.fd[1] = -1; msg.buf = &buf; msg.port = port; @@ -500,7 +501,8 @@ nxt_port_rpc_close(nxt_task_t *task, nxt_port_t *port) return; } - msg.fd = -1; + msg.fd[0] = -1; + msg.fd[1] = -1; msg.buf = &nxt_port_close_dummy_buf; msg.port = port; msg.port_msg.stream = reg->stream; diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 4e3eaef6..9d8096b2 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -5,6 +5,7 @@ */ #include <nxt_main.h> +#include <nxt_port_queue.h> static nxt_int_t nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, @@ -17,6 +18,8 @@ static nxt_buf_t *nxt_port_buf_completion(nxt_task_t *task, static nxt_port_send_msg_t *nxt_port_msg_insert_tail(nxt_port_t *port, nxt_port_send_msg_t *msg); static void nxt_port_read_handler(nxt_task_t *task, void *obj, void *data); +static void nxt_port_queue_read_handler(nxt_task_t *task, void *obj, + void *data); static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg); static nxt_buf_t *nxt_port_buf_alloc(nxt_port_t *port); @@ -143,26 +146,26 @@ nxt_port_release_send_msg(nxt_port_send_msg_t *msg) nxt_int_t -nxt_port_socket_twrite(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, - nxt_fd_t fd, uint32_t stream, nxt_port_id_t reply_port, nxt_buf_t *b, - void *tracking) +nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, + nxt_fd_t fd, nxt_fd_t fd2, uint32_t stream, nxt_port_id_t reply_port, + nxt_buf_t *b) { + int notify; + uint8_t *p; nxt_int_t res; nxt_port_send_msg_t msg; + uint8_t qmsg[NXT_PORT_QUEUE_MSG_SIZE]; msg.link.next = NULL; msg.link.prev = NULL; msg.buf = b; msg.share = 0; - msg.fd = fd; + msg.fd[0] = fd; + msg.fd[1] = fd2; msg.close_fd = (type & NXT_PORT_MSG_CLOSE_FD) != 0; msg.allocated = 0; - if (tracking != NULL) { - nxt_port_mmap_tracking_write(msg.tracking_msg, tracking); - } - msg.port_msg.stream = stream; msg.port_msg.pid = nxt_pid; msg.port_msg.reply_port = reply_port; @@ -171,7 +174,46 @@ nxt_port_socket_twrite(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, msg.port_msg.mmap = 0; msg.port_msg.nf = 0; msg.port_msg.mf = 0; - msg.port_msg.tracking = tracking != NULL; + + if (port->queue != NULL && type != _NXT_PORT_MSG_READ_QUEUE) { + + if (fd == -1 + && (b == NULL + || nxt_buf_mem_used_size(&b->mem) + <= (int) (NXT_PORT_QUEUE_MSG_SIZE - sizeof(nxt_port_msg_t)))) + { + p = nxt_cpymem(qmsg, &msg.port_msg, sizeof(nxt_port_msg_t)); + if (b != NULL) { + p = nxt_cpymem(p, b->mem.pos, nxt_buf_mem_used_size(&b->mem)); + } + + res = nxt_port_queue_send(port->queue, qmsg, p - qmsg, ¬ify); + + nxt_debug(task, "port{%d,%d} %d: enqueue %d notify %d, %d", + (int) port->pid, (int) port->id, port->socket.fd, + (int) (p - qmsg), notify, res); + + if (notify == 0) { + return res; + } + + msg.port_msg.type = _NXT_PORT_MSG_READ_QUEUE; + msg.buf = NULL; + + } else { + qmsg[0] = _NXT_PORT_MSG_READ_SOCKET; + + res = nxt_port_queue_send(port->queue, qmsg, 1, ¬ify); + + nxt_debug(task, "port{%d,%d} %d: enqueue 1 notify %d, %d", + (int) port->pid, (int) port->id, port->socket.fd, + notify, res); + + if (nxt_slow_path(res == NXT_AGAIN)) { + return NXT_AGAIN; + } + } + } res = nxt_port_msg_chk_insert(task, port, &msg); if (nxt_fast_path(res == NXT_DECLINED)) { @@ -307,10 +349,6 @@ next_fragment: port->max_size / PORT_MMAP_MIN_SIZE); } - if (msg->port_msg.tracking) { - iov[0].iov_len += sizeof(msg->tracking_msg); - } - sb.limit -= iov[0].iov_len; nxt_sendbuf_mem_coalesce(task, &sb); @@ -340,10 +378,18 @@ next_fragment: goto fail; } - if (msg->fd != -1 && msg->close_fd != 0) { - nxt_fd_close(msg->fd); + if (msg->close_fd) { + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + + msg->fd[0] = -1; + } + + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); - msg->fd = -1; + msg->fd[1] = -1; + } } msg->buf = nxt_port_buf_completion(task, wq, msg->buf, plain_size, @@ -357,10 +403,10 @@ next_fragment: * A file descriptor is sent only * in the first message of a stream. */ - msg->fd = -1; + msg->fd[0] = -1; + msg->fd[1] = -1; msg->share += n; msg->port_msg.nf = 1; - msg->port_msg.tracking = 0; if (msg->share >= port->max_share) { msg->share = 0; @@ -568,7 +614,9 @@ nxt_port_read_enable(nxt_task_t *task, nxt_port_t *port) port->engine = task->thread->engine; port->socket.read_work_queue = &port->engine->fast_work_queue; - port->socket.read_handler = nxt_port_read_handler; + port->socket.read_handler = port->queue != NULL + ? nxt_port_queue_read_handler + : nxt_port_read_handler; port->socket.error_handler = nxt_port_error_handler; nxt_fd_event_enable_read(port->engine, &port->socket); @@ -612,7 +660,7 @@ nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) iov[1].iov_base = b->mem.pos; iov[1].iov_len = port->max_size; - n = nxt_socketpair_recv(&port->socket, &msg.fd, iov, 2); + n = nxt_socketpair_recv(&port->socket, msg.fd, iov, 2); if (n > 0) { @@ -652,6 +700,204 @@ nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) } +static void +nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) +{ + ssize_t n; + nxt_buf_t *b; + nxt_port_t *port; + struct iovec iov[2]; + nxt_port_queue_t *queue; + nxt_port_recv_msg_t msg, *smsg; + uint8_t qmsg[NXT_PORT_QUEUE_MSG_SIZE]; + + port = nxt_container_of(obj, nxt_port_t, socket); + msg.port = port; + + nxt_assert(port->engine == task->thread->engine); + + queue = port->queue; + nxt_atomic_fetch_add(&queue->nitems, 1); + + for ( ;; ) { + + if (port->from_socket == 0) { + n = nxt_port_queue_recv(queue, qmsg); + + if (n < 0 && !port->socket.read_ready) { + nxt_atomic_fetch_add(&queue->nitems, -1); + + n = nxt_port_queue_recv(queue, qmsg); + if (n < 0) { + return; + } + + nxt_atomic_fetch_add(&queue->nitems, 1); + } + + if (n == 1 && qmsg[0] == _NXT_PORT_MSG_READ_SOCKET) { + port->from_socket++; + + nxt_debug(task, "port{%d,%d} %d: dequeue 1 read_socket %d", + (int) port->pid, (int) port->id, port->socket.fd, + port->from_socket); + + continue; + } + + nxt_debug(task, "port{%d,%d} %d: dequeue %d", + (int) port->pid, (int) port->id, port->socket.fd, + (int) n); + + } else { + if ((smsg = port->socket_msg) != NULL && smsg->size != 0) { + msg.port_msg = smsg->port_msg; + b = smsg->buf; + n = smsg->size; + msg.fd[0] = smsg->fd[0]; + msg.fd[1] = smsg->fd[1]; + + smsg->size = 0; + + port->from_socket--; + + nxt_debug(task, "port{%d,%d} %d: use suspended message %d", + (int) port->pid, (int) port->id, port->socket.fd, + (int) n); + + goto process; + } + + n = -1; + } + + if (n < 0 && !port->socket.read_ready) { + nxt_atomic_fetch_add(&queue->nitems, -1); + return; + } + + b = nxt_port_buf_alloc(port); + + if (nxt_slow_path(b == NULL)) { + /* TODO: disable event for some time */ + } + + if (n >= (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_memcpy(&msg.port_msg, qmsg, sizeof(nxt_port_msg_t)); + + if (n > (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_memcpy(b->mem.pos, qmsg + sizeof(nxt_port_msg_t), + n - sizeof(nxt_port_msg_t)); + } + + } else { + iov[0].iov_base = &msg.port_msg; + iov[0].iov_len = sizeof(nxt_port_msg_t); + + iov[1].iov_base = b->mem.pos; + iov[1].iov_len = port->max_size; + + n = nxt_socketpair_recv(&port->socket, msg.fd, iov, 2); + + if (n == (ssize_t) sizeof(nxt_port_msg_t) + && msg.port_msg.type == _NXT_PORT_MSG_READ_QUEUE) + { + nxt_port_buf_free(port, b); + + nxt_debug(task, "port{%d,%d} %d: recv %d read_queue", + (int) port->pid, (int) port->id, port->socket.fd, + (int) n); + + continue; + } + + nxt_debug(task, "port{%d,%d} %d: recvmsg %d", + (int) port->pid, (int) port->id, port->socket.fd, + (int) n); + + if (n > 0) { + if (port->from_socket == 0) { + nxt_debug(task, "port{%d,%d} %d: suspend message %d", + (int) port->pid, (int) port->id, port->socket.fd, + (int) n); + + smsg = port->socket_msg; + + if (nxt_slow_path(smsg == NULL)) { + smsg = nxt_mp_alloc(port->mem_pool, + sizeof(nxt_port_recv_msg_t)); + + if (nxt_slow_path(smsg == NULL)) { + nxt_alert(task, "port{%d,%d} %d: suspend message " + "failed", + (int) port->pid, (int) port->id, + port->socket.fd); + + return; + } + + port->socket_msg = smsg; + + } else { + if (nxt_slow_path(smsg->size != 0)) { + nxt_alert(task, "port{%d,%d} %d: too many suspend " + "messages", + (int) port->pid, (int) port->id, + port->socket.fd); + + return; + } + } + + smsg->port_msg = msg.port_msg; + smsg->buf = b; + smsg->size = n; + smsg->fd[0] = msg.fd[0]; + smsg->fd[1] = msg.fd[1]; + + continue; + } + + port->from_socket--; + } + } + + process: + + if (n > 0) { + msg.buf = b; + msg.size = n; + + nxt_port_read_msg_process(task, port, &msg); + + /* + * To disable instant completion or buffer re-usage, + * handler should reset 'msg.buf'. + */ + if (msg.buf == b) { + nxt_port_buf_free(port, b); + } + + continue; + } + + if (n == NXT_AGAIN) { + nxt_port_buf_free(port, b); + + nxt_fd_event_enable_read(task->thread->engine, &port->socket); + + continue; + } + + /* n == 0 || n == NXT_ERROR */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + nxt_port_error_handler, task, &port->socket, NULL); + return; + } +} + + typedef struct { uint32_t stream; uint32_t pid; @@ -806,8 +1052,12 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_alert(task, "port %d: too small message:%uz", port->socket.fd, msg->size); - if (msg->fd != -1) { - nxt_fd_close(msg->fd); + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + } + + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); } return; @@ -819,12 +1069,7 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, b = orig_b = msg->buf; b->mem.free += msg->size; - if (msg->port_msg.tracking) { - msg->cancelled = nxt_port_mmap_tracking_read(task, msg) == 0; - - } else { - msg->cancelled = 0; - } + msg->cancelled = 0; if (nxt_slow_path(msg->port_msg.nf != 0)) { @@ -853,7 +1098,8 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, port->handler(task, fmsg); msg->buf = fmsg->buf; - msg->fd = fmsg->fd; + msg->fd[0] = fmsg->fd[0]; + msg->fd[1] = fmsg->fd[1]; /* * To disable instant completion or buffer re-usage, @@ -887,12 +1133,17 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, if (nxt_fast_path(msg->cancelled == 0)) { msg->buf = NULL; - msg->fd = -1; + msg->fd[0] = -1; + msg->fd[1] = -1; b = NULL; } else { - if (msg->fd != -1) { - nxt_fd_close(msg->fd); + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + } + + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); } } } else { @@ -993,10 +1244,18 @@ nxt_port_error_handler(nxt_task_t *task, void *obj, void *data) nxt_queue_each(msg, &port->messages, nxt_port_send_msg_t, link) { - if (msg->fd != -1 && msg->close_fd != 0) { - nxt_fd_close(msg->fd); + if (msg->close_fd) { + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); - msg->fd = -1; + msg->fd[0] = -1; + } + + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); + + msg->fd[1] = -1; + } } for (b = msg->buf; b != NULL; b = next) { diff --git a/src/nxt_process.c b/src/nxt_process.c index c4c44d14..9bfae395 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -35,17 +35,14 @@ static void nxt_process_created_error(nxt_task_t *task, #if (NXT_HAVE_ISOLATION_ROOTFS) static nxt_int_t nxt_process_chroot(nxt_task_t *task, const char *path); -#endif -#if (NXT_HAVE_PIVOT_ROOT) +#if (NXT_HAVE_PIVOT_ROOT) && (NXT_HAVE_CLONE_NEWNS) static nxt_int_t nxt_process_pivot_root(nxt_task_t *task, const char *rootfs); static nxt_int_t nxt_process_private_mount(nxt_task_t *task, const char *rootfs); -#endif - -#if (NXT_HAVE_PIVOT_ROOT) static int nxt_pivot_root(const char *new_root, const char *old_root); #endif +#endif /* A cached process pid. */ nxt_pid_t nxt_pid; @@ -64,7 +61,7 @@ nxt_bool_t nxt_proc_conn_matrix[NXT_PROCESS_MAX][NXT_PROCESS_MAX] = { { 1, 0, 0, 0, 0 }, { 1, 0, 0, 1, 0 }, { 1, 0, 1, 0, 1 }, - { 1, 0, 0, 0, 0 }, + { 1, 0, 0, 1, 0 }, }; nxt_bool_t nxt_proc_remove_notify_matrix[NXT_PROCESS_MAX][NXT_PROCESS_MAX] = { @@ -149,7 +146,6 @@ nxt_process_child_fixup(nxt_task_t *task, nxt_process_t *process) } nxt_port_mmaps_destroy(&p->incoming, 0); - nxt_port_mmaps_destroy(&p->outgoing, 0); } nxt_runtime_process_loop; @@ -590,11 +586,6 @@ nxt_process_change_root(nxt_task_t *task, nxt_process_t *process) #endif -#endif - - -#if (NXT_HAVE_ISOLATION_ROOTFS) - static nxt_int_t nxt_process_chroot(nxt_task_t *task, const char *path) { @@ -625,8 +616,6 @@ nxt_process_unmount_all(nxt_task_t *task, nxt_process_t *process) } } -#endif - #if (NXT_HAVE_PIVOT_ROOT) && (NXT_HAVE_CLONE_NEWNS) @@ -856,6 +845,8 @@ nxt_pivot_root(const char *new_root, const char *old_root) #endif +#endif + static nxt_int_t nxt_process_send_ready(nxt_task_t *task, nxt_process_t *process) @@ -1116,43 +1107,6 @@ nxt_process_close_ports(nxt_task_t *task, nxt_process_t *process) void -nxt_process_connected_port_add(nxt_process_t *process, nxt_port_t *port) -{ - nxt_thread_mutex_lock(&process->cp_mutex); - - nxt_port_hash_add(&process->connected_ports, port); - - nxt_thread_mutex_unlock(&process->cp_mutex); -} - - -void -nxt_process_connected_port_remove(nxt_process_t *process, nxt_port_t *port) -{ - nxt_thread_mutex_lock(&process->cp_mutex); - - nxt_port_hash_remove(&process->connected_ports, port); - - nxt_thread_mutex_unlock(&process->cp_mutex); -} - - -nxt_port_t * -nxt_process_connected_port_find(nxt_process_t *process, nxt_port_t *port) -{ - nxt_port_t *res; - - nxt_thread_mutex_lock(&process->cp_mutex); - - res = nxt_port_hash_find(&process->connected_ports, port->pid, port->id); - - nxt_thread_mutex_unlock(&process->cp_mutex); - - return res; -} - - -void nxt_process_quit(nxt_task_t *task, nxt_uint_t exit_status) { nxt_uint_t n; diff --git a/src/nxt_process.h b/src/nxt_process.h index d3311722..ecd813e2 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -92,10 +92,8 @@ typedef struct { nxt_int_t use_count; nxt_port_mmaps_t incoming; - nxt_port_mmaps_t outgoing; nxt_thread_mutex_t cp_mutex; - nxt_lvlhsh_t connected_ports; /* of nxt_port_t */ uint32_t stream; @@ -172,14 +170,6 @@ void nxt_process_use(nxt_task_t *task, nxt_process_t *process, int i); void nxt_process_close_ports(nxt_task_t *task, nxt_process_t *process); -void nxt_process_connected_port_add(nxt_process_t *process, nxt_port_t *port); - -void nxt_process_connected_port_remove(nxt_process_t *process, - nxt_port_t *port); - -nxt_port_t *nxt_process_connected_port_find(nxt_process_t *process, - nxt_port_t *port); - void nxt_process_quit(nxt_task_t *task, nxt_uint_t exit_status); void nxt_signal_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/nxt_python_wsgi.c b/src/nxt_python_wsgi.c index b9033a75..c4b7702e 100644 --- a/src/nxt_python_wsgi.c +++ b/src/nxt_python_wsgi.c @@ -18,7 +18,8 @@ #include <nxt_unit_field.h> #include <nxt_unit_request.h> #include <nxt_unit_response.h> -#include <nxt_python_mounts.h> + +#include NXT_PYTHON_MOUNTS_H /* * According to "PEP 3333 / A Note On String Types" @@ -39,11 +40,6 @@ */ -#define _NXT_PYTHON_MOUNTS(major, minor) \ - nxt_python ## major ## minor ## _mounts - -#define NXT_PYTHON_MOUNTS(major, minor) _NXT_PYTHON_MOUNTS(major, minor) - #if PY_MAJOR_VERSION == 3 #define NXT_PYTHON_BYTES_TYPE "bytestring" @@ -123,8 +119,8 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { compat, nxt_string("python"), PY_VERSION, - NXT_PYTHON_MOUNTS(PY_MAJOR_VERSION, PY_MINOR_VERSION), - nxt_nitems(NXT_PYTHON_MOUNTS(PY_MAJOR_VERSION, PY_MINOR_VERSION)), + nxt_python_mounts, + nxt_nitems(nxt_python_mounts), NULL, nxt_python_start, }; @@ -243,11 +239,6 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) app_conf = data->app; c = &app_conf->u.python; - if (c->module.length == 0) { - nxt_alert(task, "python module is empty"); - return NXT_ERROR; - } - if (c->home != NULL) { len = nxt_strlen(c->home); diff --git a/src/nxt_router.c b/src/nxt_router.c index 788199c7..0e1de6fa 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -15,6 +15,8 @@ #include <nxt_unit_request.h> #include <nxt_unit_response.h> #include <nxt_router_request.h> +#include <nxt_app_queue.h> +#include <nxt_port_queue.h> typedef struct { nxt_str_t type; @@ -61,59 +63,22 @@ typedef struct { } nxt_app_rpc_t; -struct nxt_port_select_state_s { - nxt_app_t *app; - nxt_request_app_link_t *req_app_link; - - nxt_port_t *failed_port; - int failed_port_use_delta; - - uint8_t start_process; /* 1 bit */ - nxt_request_app_link_t *shared_ra; - nxt_port_t *port; -}; - -typedef struct nxt_port_select_state_s nxt_port_select_state_t; - static nxt_int_t nxt_router_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp); static nxt_int_t nxt_router_start(nxt_task_t *task, nxt_process_data_t *data); static void nxt_router_greet_controller(nxt_task_t *task, nxt_port_t *controller_port); -static void nxt_router_port_select(nxt_task_t *task, - nxt_port_select_state_t *state); - -static nxt_int_t nxt_router_port_post_select(nxt_task_t *task, - nxt_port_select_state_t *state); - static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app); -static void nxt_request_app_link_update_peer(nxt_task_t *task, - nxt_request_app_link_t *req_app_link); - -nxt_inline void -nxt_request_app_link_inc_use(nxt_request_app_link_t *req_app_link) -{ - nxt_atomic_fetch_add(&req_app_link->use_count, 1); -} - -nxt_inline void -nxt_request_app_link_chk_use(nxt_request_app_link_t *req_app_link, int i) -{ -#if (NXT_DEBUG) - int c; - - c = nxt_atomic_fetch_add(&req_app_link->use_count, i); - - nxt_assert((c + i) > 0); -#else - (void) nxt_atomic_fetch_add(&req_app_link->use_count, i); -#endif -} - -static void nxt_request_app_link_use(nxt_task_t *task, - nxt_request_app_link_t *req_app_link, int i); +static void nxt_router_new_port_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_conf_data_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_remove_pid_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_access_log_reopen_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); static nxt_router_temp_conf_t *nxt_router_temp_conf(nxt_task_t *task); static void nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data); @@ -128,7 +93,22 @@ static nxt_int_t nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, u_char *start, u_char *end); static nxt_int_t nxt_router_conf_process_static(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *conf); + static nxt_app_t *nxt_router_app_find(nxt_queue_t *queue, nxt_str_t *name); +static nxt_int_t nxt_router_apps_hash_test(nxt_lvlhsh_query_t *lhq, void *data); +static nxt_int_t nxt_router_apps_hash_add(nxt_router_conf_t *rtcf, + nxt_app_t *app); +static nxt_app_t *nxt_router_apps_hash_get(nxt_router_conf_t *rtcf, + nxt_str_t *name); +static void nxt_router_apps_hash_use(nxt_task_t *task, nxt_router_conf_t *rtcf, + int i); + +static nxt_int_t nxt_router_app_queue_init(nxt_task_t *task, + nxt_port_t *port); +static nxt_int_t nxt_router_port_queue_init(nxt_task_t *task, + nxt_port_t *port); +static nxt_int_t nxt_router_port_queue_map(nxt_task_t *task, + nxt_port_t *port, nxt_fd_t fd); static void nxt_router_listen_socket_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_socket_conf_t *skcf); static void nxt_router_listen_socket_ready(nxt_task_t *task, @@ -182,6 +162,8 @@ static void nxt_router_engine_post(nxt_event_engine_t *engine, nxt_work_t *jobs); static void nxt_router_thread_start(void *data); +static void nxt_router_rt_add_port(nxt_task_t *task, void *obj, + void *data); static void nxt_router_listen_socket_create(nxt_task_t *task, void *obj, void *data); static void nxt_router_listen_socket_update(nxt_task_t *task, void *obj, @@ -194,6 +176,8 @@ static void nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data); static void nxt_router_thread_exit_handler(nxt_task_t *task, void *obj, void *data); +static void nxt_router_req_headers_ack_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_request_rpc_data_t *req_rpc_data); static void nxt_router_listen_socket_release(nxt_task_t *task, nxt_socket_conf_t *skcf); @@ -218,20 +202,29 @@ static void nxt_router_access_log_reopen_error(nxt_task_t *task, static void nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); +static nxt_int_t nxt_router_app_shared_port_send(nxt_task_t *task, + nxt_port_t *app_port); static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); +static void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i); static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app); static void nxt_router_app_port_release(nxt_task_t *task, nxt_port_t *port, nxt_apr_action_t action); -static nxt_int_t nxt_router_app_port(nxt_task_t *task, nxt_app_t *app, - nxt_request_app_link_t *req_app_link); +static void nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, + nxt_request_rpc_data_t *req_rpc_data); +static void nxt_router_http_request_error(nxt_task_t *task, void *obj, + void *data); +static void nxt_router_http_request_done(nxt_task_t *task, void *obj, + void *data); +static void nxt_router_dummy_buf_completion(nxt_task_t *task, void *obj, + void *data); static void nxt_router_app_prepare_request(nxt_task_t *task, - nxt_request_app_link_t *req_app_link); + nxt_request_rpc_data_t *req_rpc_data); static nxt_buf_t *nxt_router_prepare_msg(nxt_task_t *task, - nxt_http_request_t *r, nxt_port_t *port, const nxt_str_t *prefix); + nxt_http_request_t *r, nxt_app_t *app, const nxt_str_t *prefix); static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data); static void nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, @@ -248,11 +241,15 @@ static void nxt_http_request_send_body(nxt_task_t *task, void *obj, void *data); static void nxt_router_app_joint_use(nxt_task_t *task, nxt_app_joint_t *app_joint, int i); -static nxt_int_t nxt_router_http_request_done(nxt_task_t *task, +static void nxt_router_http_request_release_post(nxt_task_t *task, nxt_http_request_t *r); static void nxt_router_http_request_release(nxt_task_t *task, void *obj, void *data); static void nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +static void nxt_router_get_port_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_get_mmap_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); extern const nxt_http_request_state_t nxt_http_websocket; @@ -274,8 +271,10 @@ static const nxt_str_t *nxt_app_msg_prefix[] = { static const nxt_port_handlers_t nxt_router_process_port_handlers = { .quit = nxt_signal_quit_handler, .new_port = nxt_router_new_port_handler, + .get_port = nxt_router_get_port_handler, .change_file = nxt_port_change_log_file_handler, .mmap = nxt_port_mmap_handler, + .get_mmap = nxt_router_get_mmap_handler, .data = nxt_router_conf_data_handler, .remove_pid = nxt_router_remove_pid_handler, .access_log = nxt_router_access_log_reopen_handler, @@ -297,6 +296,14 @@ const nxt_process_init_t nxt_router_process = { }; +/* Queues of nxt_socket_conf_t */ +nxt_queue_t creating_sockets; +nxt_queue_t pending_sockets; +nxt_queue_t updating_sockets; +nxt_queue_t keeping_sockets; +nxt_queue_t deleting_sockets; + + static nxt_int_t nxt_router_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) { @@ -461,6 +468,8 @@ nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app) nxt_port_t *router_port; nxt_runtime_t *rt; + nxt_debug(task, "app '%V' start process", &app->name); + rt = task->thread->runtime; router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; @@ -485,99 +494,26 @@ nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app) } -nxt_inline void -nxt_request_app_link_init(nxt_task_t *task, - nxt_request_app_link_t *req_app_link, nxt_request_rpc_data_t *req_rpc_data) -{ - nxt_buf_t *body; - nxt_event_engine_t *engine; - - engine = task->thread->engine; - - nxt_memzero(req_app_link, sizeof(nxt_request_app_link_t)); - - req_app_link->stream = req_rpc_data->stream; - req_app_link->use_count = 1; - req_app_link->req_rpc_data = req_rpc_data; - req_rpc_data->req_app_link = req_app_link; - req_app_link->reply_port = engine->port; - req_app_link->request = req_rpc_data->request; - req_app_link->apr_action = NXT_APR_GOT_RESPONSE; - - req_app_link->work.handler = NULL; - req_app_link->work.task = &engine->task; - req_app_link->work.obj = req_app_link; - req_app_link->work.data = engine; - - body = req_rpc_data->request->body; - - if (body != NULL && nxt_buf_is_file(body)) { - req_app_link->body_fd = body->file->fd; - - body->file->fd = -1; - - } else { - req_app_link->body_fd = -1; - } -} - - -nxt_inline nxt_request_app_link_t * -nxt_request_app_link_alloc(nxt_task_t *task, - nxt_request_app_link_t *ra_src, nxt_request_rpc_data_t *req_rpc_data) -{ - nxt_mp_t *mp; - nxt_request_app_link_t *req_app_link; - - if (ra_src != NULL && ra_src->mem_pool != NULL) { - return ra_src; - } - - mp = req_rpc_data->request->mem_pool; - - req_app_link = nxt_mp_alloc(mp, sizeof(nxt_request_app_link_t)); - - if (nxt_slow_path(req_app_link == NULL)) { - - req_rpc_data->req_app_link = NULL; - - if (ra_src != NULL) { - ra_src->req_rpc_data = NULL; - } - - return NULL; - } - - nxt_mp_retain(mp); - - nxt_request_app_link_init(task, req_app_link, req_rpc_data); - - if (ra_src != NULL) { - req_app_link->body_fd = ra_src->body_fd; - } - - req_app_link->mem_pool = mp; - - return req_app_link; -} - - nxt_inline nxt_bool_t -nxt_router_msg_cancel(nxt_task_t *task, nxt_msg_info_t *msg_info, - uint32_t stream) +nxt_router_msg_cancel(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) { - nxt_buf_t *b, *next; - nxt_bool_t cancelled; + nxt_buf_t *b, *next; + nxt_bool_t cancelled; + nxt_msg_info_t *msg_info; + + msg_info = &req_rpc_data->msg_info; if (msg_info->buf == NULL) { return 0; } - cancelled = nxt_port_mmap_tracking_cancel(task, &msg_info->tracking, - stream); + cancelled = nxt_app_queue_cancel(req_rpc_data->app->shared_port->queue, + msg_info->tracking_cookie, + req_rpc_data->stream); if (cancelled) { - nxt_debug(task, "stream #%uD: cancelled by router", stream); + nxt_debug(task, "stream #%uD: cancelled by router", + req_rpc_data->stream); } for (b = msg_info->buf; b != NULL; b = next) { @@ -598,320 +534,204 @@ nxt_router_msg_cancel(nxt_task_t *task, nxt_msg_info_t *msg_info, } -static void -nxt_request_app_link_update_peer_handler(nxt_task_t *task, void *obj, - void *data) +nxt_inline nxt_bool_t +nxt_queue_chk_remove(nxt_queue_link_t *lnk) { - nxt_request_app_link_t *req_app_link; + if (lnk->next != NULL) { + nxt_queue_remove(lnk); - req_app_link = obj; + lnk->next = NULL; - nxt_request_app_link_update_peer(task, req_app_link); + return 1; + } - nxt_request_app_link_use(task, req_app_link, -1); + return 0; } -static void -nxt_request_app_link_update_peer(nxt_task_t *task, - nxt_request_app_link_t *req_app_link) +nxt_inline void +nxt_request_rpc_data_unlink(nxt_task_t *task, + nxt_request_rpc_data_t *req_rpc_data) { - nxt_event_engine_t *engine; - nxt_request_rpc_data_t *req_rpc_data; - - engine = req_app_link->work.data; - - if (task->thread->engine != engine) { - nxt_request_app_link_inc_use(req_app_link); - - req_app_link->work.handler = nxt_request_app_link_update_peer_handler; - req_app_link->work.task = &engine->task; - req_app_link->work.next = NULL; + nxt_app_t *app; + nxt_bool_t unlinked; + nxt_http_request_t *r; - nxt_debug(task, "req_app_link stream #%uD post update peer to %p", - req_app_link->stream, engine); + nxt_router_msg_cancel(task, req_rpc_data); - nxt_event_engine_post(engine, &req_app_link->work); + if (req_rpc_data->app_port != NULL) { + nxt_router_app_port_release(task, req_rpc_data->app_port, + req_rpc_data->apr_action); - return; + req_rpc_data->app_port = NULL; } - nxt_debug(task, "req_app_link stream #%uD update peer", - req_app_link->stream); - - req_rpc_data = req_app_link->req_rpc_data; - - if (req_rpc_data != NULL && req_app_link->app_port != NULL) { - nxt_port_rpc_ex_set_peer(task, engine->port, req_rpc_data, - req_app_link->app_port->pid); - } -} + app = req_rpc_data->app; + r = req_rpc_data->request; + if (r != NULL) { + r->timer_data = NULL; -static void -nxt_request_app_link_release(nxt_task_t *task, - nxt_request_app_link_t *req_app_link) -{ - nxt_mp_t *mp; - nxt_http_request_t *r; - nxt_request_rpc_data_t *req_rpc_data; + nxt_router_http_request_release_post(task, r); - nxt_assert(task->thread->engine == req_app_link->work.data); - nxt_assert(req_app_link->use_count == 0); + r->req_rpc_data = NULL; + req_rpc_data->request = NULL; - nxt_debug(task, "req_app_link stream #%uD release", req_app_link->stream); + if (app != NULL) { + unlinked = 0; - req_rpc_data = req_app_link->req_rpc_data; + nxt_thread_mutex_lock(&app->mutex); - if (req_rpc_data != NULL) { - if (nxt_slow_path(req_app_link->err_code != 0)) { - nxt_http_request_error(task, req_rpc_data->request, - req_app_link->err_code); + if (r->app_link.next != NULL) { + nxt_queue_remove(&r->app_link); + r->app_link.next = NULL; - } else { - req_rpc_data->app_port = req_app_link->app_port; - req_rpc_data->apr_action = req_app_link->apr_action; - req_rpc_data->msg_info = req_app_link->msg_info; + unlinked = 1; + } - if (req_rpc_data->app->timeout != 0) { - r = req_rpc_data->request; + nxt_thread_mutex_unlock(&app->mutex); - r->timer.handler = nxt_router_app_timeout; - r->timer_data = req_rpc_data; - nxt_timer_add(task->thread->engine, &r->timer, - req_rpc_data->app->timeout); + if (unlinked) { + nxt_mp_release(r->mem_pool); } - - req_app_link->app_port = NULL; - req_app_link->msg_info.buf = NULL; } - - req_rpc_data->req_app_link = NULL; - req_app_link->req_rpc_data = NULL; } - if (req_app_link->app_port != NULL) { - nxt_router_app_port_release(task, req_app_link->app_port, - req_app_link->apr_action); + if (app != NULL) { + nxt_router_app_use(task, app, -1); - req_app_link->app_port = NULL; + req_rpc_data->app = NULL; } - if (req_app_link->body_fd != -1) { - nxt_fd_close(req_app_link->body_fd); + if (req_rpc_data->msg_info.body_fd != -1) { + nxt_fd_close(req_rpc_data->msg_info.body_fd); - req_app_link->body_fd = -1; + req_rpc_data->msg_info.body_fd = -1; } - nxt_router_msg_cancel(task, &req_app_link->msg_info, req_app_link->stream); + if (req_rpc_data->rpc_cancel) { + req_rpc_data->rpc_cancel = 0; - mp = req_app_link->mem_pool; - - if (mp != NULL) { - nxt_mp_free(mp, req_app_link); - nxt_mp_release(mp); + nxt_port_rpc_cancel(task, task->thread->engine->port, + req_rpc_data->stream); } } static void -nxt_request_app_link_release_handler(nxt_task_t *task, void *obj, void *data) +nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - nxt_request_app_link_t *req_app_link; - - req_app_link = obj; - - nxt_assert(req_app_link->work.data == data); - - nxt_atomic_fetch_add(&req_app_link->use_count, -1); - - nxt_request_app_link_release(task, req_app_link); -} - + nxt_int_t res; + nxt_app_t *app; + nxt_port_t *port, *main_app_port; + nxt_runtime_t *rt; -static void -nxt_request_app_link_use(nxt_task_t *task, nxt_request_app_link_t *req_app_link, - int i) -{ - int c; - nxt_event_engine_t *engine; + nxt_port_new_port_handler(task, msg); - c = nxt_atomic_fetch_add(&req_app_link->use_count, i); + port = msg->u.new_port; - if (i < 0 && c == -i) { - engine = req_app_link->work.data; + if (port != NULL && port->type == NXT_PROCESS_CONTROLLER) { + nxt_router_greet_controller(task, msg->u.new_port); + } - if (task->thread->engine == engine) { - nxt_request_app_link_release(task, req_app_link); + if (port == NULL || port->type != NXT_PROCESS_APP) { + if (msg->port_msg.stream == 0) { return; } - nxt_request_app_link_inc_use(req_app_link); - - req_app_link->work.handler = nxt_request_app_link_release_handler; - req_app_link->work.task = &engine->task; - req_app_link->work.next = NULL; + msg->port_msg.type = _NXT_PORT_MSG_RPC_ERROR; - nxt_debug(task, "req_app_link stream #%uD post release to %p", - req_app_link->stream, engine); + } else { + if (msg->fd[1] != -1) { + res = nxt_router_port_queue_map(task, port, msg->fd[1]); + if (nxt_slow_path(res != NXT_OK)) { + return; + } - nxt_event_engine_post(engine, &req_app_link->work); + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + } } -} - - -nxt_inline void -nxt_request_app_link_error(nxt_task_t *task, nxt_app_t *app, - nxt_request_app_link_t *req_app_link, const char *str) -{ - req_app_link->app_port = NULL; - req_app_link->err_code = 500; - req_app_link->err_str = str; - - nxt_alert(task, "app \"%V\" internal error: %s on #%uD", - &app->name, str, req_app_link->stream); -} - -nxt_inline void -nxt_request_app_link_pending(nxt_task_t *task, nxt_app_t *app, - nxt_request_app_link_t *req_app_link) -{ - nxt_queue_insert_tail(&req_app_link->app_port->pending_requests, - &req_app_link->link_port_pending); - nxt_queue_insert_tail(&app->pending, &req_app_link->link_app_pending); + if (msg->port_msg.stream != 0) { + nxt_port_rpc_handler(task, msg); + return; + } - nxt_request_app_link_inc_use(req_app_link); + /* + * Port with "id == 0" is application 'main' port and it always + * should come with non-zero stream. + */ + nxt_assert(port->id != 0); - req_app_link->res_time = nxt_thread_monotonic_time(task->thread) - + app->res_timeout; + /* Find 'main' app port and get app reference. */ + rt = task->thread->runtime; - nxt_debug(task, "req_app_link stream #%uD enqueue to pending_requests", - req_app_link->stream); -} + /* + * It is safe to access 'runtime->ports' hash because 'NEW_PORT' + * sent to main port (with id == 0) and processed in main thread. + */ + main_app_port = nxt_port_hash_find(&rt->ports, port->pid, 0); + nxt_assert(main_app_port != NULL); + app = main_app_port->app; + nxt_assert(app != NULL); -nxt_inline nxt_bool_t -nxt_queue_chk_remove(nxt_queue_link_t *lnk) -{ - if (lnk->next != NULL) { - nxt_queue_remove(lnk); + nxt_thread_mutex_lock(&app->mutex); - lnk->next = NULL; + /* TODO here should be find-and-add code because there can be + port waiters in port_hash */ + nxt_port_hash_add(&app->port_hash, port); + app->port_hash_count++; - return 1; - } + nxt_thread_mutex_unlock(&app->mutex); - return 0; + port->app = app; + port->main_app_port = main_app_port; } -nxt_inline void -nxt_request_rpc_data_unlink(nxt_task_t *task, - nxt_request_rpc_data_t *req_rpc_data) +static void +nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - int ra_use_delta; - nxt_request_app_link_t *req_app_link; - - if (req_rpc_data->app_port != NULL) { - nxt_router_app_port_release(task, req_rpc_data->app_port, - req_rpc_data->apr_action); - - req_rpc_data->app_port = NULL; - } - - nxt_router_msg_cancel(task, &req_rpc_data->msg_info, req_rpc_data->stream); - - req_app_link = req_rpc_data->req_app_link; - if (req_app_link != NULL) { - req_rpc_data->req_app_link = NULL; - req_app_link->req_rpc_data = NULL; - - ra_use_delta = 0; - - nxt_thread_mutex_lock(&req_rpc_data->app->mutex); - - if (req_app_link->link_app_requests.next == NULL - && req_app_link->link_port_pending.next == NULL - && req_app_link->link_app_pending.next == NULL - && req_app_link->link_port_websockets.next == NULL) - { - req_app_link = NULL; - - } else { - ra_use_delta -= - nxt_queue_chk_remove(&req_app_link->link_app_requests) - + nxt_queue_chk_remove(&req_app_link->link_port_pending) - + nxt_queue_chk_remove(&req_app_link->link_port_websockets); - - nxt_queue_chk_remove(&req_app_link->link_app_pending); - } - - nxt_thread_mutex_unlock(&req_rpc_data->app->mutex); - - if (req_app_link != NULL) { - nxt_request_app_link_use(task, req_app_link, ra_use_delta); - } - } - - if (req_rpc_data->app != NULL) { - nxt_router_app_use(task, req_rpc_data->app, -1); + void *p; + size_t size; + nxt_int_t ret; + nxt_router_temp_conf_t *tmcf; - req_rpc_data->app = NULL; + tmcf = nxt_router_temp_conf(task); + if (nxt_slow_path(tmcf == NULL)) { + return; } - if (req_rpc_data->request != NULL) { - req_rpc_data->request->timer_data = NULL; - - nxt_router_http_request_done(task, req_rpc_data->request); - - req_rpc_data->request->req_rpc_data = NULL; - req_rpc_data->request = NULL; + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "conf_data_handler: invalid file shm fd"); + return; } -} + if (nxt_buf_mem_used_size(&msg->buf->mem) != sizeof(size_t)) { + nxt_alert(task, "conf_data_handler: unexpected buffer size (%d)", + (int) nxt_buf_mem_used_size(&msg->buf->mem)); -void -nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) -{ - nxt_port_new_port_handler(task, msg); - - if (msg->u.new_port != NULL - && msg->u.new_port->type == NXT_PROCESS_CONTROLLER) - { - nxt_router_greet_controller(task, msg->u.new_port); - } + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; - if (msg->port_msg.stream == 0) { return; } - if (msg->u.new_port == NULL - || msg->u.new_port->type != NXT_PROCESS_APP) - { - msg->port_msg.type = _NXT_PORT_MSG_RPC_ERROR; - } - - nxt_port_rpc_handler(task, msg); -} + nxt_memcpy(&size, msg->buf->mem.pos, sizeof(size_t)); + p = nxt_mem_mmap(NULL, size, PROT_READ, MAP_SHARED, msg->fd[0], 0); -void -nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) -{ - nxt_int_t ret; - nxt_buf_t *b; - nxt_router_temp_conf_t *tmcf; + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; - tmcf = nxt_router_temp_conf(task); - if (nxt_slow_path(tmcf == NULL)) { + if (nxt_slow_path(p == MAP_FAILED)) { return; } - nxt_debug(task, "nxt_router_conf_data_handler(%O): %*s", - nxt_buf_used_size(msg->buf), - (size_t) nxt_buf_used_size(msg->buf), msg->buf->mem.pos); + nxt_debug(task, "conf_data_handler(%uz): %*s", size, size, p); tmcf->router_conf->router = nxt_router; tmcf->stream = msg->port_msg.stream; @@ -922,20 +742,12 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_slow_path(tmcf->port == NULL)) { nxt_alert(task, "reply port not found"); - return; + goto fail; } nxt_port_use(task, tmcf->port, 1); - b = nxt_buf_chk_make_plain(tmcf->router_conf->mem_pool, - msg->buf, msg->size); - if (nxt_slow_path(b == NULL)) { - nxt_router_conf_error(task, tmcf); - - return; - } - - ret = nxt_router_conf_create(task, tmcf, b->mem.pos, b->mem.free); + ret = nxt_router_conf_create(task, tmcf, p, nxt_pointer_to(p, size)); if (nxt_fast_path(ret == NXT_OK)) { nxt_router_conf_apply(task, tmcf, NULL); @@ -943,6 +755,10 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } else { nxt_router_conf_error(task, tmcf); } + +fail: + + nxt_mem_munmap(p, size); } @@ -961,7 +777,7 @@ nxt_router_app_process_remove_pid(nxt_task_t *task, nxt_port_t *port, } -void +static void nxt_router_remove_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_event_engine_t *engine; @@ -1027,11 +843,11 @@ nxt_router_temp_conf(nxt_task_t *task) goto temp_fail; } - nxt_queue_init(&tmcf->deleting); - nxt_queue_init(&tmcf->keeping); - nxt_queue_init(&tmcf->updating); - nxt_queue_init(&tmcf->pending); - nxt_queue_init(&tmcf->creating); + nxt_queue_init(&creating_sockets); + nxt_queue_init(&pending_sockets); + nxt_queue_init(&updating_sockets); + nxt_queue_init(&keeping_sockets); + nxt_queue_init(&deleting_sockets); #if (NXT_TLS) nxt_queue_init(&tmcf->tls); @@ -1065,8 +881,10 @@ nxt_router_app_can_start(nxt_app_t *app) nxt_inline nxt_bool_t nxt_router_app_need_start(nxt_app_t *app) { - return app->idle_processes + app->pending_processes - < app->spare_processes; + return (app->active_requests + > app->port_hash_count + app->pending_processes) + || (app->spare_processes + > app->idle_processes + app->pending_processes); } @@ -1088,11 +906,11 @@ nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data) tmcf = obj; - qlk = nxt_queue_first(&tmcf->pending); + qlk = nxt_queue_first(&pending_sockets); - if (qlk != nxt_queue_tail(&tmcf->pending)) { + if (qlk != nxt_queue_tail(&pending_sockets)) { nxt_queue_remove(qlk); - nxt_queue_insert_tail(&tmcf->creating, qlk); + nxt_queue_insert_tail(&creating_sockets, qlk); skcf = nxt_queue_link_data(qlk, nxt_socket_conf_t, link); @@ -1148,10 +966,12 @@ nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data) nxt_router_apps_sort(task, router, tmcf); + nxt_router_apps_hash_use(task, rtcf, 1); + nxt_router_engines_post(router, tmcf); - nxt_queue_add(&router->sockets, &tmcf->updating); - nxt_queue_add(&router->sockets, &tmcf->creating); + nxt_queue_add(&router->sockets, &updating_sockets); + nxt_queue_add(&router->sockets, &creating_sockets); router->access_log = rtcf->access_log; @@ -1181,11 +1001,39 @@ nxt_router_conf_wait(nxt_task_t *task, void *obj, void *data) static void nxt_router_conf_ready(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) { - nxt_debug(task, "temp conf count:%D", tmcf->count); + uint32_t count; + nxt_router_conf_t *rtcf; + nxt_thread_spinlock_t *lock; + + nxt_debug(task, "temp conf %p count: %D", tmcf, tmcf->count); - if (--tmcf->count == 0) { - nxt_router_conf_send(task, tmcf, NXT_PORT_MSG_RPC_READY_LAST); + if (--tmcf->count > 0) { + return; } + + nxt_router_conf_send(task, tmcf, NXT_PORT_MSG_RPC_READY_LAST); + + rtcf = tmcf->router_conf; + + lock = &rtcf->router->lock; + + nxt_thread_spin_lock(lock); + + count = rtcf->count; + + nxt_thread_spin_unlock(lock); + + nxt_debug(task, "rtcf %p: %D", rtcf, count); + + if (count == 0) { + nxt_router_apps_hash_use(task, rtcf, -1); + + nxt_router_access_log_release(task, lock, rtcf->access_log); + + nxt_mp_destroy(rtcf->mem_pool); + } + + nxt_mp_destroy(tmcf->mem_pool); } @@ -1202,8 +1050,8 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) nxt_alert(task, "failed to apply new conf"); - for (qlk = nxt_queue_first(&tmcf->creating); - qlk != nxt_queue_tail(&tmcf->creating); + for (qlk = nxt_queue_first(&creating_sockets); + qlk != nxt_queue_tail(&creating_sockets); qlk = nxt_queue_next(qlk)) { skcf = nxt_queue_link_data(qlk, nxt_socket_conf_t, link); @@ -1217,22 +1065,12 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) } nxt_queue_init(&new_socket_confs); - nxt_queue_add(&new_socket_confs, &tmcf->updating); - nxt_queue_add(&new_socket_confs, &tmcf->pending); - nxt_queue_add(&new_socket_confs, &tmcf->creating); + nxt_queue_add(&new_socket_confs, &updating_sockets); + nxt_queue_add(&new_socket_confs, &pending_sockets); + nxt_queue_add(&new_socket_confs, &creating_sockets); rtcf = tmcf->router_conf; - nxt_http_routes_cleanup(task, rtcf->routes); - - nxt_queue_each(skcf, &new_socket_confs, nxt_socket_conf_t, link) { - - if (skcf->action != NULL) { - nxt_http_action_cleanup(task, skcf->action); - } - - } nxt_queue_loop; - nxt_queue_each(app, &tmcf->apps, nxt_app_t, link) { nxt_router_app_unlink(task, app); @@ -1241,8 +1079,8 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) router = rtcf->router; - nxt_queue_add(&router->sockets, &tmcf->keeping); - nxt_queue_add(&router->sockets, &tmcf->deleting); + nxt_queue_add(&router->sockets, &keeping_sockets); + nxt_queue_add(&router->sockets, &deleting_sockets); nxt_queue_add(&router->apps, &tmcf->previous); @@ -1253,6 +1091,8 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) nxt_mp_destroy(rtcf->mem_pool); nxt_router_conf_send(task, tmcf, NXT_PORT_MSG_RPC_ERROR); + + nxt_mp_destroy(tmcf->mem_pool); } @@ -1465,6 +1305,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_app_t *app, *prev; nxt_str_t *t, *s, *targets; nxt_uint_t n, i; + nxt_port_t *port; nxt_router_t *router; nxt_app_joint_t *app_joint; nxt_conf_value_t *conf, *http, *value, *websocket; @@ -1569,6 +1410,12 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_queue_remove(&prev->link); nxt_queue_insert_tail(&tmcf->previous, &prev->link); + + ret = nxt_router_apps_hash_add(tmcf->router_conf, prev); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + continue; } @@ -1679,8 +1526,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_queue_init(&app->ports); nxt_queue_init(&app->spare_ports); nxt_queue_init(&app->idle_ports); - nxt_queue_init(&app->requests); - nxt_queue_init(&app->pending); + nxt_queue_init(&app->ack_waiting_req); app->name.length = name.length; nxt_memcpy(app->name.start, name.start, name.length); @@ -1693,7 +1539,6 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, app->timeout = apcf.timeout; app->res_timeout = apcf.res_timeout * 1000000; app->idle_timeout = apcf.idle_timeout; - app->max_pending_responses = 2; app->max_requests = apcf.requests; app->targets = targets; @@ -1708,6 +1553,11 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_queue_insert_tail(&tmcf->apps, &app->link); + ret = nxt_router_apps_hash_add(tmcf->router_conf, app); + if (nxt_slow_path(ret != NXT_OK)) { + goto app_fail; + } + nxt_router_app_use(task, app, 1); app->joint = app_joint; @@ -1724,6 +1574,31 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, app_joint->free_app_work.handler = nxt_router_free_app; app_joint->free_app_work.task = &engine->task; app_joint->free_app_work.obj = app_joint; + + port = nxt_port_new(task, (nxt_port_id_t) -1, nxt_pid, + NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + return NXT_ERROR; + } + + ret = nxt_port_socket_init(task, port, 0); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_use(task, port, -1); + return NXT_ERROR; + } + + ret = nxt_router_app_queue_init(task, port); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_use(task, port, -1); + return NXT_ERROR; + } + + nxt_port_write_enable(task, port); + port->app = app; + + app->shared_port = port; + + nxt_thread_mutex_create(&app->outgoing.mutex); } } @@ -1857,7 +1732,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, /* COMPATIBILITY: listener application. */ } else if (lscf.application.length > 0) { - skcf->action = nxt_http_pass_application(task, tmcf, + skcf->action = nxt_http_pass_application(task, + tmcf->router_conf, &lscf.application); } } @@ -1902,7 +1778,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, tmcf->router_conf->access_log = access_log; } - nxt_queue_add(&tmcf->deleting, &router->sockets); + nxt_queue_add(&deleting_sockets, &router->sockets); nxt_queue_init(&router->sockets); return NXT_OK; @@ -2023,20 +1899,182 @@ nxt_router_app_find(nxt_queue_t *queue, nxt_str_t *name) } -void -nxt_router_listener_application(nxt_router_temp_conf_t *tmcf, nxt_str_t *name, +static nxt_int_t +nxt_router_app_queue_init(nxt_task_t *task, nxt_port_t *port) +{ + void *mem; + nxt_int_t fd; + + fd = nxt_shm_open(task, sizeof(nxt_app_queue_t)); + if (nxt_slow_path(fd == -1)) { + return NXT_ERROR; + } + + mem = nxt_mem_mmap(NULL, sizeof(nxt_app_queue_t), + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_fd_close(fd); + + return NXT_ERROR; + } + + nxt_app_queue_init(mem); + + port->queue_fd = fd; + port->queue = mem; + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_port_queue_init(nxt_task_t *task, nxt_port_t *port) +{ + void *mem; + nxt_int_t fd; + + fd = nxt_shm_open(task, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(fd == -1)) { + return NXT_ERROR; + } + + mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_fd_close(fd); + + return NXT_ERROR; + } + + nxt_port_queue_init(mem); + + port->queue_fd = fd; + port->queue = mem; + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_port_queue_map(nxt_task_t *task, nxt_port_t *port, nxt_fd_t fd) +{ + void *mem; + + nxt_assert(fd != -1); + + mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + + return NXT_ERROR; + } + + port->queue = mem; + + return NXT_OK; +} + + +static const nxt_lvlhsh_proto_t nxt_router_apps_hash_proto nxt_aligned(64) = { + NXT_LVLHSH_DEFAULT, + nxt_router_apps_hash_test, + nxt_mp_lvlhsh_alloc, + nxt_mp_lvlhsh_free, +}; + + +static nxt_int_t +nxt_router_apps_hash_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + nxt_app_t *app; + + app = data; + + return nxt_strstr_eq(&lhq->key, &app->name) ? NXT_OK : NXT_DECLINED; +} + + +static nxt_int_t +nxt_router_apps_hash_add(nxt_router_conf_t *rtcf, nxt_app_t *app) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(app->name.start, app->name.length); + lhq.replace = 0; + lhq.key = app->name; + lhq.value = app; + lhq.proto = &nxt_router_apps_hash_proto; + lhq.pool = rtcf->mem_pool; + + switch (nxt_lvlhsh_insert(&rtcf->apps_hash, &lhq)) { + + case NXT_OK: + return NXT_OK; + + case NXT_DECLINED: + nxt_thread_log_alert("router app hash adding failed: " + "\"%V\" is already in hash", &lhq.key); + /* Fall through. */ + default: + return NXT_ERROR; + } +} + + +static nxt_app_t * +nxt_router_apps_hash_get(nxt_router_conf_t *rtcf, nxt_str_t *name) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(name->start, name->length); + lhq.key = *name; + lhq.proto = &nxt_router_apps_hash_proto; + + if (nxt_lvlhsh_find(&rtcf->apps_hash, &lhq) != NXT_OK) { + return NULL; + } + + return lhq.value; +} + + +static void +nxt_router_apps_hash_use(nxt_task_t *task, nxt_router_conf_t *rtcf, int i) +{ + nxt_app_t *app; + nxt_lvlhsh_each_t lhe; + + nxt_lvlhsh_each_init(&lhe, &nxt_router_apps_hash_proto); + + for ( ;; ) { + app = nxt_lvlhsh_each(&rtcf->apps_hash, &lhe); + + if (app == NULL) { + break; + } + + nxt_router_app_use(task, app, i); + } +} + + + +nxt_int_t +nxt_router_listener_application(nxt_router_conf_t *rtcf, nxt_str_t *name, nxt_http_action_t *action) { nxt_app_t *app; - app = nxt_router_app_find(&tmcf->apps, name); + app = nxt_router_apps_hash_get(rtcf, name); if (app == NULL) { - app = nxt_router_app_find(&tmcf->previous, name); + return NXT_DECLINED; } action->u.application = app; action->handler = nxt_http_application_handler; + + return NXT_OK; } @@ -2141,15 +2179,15 @@ nxt_router_listen_socket_find(nxt_router_temp_conf_t *tmcf, nskcf->listen = skcf->listen; nxt_queue_remove(qlk); - nxt_queue_insert_tail(&tmcf->keeping, qlk); + nxt_queue_insert_tail(&keeping_sockets, qlk); - nxt_queue_insert_tail(&tmcf->updating, &nskcf->link); + nxt_queue_insert_tail(&updating_sockets, &nskcf->link); return NXT_OK; } } - nxt_queue_insert_tail(&tmcf->pending, &nskcf->link); + nxt_queue_insert_tail(&pending_sockets, &nskcf->link); return NXT_DECLINED; } @@ -2182,6 +2220,8 @@ nxt_router_listen_socket_rpc_create(nxt_task_t *task, goto fail; } + b->completion_handler = nxt_router_dummy_buf_completion; + b->mem.free = nxt_cpymem(b->mem.free, skcf->listen->sockaddr, size); rt = task->thread->runtime; @@ -2222,7 +2262,7 @@ nxt_router_listen_socket_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, rpc = data; - s = msg->fd; + s = msg->fd[0]; ret = nxt_socket_nonblocking(task, s); if (nxt_slow_path(ret != NXT_OK)) { @@ -2360,7 +2400,7 @@ nxt_router_tls_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, goto fail; } - tlscf->chain_file = msg->fd; + tlscf->chain_file = msg->fd[0]; ret = task->thread->runtime->tls->server_init(task, tlscf); if (nxt_slow_path(ret != NXT_OK)) { @@ -2410,6 +2450,8 @@ nxt_router_app_rpc_create(nxt_task_t *task, goto fail; } + b->completion_handler = nxt_router_dummy_buf_completion; + nxt_buf_cpystr(b, &app->name); *b->mem.free++ = '\0'; nxt_buf_cpystr(b, &app->conf); @@ -2457,7 +2499,13 @@ nxt_router_app_prefork_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, app = rpc->app; port = msg->u.new_port; + + nxt_assert(port != NULL); + nxt_assert(port->type == NXT_PROCESS_APP); + nxt_assert(port->id == 0); + port->app = app; + port->main_app_port = port; app->pending_processes--; app->processes++; @@ -2468,10 +2516,18 @@ nxt_router_app_prefork_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_queue_insert_tail(&app->ports, &port->app_link); nxt_queue_insert_tail(&app->spare_ports, &port->idle_link); + nxt_debug(task, "app '%V' move new port %PI:%d to spare_ports", + &app->name, port->pid, port->id); + + nxt_port_hash_add(&app->port_hash, port); + app->port_hash_count++; + port->idle_start = 0; nxt_port_inc_use(port); + nxt_router_app_shared_port_send(task, port); + nxt_work_queue_add(&engine->fast_work_queue, nxt_router_conf_apply, task, rpc->temp_conf, NULL); } @@ -2577,13 +2633,13 @@ nxt_router_engine_conf_create(nxt_router_temp_conf_t *tmcf, { nxt_int_t ret; - ret = nxt_router_engine_joints_create(tmcf, recf, &tmcf->creating, + ret = nxt_router_engine_joints_create(tmcf, recf, &creating_sockets, nxt_router_listen_socket_create); if (nxt_slow_path(ret != NXT_OK)) { return ret; } - ret = nxt_router_engine_joints_create(tmcf, recf, &tmcf->updating, + ret = nxt_router_engine_joints_create(tmcf, recf, &updating_sockets, nxt_router_listen_socket_create); if (nxt_slow_path(ret != NXT_OK)) { return ret; @@ -2599,19 +2655,19 @@ nxt_router_engine_conf_update(nxt_router_temp_conf_t *tmcf, { nxt_int_t ret; - ret = nxt_router_engine_joints_create(tmcf, recf, &tmcf->creating, + ret = nxt_router_engine_joints_create(tmcf, recf, &creating_sockets, nxt_router_listen_socket_create); if (nxt_slow_path(ret != NXT_OK)) { return ret; } - ret = nxt_router_engine_joints_create(tmcf, recf, &tmcf->updating, + ret = nxt_router_engine_joints_create(tmcf, recf, &updating_sockets, nxt_router_listen_socket_update); if (nxt_slow_path(ret != NXT_OK)) { return ret; } - ret = nxt_router_engine_joints_delete(tmcf, recf, &tmcf->deleting); + ret = nxt_router_engine_joints_delete(tmcf, recf, &deleting_sockets); if (nxt_slow_path(ret != NXT_OK)) { return ret; } @@ -2631,12 +2687,12 @@ nxt_router_engine_conf_delete(nxt_router_temp_conf_t *tmcf, return ret; } - ret = nxt_router_engine_joints_delete(tmcf, recf, &tmcf->updating); + ret = nxt_router_engine_joints_delete(tmcf, recf, &updating_sockets); if (nxt_slow_path(ret != NXT_OK)) { return ret; } - return nxt_router_engine_joints_delete(tmcf, recf, &tmcf->deleting); + return nxt_router_engine_joints_delete(tmcf, recf, &deleting_sockets); } @@ -2874,10 +2930,11 @@ nxt_router_engine_post(nxt_event_engine_t *engine, nxt_work_t *jobs) static nxt_port_handlers_t nxt_router_app_port_handlers = { - .rpc_error = nxt_port_rpc_handler, - .mmap = nxt_port_mmap_handler, - .data = nxt_port_rpc_handler, - .oosm = nxt_router_oosm_handler, + .rpc_error = nxt_port_rpc_handler, + .mmap = nxt_port_mmap_handler, + .data = nxt_port_rpc_handler, + .oosm = nxt_router_oosm_handler, + .req_headers_ack = nxt_port_rpc_handler, }; @@ -2887,6 +2944,7 @@ nxt_router_thread_start(void *data) nxt_int_t ret; nxt_port_t *port; nxt_task_t *task; + nxt_work_t *work; nxt_thread_t *thread; nxt_thread_link_t *link; nxt_event_engine_t *engine; @@ -2927,15 +2985,53 @@ nxt_router_thread_start(void *data) return; } + ret = nxt_router_port_queue_init(task, port); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_use(task, port, -1); + return; + } + engine->port = port; nxt_port_enable(task, port, &nxt_router_app_port_handlers); + work = nxt_zalloc(sizeof(nxt_work_t)); + if (nxt_slow_path(work == NULL)) { + return; + } + + work->handler = nxt_router_rt_add_port; + work->task = link->work.task; + work->obj = work; + work->data = port; + + nxt_event_engine_post(link->work.task->thread->engine, work); + nxt_event_engine_start(engine); } static void +nxt_router_rt_add_port(nxt_task_t *task, void *obj, void *data) +{ + nxt_int_t res; + nxt_port_t *port; + nxt_runtime_t *rt; + + rt = task->thread->runtime; + port = data; + + nxt_free(obj); + + res = nxt_port_hash_add(&rt->ports, port); + + if (nxt_fast_path(res == NXT_OK)) { + nxt_port_use(task, port, 1); + } +} + + +static void nxt_router_listen_socket_create(nxt_task_t *task, void *obj, void *data) { nxt_joint_job_t *job; @@ -3146,7 +3242,15 @@ nxt_router_listen_event_release(nxt_task_t *task, nxt_listen_event_t *lev, nxt_debug(task, "listen event count: %D", lev->count); + engine = task->thread->engine; + if (--lev->count == 0) { + if (lev->next != NULL) { + nxt_sockaddr_cache_free(engine, lev->next); + + nxt_conn_free(task, lev->next); + } + nxt_free(lev); } @@ -3154,8 +3258,6 @@ nxt_router_listen_event_release(nxt_task_t *task, nxt_listen_event_t *lev, nxt_router_conf_release(task, joint); } - engine = task->thread->engine; - if (engine->shutdown && nxt_queue_is_empty(&engine->joints)) { nxt_thread_exit(task->thread); } @@ -3205,25 +3307,18 @@ nxt_router_conf_release(nxt_task_t *task, nxt_socket_conf_joint_t *joint) nxt_thread_spin_unlock(lock); - if (skcf != NULL) { - if (skcf->action != NULL) { - nxt_http_action_cleanup(task, skcf->action); - } - #if (NXT_TLS) - if (skcf->tls != NULL) { - task->thread->runtime->tls->server_free(task, skcf->tls); - } -#endif + if (skcf != NULL && skcf->tls != NULL) { + task->thread->runtime->tls->server_free(task, skcf->tls); } +#endif /* TODO remove engine->port */ - /* TODO excude from connected ports */ if (rtcf != NULL) { nxt_debug(task, "old router conf is destroyed"); - nxt_http_routes_cleanup(task, rtcf->routes); + nxt_router_apps_hash_use(task, rtcf, -1); nxt_router_access_log_release(task, lock, rtcf->access_log); @@ -3380,6 +3475,8 @@ nxt_router_access_log_open(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) goto fail; } + b->completion_handler = nxt_router_dummy_buf_completion; + nxt_buf_cpystr(b, &access_log->path); *b->mem.free++ = '\0'; @@ -3422,7 +3519,7 @@ nxt_router_access_log_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, access_log = tmcf->router_conf->access_log; - access_log->fd = msg->fd; + access_log->fd = msg->fd[0]; nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_router_conf_apply, task, tmcf, NULL); @@ -3474,7 +3571,7 @@ typedef struct { } nxt_router_access_log_reopen_t; -void +static void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_mp_t *mp; @@ -3571,13 +3668,13 @@ nxt_router_access_log_reopen_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, if (access_log == nxt_router->access_log) { - if (nxt_slow_path(dup2(msg->fd, access_log->fd) == -1)) { + if (nxt_slow_path(dup2(msg->fd[0], access_log->fd) == -1)) { nxt_alert(task, "dup2(%FD, %FD) failed %E", - msg->fd, access_log->fd, nxt_errno); + msg->fd[0], access_log->fd, nxt_errno); } } - nxt_fd_close(msg->fd); + nxt_fd_close(msg->fd[0]); nxt_mp_release(reopen->mem_pool); } @@ -3633,22 +3730,17 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { nxt_int_t ret; + nxt_app_t *app; nxt_buf_t *b, *next; nxt_port_t *app_port; nxt_unit_field_t *f; nxt_http_field_t *field; nxt_http_request_t *r; nxt_unit_response_t *resp; - nxt_request_app_link_t *req_app_link; nxt_request_rpc_data_t *req_rpc_data; - b = msg->buf; req_rpc_data = data; - if (msg->size == 0) { - b = NULL; - } - r = req_rpc_data->request; if (nxt_slow_path(r == NULL)) { return; @@ -3659,19 +3751,32 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, return; } + app = req_rpc_data->app; + nxt_assert(app != NULL); + + if (msg->port_msg.type == _NXT_PORT_MSG_REQ_HEADERS_ACK) { + nxt_router_req_headers_ack_handler(task, msg, req_rpc_data); + + return; + } + + b = (msg->size == 0) ? NULL : msg->buf; + if (msg->port_msg.last != 0) { nxt_debug(task, "router data create last buf"); nxt_buf_chain_add(&b, nxt_http_buf_last(r)); + req_rpc_data->rpc_cancel = 0; + req_rpc_data->apr_action = NXT_APR_GOT_RESPONSE; + nxt_request_rpc_data_unlink(task, req_rpc_data); } else { - if (req_rpc_data->app != NULL && req_rpc_data->app->timeout != 0) { + if (app->timeout != 0) { r->timer.handler = nxt_router_app_timeout; r->timer_data = req_rpc_data; - nxt_timer_add(task->thread->engine, &r->timer, - req_rpc_data->app->timeout); + nxt_timer_add(task->thread->engine, &r->timer, app->timeout); } } @@ -3767,39 +3872,21 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, if (r->websocket_handshake && r->status == NXT_HTTP_SWITCHING_PROTOCOLS) { - req_app_link = nxt_request_app_link_alloc(task, - req_rpc_data->req_app_link, - req_rpc_data); - if (nxt_slow_path(req_app_link == NULL)) { - goto fail; - } - - app_port = req_app_link->app_port; - - if (app_port == NULL && req_rpc_data->app_port != NULL) { - req_app_link->app_port = req_rpc_data->app_port; - app_port = req_app_link->app_port; - req_app_link->apr_action = req_rpc_data->apr_action; - - req_rpc_data->app_port = NULL; - } - + app_port = req_rpc_data->app_port; if (nxt_slow_path(app_port == NULL)) { goto fail; } - nxt_thread_mutex_lock(&req_rpc_data->app->mutex); + nxt_thread_mutex_lock(&app->mutex); - nxt_queue_insert_tail(&app_port->active_websockets, - &req_app_link->link_port_websockets); + app_port->main_app_port->active_websockets++; - nxt_thread_mutex_unlock(&req_rpc_data->app->mutex); + nxt_thread_mutex_unlock(&app->mutex); nxt_router_app_port_release(task, app_port, NXT_APR_UPGRADE); - req_app_link->apr_action = NXT_APR_CLOSE; + req_rpc_data->apr_action = NXT_APR_CLOSE; - nxt_debug(task, "req_app_link stream #%uD upgrade", - req_app_link->stream); + nxt_debug(task, "stream #%uD upgrade", req_rpc_data->stream); r->state = &nxt_http_websocket; @@ -3818,6 +3905,133 @@ fail: } +static void +nxt_router_req_headers_ack_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_request_rpc_data_t *req_rpc_data) +{ + int res; + nxt_app_t *app; + nxt_bool_t start_process, unlinked; + nxt_port_t *app_port, *main_app_port, *idle_port; + nxt_queue_link_t *idle_lnk; + nxt_http_request_t *r; + + nxt_debug(task, "stream #%uD: got ack from %PI:%d", + req_rpc_data->stream, + msg->port_msg.pid, msg->port_msg.reply_port); + + nxt_port_rpc_ex_set_peer(task, msg->port, req_rpc_data, + msg->port_msg.pid); + + app = req_rpc_data->app; + r = req_rpc_data->request; + + start_process = 0; + unlinked = 0; + + nxt_thread_mutex_lock(&app->mutex); + + if (r->app_link.next != NULL) { + nxt_queue_remove(&r->app_link); + r->app_link.next = NULL; + + unlinked = 1; + } + + app_port = nxt_port_hash_find(&app->port_hash, msg->port_msg.pid, + msg->port_msg.reply_port); + if (nxt_slow_path(app_port == NULL)) { + nxt_thread_mutex_unlock(&app->mutex); + + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); + + if (unlinked) { + nxt_mp_release(r->mem_pool); + } + + return; + } + + main_app_port = app_port->main_app_port; + + if (nxt_queue_chk_remove(&main_app_port->idle_link)) { + app->idle_processes--; + + nxt_debug(task, "app '%V' move port %PI:%d out of %s (ack)", + &app->name, main_app_port->pid, main_app_port->id, + (main_app_port->idle_start ? "idle_ports" : "spare_ports")); + + /* Check port was in 'spare_ports' using idle_start field. */ + if (main_app_port->idle_start == 0 + && app->idle_processes >= app->spare_processes) + { + /* + * If there is a vacant space in spare ports, + * move the last idle to spare_ports. + */ + nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); + + idle_lnk = nxt_queue_last(&app->idle_ports); + idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); + nxt_queue_remove(idle_lnk); + + nxt_queue_insert_tail(&app->spare_ports, idle_lnk); + + idle_port->idle_start = 0; + + nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " + "to spare_ports", + &app->name, idle_port->pid, idle_port->id); + } + + if (nxt_router_app_can_start(app) && nxt_router_app_need_start(app)) { + app->pending_processes++; + start_process = 1; + } + } + + main_app_port->active_requests++; + + nxt_port_inc_use(app_port); + + nxt_thread_mutex_unlock(&app->mutex); + + if (unlinked) { + nxt_mp_release(r->mem_pool); + } + + if (start_process) { + nxt_router_start_app_process(task, app); + } + + nxt_port_use(task, req_rpc_data->app_port, -1); + + req_rpc_data->app_port = app_port; + + if (req_rpc_data->msg_info.body_fd != -1) { + nxt_debug(task, "stream #%uD: send body fd %d", req_rpc_data->stream, + req_rpc_data->msg_info.body_fd); + + lseek(req_rpc_data->msg_info.body_fd, 0, SEEK_SET); + + res = nxt_port_socket_write(task, app_port, NXT_PORT_MSG_REQ_BODY, + req_rpc_data->msg_info.body_fd, + req_rpc_data->stream, + task->thread->engine->port->id, NULL); + + if (nxt_slow_path(res != NXT_OK)) { + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); + } + } + + if (app->timeout != 0) { + r->timer.handler = nxt_router_app_timeout; + r->timer_data = req_rpc_data; + nxt_timer_add(task->thread->engine, &r->timer, app->timeout); + } +} + + static const nxt_http_request_state_t nxt_http_request_send_state nxt_aligned(64) = { @@ -3846,42 +4060,14 @@ static void nxt_router_response_error_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_int_t res; - nxt_port_t *port; - nxt_bool_t cancelled; - nxt_request_app_link_t *req_app_link; nxt_request_rpc_data_t *req_rpc_data; req_rpc_data = data; - req_app_link = req_rpc_data->req_app_link; - - if (req_app_link != NULL) { - cancelled = nxt_router_msg_cancel(task, &req_app_link->msg_info, - req_app_link->stream); - if (cancelled) { - res = nxt_router_app_port(task, req_rpc_data->app, req_app_link); + req_rpc_data->rpc_cancel = 0; - if (res == NXT_OK) { - port = req_app_link->app_port; - - if (nxt_slow_path(port == NULL)) { - nxt_log(task, NXT_LOG_ERR, - "port is NULL in cancelled req_app_link"); - return; - } - - nxt_port_rpc_ex_set_peer(task, task->thread->engine->port, - req_rpc_data, port->pid); - - nxt_router_app_prepare_request(task, req_app_link); - } - - msg->port_msg.last = 0; - - return; - } - } + /* TODO cancel message and return if cancelled. */ + // nxt_router_msg_cancel(task, &req_rpc_data->msg_info, req_rpc_data->stream); if (req_rpc_data->request != NULL) { nxt_http_request_error(task, req_rpc_data->request, @@ -3905,6 +4091,8 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_assert(app_joint != NULL); nxt_assert(port != NULL); + nxt_assert(port->type == NXT_PROCESS_APP); + nxt_assert(port->id == 0); app = app_joint->app; @@ -3919,6 +4107,7 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, } port->app = app; + port->main_app_port = port; nxt_thread_mutex_lock(&app->mutex); @@ -3926,24 +4115,62 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, app->pending_processes--; app->processes++; + nxt_port_hash_add(&app->port_hash, port); + app->port_hash_count++; nxt_thread_mutex_unlock(&app->mutex); nxt_debug(task, "app '%V' new port ready, pid %PI, %d/%d", &app->name, port->pid, app->processes, app->pending_processes); + nxt_router_app_shared_port_send(task, port); + nxt_router_app_port_release(task, port, NXT_APR_NEW_PORT); } +static nxt_int_t +nxt_router_app_shared_port_send(nxt_task_t *task, nxt_port_t *app_port) +{ + nxt_buf_t *b; + nxt_port_t *port; + nxt_port_msg_new_port_t *msg; + + b = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, + sizeof(nxt_port_data_t)); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + port = app_port->app->shared_port; + + nxt_debug(task, "send port %FD to process %PI", + port->pair[0], app_port->pid); + + b->mem.free += sizeof(nxt_port_msg_new_port_t); + msg = (nxt_port_msg_new_port_t *) b->mem.pos; + + msg->id = port->id; + msg->pid = port->pid; + msg->max_size = port->max_size; + msg->max_share = port->max_share; + msg->type = port->type; + + return nxt_port_socket_write2(task, app_port, + NXT_PORT_MSG_NEW_PORT, + port->pair[0], port->queue_fd, + 0, 0, b); +} + + static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_app_t *app; - nxt_app_joint_t *app_joint; - nxt_queue_link_t *lnk; - nxt_request_app_link_t *req_app_link; + nxt_app_t *app; + nxt_app_joint_t *app_joint; + nxt_queue_link_t *link; + nxt_http_request_t *r; app_joint = data; @@ -3961,117 +4188,49 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_debug(task, "app '%V' %p start error", &app->name, app); + link = NULL; + nxt_thread_mutex_lock(&app->mutex); nxt_assert(app->pending_processes != 0); app->pending_processes--; - if (!nxt_queue_is_empty(&app->requests)) { - lnk = nxt_queue_last(&app->requests); - nxt_queue_remove(lnk); - lnk->next = NULL; + if (app->processes == 0 && !nxt_queue_is_empty(&app->ack_waiting_req)) { + link = nxt_queue_first(&app->ack_waiting_req); - req_app_link = nxt_queue_link_data(lnk, nxt_request_app_link_t, - link_app_requests); - - } else { - req_app_link = NULL; + nxt_queue_remove(link); + link->next = NULL; } nxt_thread_mutex_unlock(&app->mutex); - if (req_app_link != NULL) { - nxt_debug(task, "app '%V' %p abort next stream #%uD", - &app->name, app, req_app_link->stream); - - nxt_request_app_link_error(task, app, req_app_link, - "Failed to start application process"); - nxt_request_app_link_use(task, req_app_link, -1); - } -} - -nxt_inline nxt_port_t * -nxt_router_app_get_port_for_quit(nxt_app_t *app); - -void -nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i) -{ - int c; - - c = nxt_atomic_fetch_add(&app->use_count, i); - - if (i < 0 && c == -i) { - - if (task->thread->engine != app->engine) { - nxt_event_engine_post(app->engine, &app->joint->free_app_work); - - } else { - nxt_router_free_app(task, app->joint, NULL); - } - } -} - - -nxt_inline nxt_bool_t -nxt_router_app_first_port_busy(nxt_app_t *app) -{ - nxt_port_t *port; - nxt_queue_link_t *lnk; - - lnk = nxt_queue_first(&app->ports); - port = nxt_queue_link_data(lnk, nxt_port_t, app_link); - - return port->app_pending_responses > 0; -} - - -nxt_inline nxt_port_t * -nxt_router_pop_first_port(nxt_app_t *app) -{ - nxt_port_t *port; - nxt_queue_link_t *lnk; - - lnk = nxt_queue_first(&app->ports); - nxt_queue_remove(lnk); + while (link != NULL) { + r = nxt_container_of(link, nxt_http_request_t, app_link); - port = nxt_queue_link_data(lnk, nxt_port_t, app_link); + nxt_event_engine_post(r->engine, &r->err_work); - port->app_pending_responses++; + link = NULL; - if (nxt_queue_chk_remove(&port->idle_link)) { - app->idle_processes--; - - if (port->idle_start == 0) { - nxt_assert(app->idle_processes < app->spare_processes); + nxt_thread_mutex_lock(&app->mutex); - } else { - nxt_assert(app->idle_processes >= app->spare_processes); + if (app->processes == 0 && app->pending_processes == 0 + && !nxt_queue_is_empty(&app->ack_waiting_req)) + { + link = nxt_queue_first(&app->ack_waiting_req); - port->idle_start = 0; + nxt_queue_remove(link); + link->next = NULL; } - } - if ((app->max_pending_responses == 0 - || port->app_pending_responses < app->max_pending_responses) - && (app->max_requests == 0 - || port->app_responses + port->app_pending_responses - < app->max_requests)) - { - nxt_queue_insert_tail(&app->ports, lnk); - - nxt_port_inc_use(port); - - } else { - lnk->next = NULL; + nxt_thread_mutex_unlock(&app->mutex); } - - return port; } + nxt_inline nxt_port_t * -nxt_router_app_get_port_for_quit(nxt_app_t *app) +nxt_router_app_get_port_for_quit(nxt_task_t *task, nxt_app_t *app) { nxt_port_t *port; @@ -4081,19 +4240,20 @@ nxt_router_app_get_port_for_quit(nxt_app_t *app) nxt_queue_each(port, &app->ports, nxt_port_t, app_link) { - if (port->app_pending_responses > 0) { - port = NULL; - - continue; - } - /* Caller is responsible to decrease port use count. */ nxt_queue_chk_remove(&port->app_link); if (nxt_queue_chk_remove(&port->idle_link)) { app->idle_processes--; + + nxt_debug(task, "app '%V' move port %PI:%d out of %s for quit", + &app->name, port->pid, port->id, + (port->idle_start ? "idle_ports" : "spare_ports")); } + nxt_port_hash_remove(&app->port_hash, port); + app->port_hash_count--; + port->app = NULL; app->processes--; @@ -4108,41 +4268,32 @@ nxt_router_app_get_port_for_quit(nxt_app_t *app) static void -nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app) +nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i) { - nxt_debug(task, "app '%V' %p unlink", &app->name, app); + int c; - nxt_queue_remove(&app->link); + c = nxt_atomic_fetch_add(&app->use_count, i); - nxt_router_app_use(task, app, -1); + if (i < 0 && c == -i) { + + if (task->thread->engine != app->engine) { + nxt_event_engine_post(app->engine, &app->joint->free_app_work); + + } else { + nxt_router_free_app(task, app->joint, NULL); + } + } } static void -nxt_router_app_process_request(nxt_task_t *task, void *obj, void *data) +nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app) { - nxt_request_app_link_t *req_app_link; - - req_app_link = data; - -#if (NXT_DEBUG) - { - nxt_app_t *app; - - app = obj; - - nxt_assert(app != NULL); - nxt_assert(req_app_link != NULL); - nxt_assert(req_app_link->app_port != NULL); - - nxt_debug(task, "app '%V' %p process next stream #%uD", - &app->name, app, req_app_link->stream); - } -#endif + nxt_debug(task, "app '%V' %p unlink", &app->name, app); - nxt_router_app_prepare_request(task, req_app_link); + nxt_queue_remove(&app->link); - nxt_request_app_link_use(task, req_app_link, -1); + nxt_router_app_use(task, app, -1); } @@ -4150,40 +4301,33 @@ static void nxt_router_app_port_release(nxt_task_t *task, nxt_port_t *port, nxt_apr_action_t action) { - int inc_use; - uint32_t dec_pending, got_response; - nxt_app_t *app; - nxt_bool_t port_unchained; - nxt_bool_t send_quit, cancelled, adjust_idle_timer; - nxt_queue_link_t *lnk; - nxt_request_app_link_t *req_app_link, *pending_ra, *re_ra; - nxt_port_select_state_t state; + int inc_use; + uint32_t got_response, dec_requests; + nxt_app_t *app; + nxt_bool_t port_unchained, send_quit, adjust_idle_timer; + nxt_port_t *main_app_port; nxt_assert(port != NULL); nxt_assert(port->app != NULL); - req_app_link = NULL; - app = port->app; inc_use = 0; - dec_pending = 0; got_response = 0; + dec_requests = 0; switch (action) { case NXT_APR_NEW_PORT: break; case NXT_APR_REQUEST_FAILED: - dec_pending = 1; + dec_requests = 1; inc_use = -1; break; case NXT_APR_GOT_RESPONSE: - dec_pending = 1; got_response = 1; inc_use = -1; break; case NXT_APR_UPGRADE: - dec_pending = 1; got_response = 1; break; case NXT_APR_CLOSE: @@ -4191,120 +4335,49 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_port_t *port, break; } - nxt_thread_mutex_lock(&app->mutex); - - port->app_pending_responses -= dec_pending; - port->app_responses += got_response; + nxt_debug(task, "app '%V' release port %PI:%d: %d %d", &app->name, + port->pid, port->id, + (int) inc_use, (int) got_response); - if (port->pair[1] != -1 - && (app->max_pending_responses == 0 - || port->app_pending_responses < app->max_pending_responses) - && (app->max_requests == 0 - || port->app_responses + port->app_pending_responses - < app->max_requests)) - { - if (port->app_link.next == NULL) { - if (port->app_pending_responses > 0) { - nxt_queue_insert_tail(&app->ports, &port->app_link); + if (port == app->shared_port) { + nxt_thread_mutex_lock(&app->mutex); - } else { - nxt_queue_insert_head(&app->ports, &port->app_link); - } + app->active_requests -= got_response + dec_requests; - nxt_port_inc_use(port); + nxt_thread_mutex_unlock(&app->mutex); - } else { - if (port->app_pending_responses == 0 - && nxt_queue_first(&app->ports) != &port->app_link) - { - nxt_queue_remove(&port->app_link); - nxt_queue_insert_head(&app->ports, &port->app_link); - } - } + goto adjust_use; } - if (!nxt_queue_is_empty(&app->ports) - && !nxt_queue_is_empty(&app->requests)) - { - lnk = nxt_queue_first(&app->requests); - nxt_queue_remove(lnk); - lnk->next = NULL; + main_app_port = port->main_app_port; - req_app_link = nxt_queue_link_data(lnk, nxt_request_app_link_t, - link_app_requests); + nxt_thread_mutex_lock(&app->mutex); - req_app_link->app_port = nxt_router_pop_first_port(app); + main_app_port->app_responses += got_response; + main_app_port->active_requests -= got_response + dec_requests; + app->active_requests -= got_response + dec_requests; - if (req_app_link->app_port->app_pending_responses > 1) { - nxt_request_app_link_pending(task, app, req_app_link); - } - } - - /* Pop first pending request for this port. */ - if (dec_pending > 0 - && !nxt_queue_is_empty(&port->pending_requests)) + if (main_app_port->pair[1] != -1 + && (app->max_requests == 0 + || main_app_port->app_responses < app->max_requests)) { - lnk = nxt_queue_first(&port->pending_requests); - nxt_queue_remove(lnk); - lnk->next = NULL; - - pending_ra = nxt_queue_link_data(lnk, nxt_request_app_link_t, - link_port_pending); - - nxt_assert(pending_ra->link_app_pending.next != NULL); - - nxt_queue_remove(&pending_ra->link_app_pending); - pending_ra->link_app_pending.next = NULL; - - } else { - pending_ra = NULL; - } - - /* Try to cancel and re-schedule first stalled request for this app. */ - if (got_response > 0 && !nxt_queue_is_empty(&app->pending)) { - lnk = nxt_queue_first(&app->pending); + if (main_app_port->app_link.next == NULL) { + nxt_queue_insert_tail(&app->ports, &main_app_port->app_link); - re_ra = nxt_queue_link_data(lnk, nxt_request_app_link_t, - link_app_pending); - - if (re_ra->res_time <= nxt_thread_monotonic_time(task->thread)) { - - nxt_debug(task, "app '%V' stalled request #%uD detected", - &app->name, re_ra->stream); - - cancelled = nxt_router_msg_cancel(task, &re_ra->msg_info, - re_ra->stream); - - if (cancelled) { - state.req_app_link = re_ra; - state.app = app; - - /* - * Need to increment use count "in advance" because - * nxt_router_port_select() will remove re_ra from lists - * and decrement use count. - */ - nxt_request_app_link_inc_use(re_ra); - - nxt_router_port_select(task, &state); - - goto re_ra_cancelled; - } + nxt_port_inc_use(main_app_port); } } - re_ra = NULL; - -re_ra_cancelled: - send_quit = (app->max_requests > 0 - && port->app_pending_responses == 0 - && port->app_responses >= app->max_requests); + && main_app_port->app_responses >= app->max_requests); if (send_quit) { - port_unchained = nxt_queue_chk_remove(&port->app_link); + port_unchained = nxt_queue_chk_remove(&main_app_port->app_link); - port->app = NULL; + nxt_port_hash_remove(&app->port_hash, main_app_port); + app->port_hash_count--; + + main_app_port->app = NULL; app->processes--; } else { @@ -4313,9 +4386,10 @@ re_ra_cancelled: adjust_idle_timer = 0; - if (port->pair[1] != -1 && !send_quit && port->app_pending_responses == 0 - && nxt_queue_is_empty(&port->active_websockets) - && port->idle_link.next == NULL) + if (main_app_port->pair[1] != -1 && !send_quit + && main_app_port->active_requests == 0 + && main_app_port->active_websockets == 0 + && main_app_port->idle_link.next == NULL) { if (app->idle_processes == app->spare_processes && app->adjust_idle_work.data == NULL) @@ -4326,12 +4400,17 @@ re_ra_cancelled: } if (app->idle_processes < app->spare_processes) { - nxt_queue_insert_tail(&app->spare_ports, &port->idle_link); + nxt_queue_insert_tail(&app->spare_ports, &main_app_port->idle_link); + nxt_debug(task, "app '%V' move port %PI:%d to spare_ports", + &app->name, main_app_port->pid, main_app_port->id); } else { - nxt_queue_insert_tail(&app->idle_ports, &port->idle_link); + nxt_queue_insert_tail(&app->idle_ports, &main_app_port->idle_link); - port->idle_start = task->thread->engine->timers.now; + main_app_port->idle_start = task->thread->engine->timers.now; + + nxt_debug(task, "app '%V' move port %PI:%d to idle_ports", + &app->name, main_app_port->pid, main_app_port->id); } app->idle_processes++; @@ -4344,60 +4423,22 @@ re_ra_cancelled: nxt_event_engine_post(app->engine, &app->adjust_idle_work); } - if (pending_ra != NULL) { - nxt_request_app_link_use(task, pending_ra, -1); - } - - if (re_ra != NULL) { - if (nxt_router_port_post_select(task, &state) == NXT_OK) { - /* - * Reference counter already incremented above, this will - * keep re_ra while nxt_router_app_process_request() - * task is in queue. Reference counter decreased in - * nxt_router_app_process_request() after processing. - */ - - nxt_work_queue_add(&task->thread->engine->fast_work_queue, - nxt_router_app_process_request, - &task->thread->engine->task, app, re_ra); - - } else { - nxt_request_app_link_use(task, re_ra, -1); - } - } - - if (req_app_link != NULL) { - /* - * There should be call nxt_request_app_link_inc_use(req_app_link), - * because of one more link in the queue. But one link was - * recently removed from app->requests linked list. - * Corresponding decrement is in nxt_router_app_process_request(). - */ - - nxt_work_queue_add(&task->thread->engine->fast_work_queue, - nxt_router_app_process_request, - &task->thread->engine->task, app, req_app_link); - - goto adjust_use; - } - /* ? */ - if (port->pair[1] == -1) { + if (main_app_port->pair[1] == -1) { nxt_debug(task, "app '%V' %p port %p already closed (pid %PI dead?)", - &app->name, app, port, port->pid); + &app->name, app, main_app_port, main_app_port->pid); goto adjust_use; } if (send_quit) { - nxt_debug(task, "app '%V' %p send QUIT to port", - &app->name, app); + nxt_debug(task, "app '%V' %p send QUIT to port", &app->name, app); - nxt_port_socket_write(task, port, NXT_PORT_MSG_QUIT, - -1, 0, 0, NULL); + nxt_port_socket_write(task, main_app_port, NXT_PORT_MSG_QUIT, -1, 0, 0, + NULL); if (port_unchained) { - nxt_port_use(task, port, -1); + nxt_port_use(task, main_app_port, -1); } goto adjust_use; @@ -4426,11 +4467,27 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) nxt_thread_mutex_lock(&app->mutex); + nxt_port_hash_remove(&app->port_hash, port); + app->port_hash_count--; + + if (port->id != 0) { + nxt_thread_mutex_unlock(&app->mutex); + + nxt_debug(task, "app '%V' port (%PI, %d) closed", &app->name, + port->pid, port->id); + + return; + } + unchain = nxt_queue_chk_remove(&port->app_link); if (nxt_queue_chk_remove(&port->idle_link)) { app->idle_processes--; + nxt_debug(task, "app '%V' move port %PI:%d out of %s before close", + &app->name, port->pid, port->id, + (port->idle_start ? "idle_ports" : "spare_ports")); + if (port->idle_start == 0 && app->idle_processes >= app->spare_processes) { @@ -4443,6 +4500,10 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) nxt_queue_insert_tail(&app->spare_ports, idle_lnk); idle_port->idle_start = 0; + + nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " + "to spare_ports", + &app->name, idle_port->pid, idle_port->id); } } @@ -4450,8 +4511,7 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) start_process = !task->thread->engine->shutdown && nxt_router_app_can_start(app) - && (!nxt_queue_is_empty(&app->requests) - || nxt_router_app_need_start(app)); + && nxt_router_app_need_start(app); if (start_process) { app->pending_processes++; @@ -4500,6 +4560,10 @@ nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, void *data) app->adjust_idle_work.data = NULL; } + nxt_debug(task, "app '%V' idle_processes %d, spare_processes %d", + &app->name, + (int) app->idle_processes, (int) app->spare_processes); + while (app->idle_processes > app->spare_processes) { nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); @@ -4509,6 +4573,10 @@ nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, void *data) timeout = port->idle_start + app->idle_timeout; + nxt_debug(task, "app '%V' pid %PI, start %M, timeout %M, threshold %M", + &app->name, port->pid, + port->idle_start, timeout, threshold); + if (timeout > threshold) { break; } @@ -4516,8 +4584,14 @@ nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, void *data) nxt_queue_remove(lnk); lnk->next = NULL; + nxt_debug(task, "app '%V' move port %PI:%d out of idle_ports (timeout)", + &app->name, port->pid, port->id); + nxt_queue_chk_remove(&port->app_link); + nxt_port_hash_remove(&app->port_hash, port); + app->port_hash_count--; + app->idle_processes--; app->processes--; port->app = NULL; @@ -4588,7 +4662,7 @@ nxt_router_free_app(nxt_task_t *task, void *obj, void *data) app = app_joint->app; for ( ;; ) { - port = nxt_router_app_get_port_for_quit(app); + port = nxt_router_app_get_port_for_quit(task, app); if (port == NULL) { break; } @@ -4601,12 +4675,23 @@ nxt_router_free_app(nxt_task_t *task, void *obj, void *data) } nxt_assert(app->processes == 0); + nxt_assert(app->active_requests == 0); + nxt_assert(app->port_hash_count == 0); nxt_assert(app->idle_processes == 0); - nxt_assert(nxt_queue_is_empty(&app->requests)); nxt_assert(nxt_queue_is_empty(&app->ports)); nxt_assert(nxt_queue_is_empty(&app->spare_ports)); nxt_assert(nxt_queue_is_empty(&app->idle_ports)); + nxt_port_mmaps_destroy(&app->outgoing, 1); + + nxt_thread_mutex_destroy(&app->outgoing.mutex); + + if (app->shared_port != NULL) { + app->shared_port->app = NULL; + nxt_port_close(task, app->shared_port); + nxt_port_use(task, app->shared_port, -1); + } + nxt_thread_mutex_destroy(&app->mutex); nxt_mp_destroy(app->mem_pool); @@ -4623,178 +4708,49 @@ nxt_router_free_app(nxt_task_t *task, void *obj, void *data) static void -nxt_router_port_select(nxt_task_t *task, nxt_port_select_state_t *state) -{ - int ra_use_delta; - nxt_app_t *app; - nxt_bool_t can_start_process; - nxt_request_app_link_t *req_app_link; - - req_app_link = state->req_app_link; - app = state->app; - - state->failed_port_use_delta = 0; - ra_use_delta = -nxt_queue_chk_remove(&req_app_link->link_app_requests); - - if (nxt_queue_chk_remove(&req_app_link->link_port_pending)) - { - nxt_assert(req_app_link->link_app_pending.next != NULL); - - nxt_queue_remove(&req_app_link->link_app_pending); - req_app_link->link_app_pending.next = NULL; - - ra_use_delta--; - } - - state->failed_port = req_app_link->app_port; - - if (req_app_link->app_port != NULL) { - state->failed_port_use_delta--; - - state->failed_port->app_pending_responses--; - - if (nxt_queue_chk_remove(&state->failed_port->app_link)) { - state->failed_port_use_delta--; - } - - req_app_link->app_port = NULL; - } - - can_start_process = nxt_router_app_can_start(app); - - state->port = NULL; - state->start_process = 0; - - if (nxt_queue_is_empty(&app->ports) - || (can_start_process && nxt_router_app_first_port_busy(app)) ) - { - req_app_link = nxt_request_app_link_alloc(task, req_app_link, - req_app_link->req_rpc_data); - if (nxt_slow_path(req_app_link == NULL)) { - goto fail; - } - - if (nxt_slow_path(state->failed_port != NULL)) { - nxt_queue_insert_head(&app->requests, - &req_app_link->link_app_requests); - - } else { - nxt_queue_insert_tail(&app->requests, - &req_app_link->link_app_requests); - } - - ra_use_delta++; - - nxt_debug(task, "req_app_link stream #%uD enqueue to app->requests", - req_app_link->stream); - - if (can_start_process) { - app->pending_processes++; - state->start_process = 1; - } - - } else { - state->port = nxt_router_pop_first_port(app); - - if (state->port->app_pending_responses > 1) { - req_app_link = nxt_request_app_link_alloc(task, req_app_link, - req_app_link->req_rpc_data); - if (nxt_slow_path(req_app_link == NULL)) { - goto fail; - } - - req_app_link->app_port = state->port; - - nxt_request_app_link_pending(task, app, req_app_link); - } - - if (can_start_process && nxt_router_app_need_start(app)) { - app->pending_processes++; - state->start_process = 1; - } - } - - nxt_request_app_link_chk_use(req_app_link, ra_use_delta); - -fail: - - state->shared_ra = req_app_link; -} - - -static nxt_int_t -nxt_router_port_post_select(nxt_task_t *task, nxt_port_select_state_t *state) +nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, + nxt_request_rpc_data_t *req_rpc_data) { - nxt_int_t res; - nxt_app_t *app; - nxt_request_app_link_t *req_app_link; + nxt_bool_t start_process; + nxt_port_t *port; + nxt_http_request_t *r; - req_app_link = state->shared_ra; - app = state->app; + start_process = 0; - if (state->failed_port_use_delta != 0) { - nxt_port_use(task, state->failed_port, state->failed_port_use_delta); - } + nxt_thread_mutex_lock(&app->mutex); - if (nxt_slow_path(req_app_link == NULL)) { - if (state->port != NULL) { - nxt_port_use(task, state->port, -1); - } + port = app->shared_port; + nxt_port_inc_use(port); - nxt_request_app_link_error(task, app, state->req_app_link, - "Failed to allocate shared req<->app link"); + app->active_requests++; - return NXT_ERROR; + if (nxt_router_app_can_start(app) && nxt_router_app_need_start(app)) { + app->pending_processes++; + start_process = 1; } - if (state->port != NULL) { - nxt_debug(task, "already have port for app '%V' %p ", &app->name, app); - - req_app_link->app_port = state->port; - - if (state->start_process) { - nxt_router_start_app_process(task, app); - } - - return NXT_OK; - } + r = req_rpc_data->request; - if (!state->start_process) { - nxt_debug(task, "app '%V' %p too many running or pending processes", - &app->name, app); + /* + * Put request into application-wide list to be able to cancel request + * if something goes wrong with application processes. + */ + nxt_queue_insert_tail(&app->ack_waiting_req, &r->app_link); - return NXT_AGAIN; - } + nxt_thread_mutex_unlock(&app->mutex); - res = nxt_router_start_app_process(task, app); + /* + * Retain request memory pool while request is linked in ack_waiting_req + * to guarantee request structure memory is accessble. + */ + nxt_mp_retain(r->mem_pool); - if (nxt_slow_path(res != NXT_OK)) { - nxt_request_app_link_error(task, app, req_app_link, - "Failed to start app process"); + req_rpc_data->app_port = port; + req_rpc_data->apr_action = NXT_APR_REQUEST_FAILED; - return NXT_ERROR; + if (start_process) { + nxt_router_start_app_process(task, app); } - - return NXT_AGAIN; -} - - -static nxt_int_t -nxt_router_app_port(nxt_task_t *task, nxt_app_t *app, - nxt_request_app_link_t *req_app_link) -{ - nxt_port_select_state_t state; - - state.req_app_link = req_app_link; - state.app = app; - - nxt_thread_mutex_lock(&app->mutex); - - nxt_router_port_select(task, &state); - - nxt_thread_mutex_unlock(&app->mutex); - - return nxt_router_port_post_select(task, &state); } @@ -4802,10 +4758,7 @@ void nxt_router_process_http_request(nxt_task_t *task, nxt_http_request_t *r, nxt_app_t *app) { - nxt_int_t res; - nxt_port_t *port; nxt_event_engine_t *engine; - nxt_request_app_link_t ra_local, *req_app_link; nxt_request_rpc_data_t *req_rpc_data; engine = task->thread->engine; @@ -4824,7 +4777,7 @@ nxt_router_process_http_request(nxt_task_t *task, nxt_http_request_t *r, * in port handlers. Need to fixup request memory pool. Counterpart * release will be called via following call chain: * nxt_request_rpc_data_unlink() -> - * nxt_router_http_request_done() -> + * nxt_router_http_request_release_post() -> * nxt_router_http_request_release() */ nxt_mp_retain(r->mem_pool); @@ -4834,31 +4787,63 @@ nxt_router_process_http_request(nxt_task_t *task, nxt_http_request_t *r, r->timer.log = engine->task.log; r->timer.bias = NXT_TIMER_DEFAULT_BIAS; + r->engine = engine; + r->err_work.handler = nxt_router_http_request_error; + r->err_work.task = task; + r->err_work.obj = r; + req_rpc_data->stream = nxt_port_rpc_ex_stream(req_rpc_data); req_rpc_data->app = app; + req_rpc_data->msg_info.body_fd = -1; + req_rpc_data->rpc_cancel = 1; nxt_router_app_use(task, app, 1); req_rpc_data->request = r; r->req_rpc_data = req_rpc_data; - req_app_link = &ra_local; - nxt_request_app_link_init(task, req_app_link, req_rpc_data); + if (r->last != NULL) { + r->last->completion_handler = nxt_router_http_request_done; + } - res = nxt_router_app_port(task, app, req_app_link); - req_app_link = req_rpc_data->req_app_link; + nxt_router_app_port_get(task, app, req_rpc_data); + nxt_router_app_prepare_request(task, req_rpc_data); +} - if (res == NXT_OK) { - port = req_app_link->app_port; - nxt_assert(port != NULL); +static void +nxt_router_http_request_error(nxt_task_t *task, void *obj, void *data) +{ + nxt_http_request_t *r; + + r = obj; + + nxt_debug(task, "router http request error (rpc_data %p)", r->req_rpc_data); - nxt_port_rpc_ex_set_peer(task, engine->port, req_rpc_data, port->pid); + nxt_http_request_error(task, r, NXT_HTTP_SERVICE_UNAVAILABLE); + + if (r->req_rpc_data != NULL) { + nxt_request_rpc_data_unlink(task, r->req_rpc_data); + } + + nxt_mp_release(r->mem_pool); +} + + +static void +nxt_router_http_request_done(nxt_task_t *task, void *obj, void *data) +{ + nxt_http_request_t *r; + + r = data; - nxt_router_app_prepare_request(task, req_app_link); + nxt_debug(task, "router http request done (rpc_data %p)", r->req_rpc_data); + + if (r->req_rpc_data != NULL) { + nxt_request_rpc_data_unlink(task, r->req_rpc_data); } - nxt_request_app_link_use(task, req_app_link, -1); + nxt_http_request_close_handler(task, r, r->proto.any); } @@ -4870,91 +4855,106 @@ nxt_router_dummy_buf_completion(nxt_task_t *task, void *obj, void *data) static void nxt_router_app_prepare_request(nxt_task_t *task, - nxt_request_app_link_t *req_app_link) + nxt_request_rpc_data_t *req_rpc_data) { - nxt_buf_t *buf; + nxt_app_t *app; + nxt_buf_t *buf, *body; nxt_int_t res; - nxt_port_t *port, *c_port, *reply_port; - nxt_apr_action_t apr_action; + nxt_port_t *port, *reply_port; - nxt_assert(req_app_link->app_port != NULL); + int notify; + struct { + nxt_port_msg_t pm; + nxt_port_mmap_msg_t mm; + } msg; - port = req_app_link->app_port; - reply_port = req_app_link->reply_port; - apr_action = NXT_APR_REQUEST_FAILED; + app = req_rpc_data->app; - c_port = nxt_process_connected_port_find(port->process, reply_port); + nxt_assert(app != NULL); - if (nxt_slow_path(c_port != reply_port)) { - res = nxt_port_send_port(task, port, reply_port, 0); + port = req_rpc_data->app_port; - if (nxt_slow_path(res != NXT_OK)) { - nxt_request_app_link_error(task, port->app, req_app_link, - "Failed to send reply port to application"); + nxt_assert(port != NULL); + nxt_assert(port->queue != NULL); - goto release_port; - } + reply_port = task->thread->engine->port; - nxt_process_connected_port_add(port->process, reply_port); - } + buf = nxt_router_prepare_msg(task, req_rpc_data->request, app, + nxt_app_msg_prefix[app->type]); + if (nxt_slow_path(buf == NULL)) { + nxt_alert(task, "stream #%uD, app '%V': failed to prepare app message", + req_rpc_data->stream, &app->name); - buf = nxt_router_prepare_msg(task, req_app_link->request, port, - nxt_app_msg_prefix[port->app->type]); + nxt_http_request_error(task, req_rpc_data->request, + NXT_HTTP_INTERNAL_SERVER_ERROR); - if (nxt_slow_path(buf == NULL)) { - nxt_request_app_link_error(task, port->app, req_app_link, - "Failed to prepare message for application"); - goto release_port; + return; } nxt_debug(task, "about to send %O bytes buffer to app process port %d", nxt_buf_used_size(buf), port->socket.fd); - apr_action = NXT_APR_NEW_PORT; - - req_app_link->msg_info.buf = buf; - req_app_link->msg_info.completion_handler = buf->completion_handler; + req_rpc_data->msg_info.buf = buf; + req_rpc_data->msg_info.completion_handler = buf->completion_handler; - for (; buf; buf = buf->next) { + do { buf->completion_handler = nxt_router_dummy_buf_completion; - } + buf = buf->next; + } while (buf != NULL); - buf = req_app_link->msg_info.buf; + buf = req_rpc_data->msg_info.buf; - res = nxt_port_mmap_get_tracking(task, port, - &req_app_link->msg_info.tracking, - req_app_link->stream); - if (nxt_slow_path(res != NXT_OK)) { - nxt_request_app_link_error(task, port->app, req_app_link, - "Failed to get tracking area"); - goto release_port; - } - - if (req_app_link->body_fd != -1) { - nxt_debug(task, "stream #%uD: send body fd %d", req_app_link->stream, - req_app_link->body_fd); + body = req_rpc_data->request->body; - lseek(req_app_link->body_fd, 0, SEEK_SET); - } + if (body != NULL && nxt_buf_is_file(body)) { + req_rpc_data->msg_info.body_fd = body->file->fd; - res = nxt_port_socket_twrite(task, port, NXT_PORT_MSG_REQ_HEADERS, - req_app_link->body_fd, - req_app_link->stream, reply_port->id, buf, - &req_app_link->msg_info.tracking); + body->file->fd = -1; - if (nxt_slow_path(res != NXT_OK)) { - nxt_request_app_link_error(task, port->app, req_app_link, - "Failed to send message to application"); - goto release_port; - } + } else { + req_rpc_data->msg_info.body_fd = -1; + } + + msg.pm.stream = req_rpc_data->stream; + msg.pm.pid = reply_port->pid; + msg.pm.reply_port = reply_port->id; + msg.pm.type = NXT_PORT_MSG_REQ_HEADERS; + msg.pm.last = 0; + msg.pm.mmap = 1; + msg.pm.nf = 0; + msg.pm.mf = 0; + msg.pm.tracking = 0; + + nxt_port_mmap_handler_t *mmap_handler = buf->parent; + nxt_port_mmap_header_t *hdr = mmap_handler->hdr; + + msg.mm.mmap_id = hdr->id; + msg.mm.chunk_id = nxt_port_mmap_chunk_id(hdr, buf->mem.pos); + msg.mm.size = nxt_buf_used_size(buf); + + res = nxt_app_queue_send(port->queue, &msg, sizeof(msg), + req_rpc_data->stream, ¬ify, + &req_rpc_data->msg_info.tracking_cookie); + if (nxt_fast_path(res == NXT_OK)) { + if (notify != 0) { + (void) nxt_port_socket_write(task, port, + NXT_PORT_MSG_READ_QUEUE, + -1, req_rpc_data->stream, + reply_port->id, NULL); -release_port: + } else { + nxt_debug(task, "queue is not empty"); + } - nxt_router_app_port_release(task, port, apr_action); + } else { + nxt_alert(task, "stream #%uD, app '%V': failed to send app message", + req_rpc_data->stream, &app->name); - nxt_request_app_link_update_peer(task, req_app_link); + nxt_http_request_error(task, req_rpc_data->request, + NXT_HTTP_INTERNAL_SERVER_ERROR); + } } @@ -5012,7 +5012,7 @@ nxt_fields_next(nxt_fields_iter_t *i) static nxt_buf_t * nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, - nxt_port_t *port, const nxt_str_t *prefix) + nxt_app_t *app, const nxt_str_t *prefix) { void *target_pos, *query_pos; u_char *pos, *end, *p, c; @@ -5053,7 +5053,7 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, return NULL; } - out = nxt_port_mmap_get_buf(task, port, + out = nxt_port_mmap_get_buf(task, &app->outgoing, nxt_min(req_size + content_length, PORT_MMAP_DATA_SIZE)); if (nxt_slow_path(out == NULL)) { return NULL; @@ -5235,7 +5235,7 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, if (buf == NULL) { free_size = nxt_min(size, PORT_MMAP_DATA_SIZE); - buf = nxt_port_mmap_get_buf(task, port, free_size); + buf = nxt_port_mmap_get_buf(task, &app->outgoing, free_size); if (nxt_slow_path(buf == NULL)) { while (out != NULL) { buf = out->next; @@ -5283,15 +5283,9 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) { - nxt_app_t *app; - nxt_bool_t cancelled, unlinked; - nxt_port_t *port; nxt_timer_t *timer; - nxt_queue_link_t *lnk; nxt_http_request_t *r; - nxt_request_app_link_t *pending_ra; nxt_request_rpc_data_t *req_rpc_data; - nxt_port_select_state_t state; timer = obj; @@ -5299,94 +5293,6 @@ nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) r = nxt_timer_data(timer, nxt_http_request_t, timer); req_rpc_data = r->timer_data; - app = req_rpc_data->app; - - if (app == NULL) { - goto generate_error; - } - - port = NULL; - pending_ra = NULL; - - if (req_rpc_data->app_port != NULL) { - port = req_rpc_data->app_port; - req_rpc_data->app_port = NULL; - } - - if (port == NULL && req_rpc_data->req_app_link != NULL - && req_rpc_data->req_app_link->app_port != NULL) - { - port = req_rpc_data->req_app_link->app_port; - req_rpc_data->req_app_link->app_port = NULL; - } - - if (port == NULL) { - goto generate_error; - } - - nxt_thread_mutex_lock(&app->mutex); - - unlinked = nxt_queue_chk_remove(&port->app_link); - - if (!nxt_queue_is_empty(&port->pending_requests)) { - lnk = nxt_queue_first(&port->pending_requests); - - pending_ra = nxt_queue_link_data(lnk, nxt_request_app_link_t, - link_port_pending); - - nxt_assert(pending_ra->link_app_pending.next != NULL); - - nxt_debug(task, "app '%V' pending request #%uD found", - &app->name, pending_ra->stream); - - cancelled = nxt_router_msg_cancel(task, &pending_ra->msg_info, - pending_ra->stream); - - if (cancelled) { - state.req_app_link = pending_ra; - state.app = app; - - /* - * Need to increment use count "in advance" because - * nxt_router_port_select() will remove pending_ra from lists - * and decrement use count. - */ - nxt_request_app_link_inc_use(pending_ra); - - nxt_router_port_select(task, &state); - - } else { - pending_ra = NULL; - } - } - - nxt_thread_mutex_unlock(&app->mutex); - - if (pending_ra != NULL) { - if (nxt_router_port_post_select(task, &state) == NXT_OK) { - /* - * Reference counter already incremented above, this will - * keep pending_ra while nxt_router_app_process_request() - * task is in queue. Reference counter decreased in - * nxt_router_app_process_request() after processing. - */ - - nxt_work_queue_add(&task->thread->engine->fast_work_queue, - nxt_router_app_process_request, - &task->thread->engine->task, app, pending_ra); - - } else { - nxt_request_app_link_use(task, pending_ra, -1); - } - } - - nxt_debug(task, "send quit to app '%V' pid %PI", &app->name, port->pid); - - nxt_port_socket_write(task, port, NXT_PORT_MSG_QUIT, -1, 0, 0, NULL); - - nxt_port_use(task, port, unlinked ? -2 : -1); - -generate_error: nxt_http_request_error(task, r, NXT_HTTP_SERVICE_UNAVAILABLE); @@ -5394,13 +5300,11 @@ generate_error: } -static nxt_int_t -nxt_router_http_request_done(nxt_task_t *task, nxt_http_request_t *r) +static void +nxt_router_http_request_release_post(nxt_task_t *task, nxt_http_request_t *r) { r->timer.handler = nxt_router_http_request_release; nxt_timer_add(task->thread->engine, &r->timer, 0); - - return NXT_OK; } @@ -5409,7 +5313,7 @@ nxt_router_http_request_release(nxt_task_t *task, void *obj, void *data) { nxt_http_request_t *r; - nxt_debug(task, "http app release"); + nxt_debug(task, "http request pool release"); r = nxt_timer_data(obj, nxt_http_request_t, timer); @@ -5468,3 +5372,120 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) -1, 0, 0, NULL); } } + + +static void +nxt_router_get_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_fd_t fd; + nxt_port_t *port; + nxt_runtime_t *rt; + nxt_port_mmaps_t *mmaps; + nxt_port_msg_get_mmap_t *get_mmap_msg; + nxt_port_mmap_handler_t *mmap_handler; + + rt = task->thread->runtime; + + port = nxt_runtime_port_find(rt, msg->port_msg.pid, + msg->port_msg.reply_port); + if (nxt_slow_path(port == NULL)) { + nxt_alert(task, "get_mmap_handler: reply_port %PI:%d not found", + msg->port_msg.pid, msg->port_msg.reply_port); + + return; + } + + if (nxt_slow_path(nxt_buf_used_size(msg->buf) + < (int) sizeof(nxt_port_msg_get_mmap_t))) + { + nxt_alert(task, "get_mmap_handler: message buffer too small (%d)", + (int) nxt_buf_used_size(msg->buf)); + + return; + } + + get_mmap_msg = (nxt_port_msg_get_mmap_t *) msg->buf->mem.pos; + + nxt_assert(port->type == NXT_PROCESS_APP); + + if (nxt_slow_path(port->app == NULL)) { + nxt_alert(task, "get_mmap_handler: app == NULL for reply port %PI:%d", + port->pid, port->id); + + // FIXME + nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, + -1, msg->port_msg.stream, 0, NULL); + + return; + } + + mmaps = &port->app->outgoing; + nxt_thread_mutex_lock(&mmaps->mutex); + + if (nxt_slow_path(get_mmap_msg->id >= mmaps->size)) { + nxt_thread_mutex_unlock(&mmaps->mutex); + + nxt_alert(task, "get_mmap_handler: mmap id is too big (%d)", + (int) get_mmap_msg->id); + + // FIXME + nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, + -1, msg->port_msg.stream, 0, NULL); + return; + } + + mmap_handler = mmaps->elts[get_mmap_msg->id].mmap_handler; + + fd = mmap_handler->fd; + + nxt_thread_mutex_unlock(&mmaps->mutex); + + nxt_debug(task, "get mmap %PI:%d found", + msg->port_msg.pid, (int) get_mmap_msg->id); + + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_MMAP, fd, 0, 0, NULL); +} + + +static void +nxt_router_get_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_t *port, *reply_port; + nxt_runtime_t *rt; + nxt_port_msg_get_port_t *get_port_msg; + + rt = task->thread->runtime; + + reply_port = nxt_runtime_port_find(rt, msg->port_msg.pid, + msg->port_msg.reply_port); + if (nxt_slow_path(reply_port == NULL)) { + nxt_alert(task, "get_port_handler: reply_port %PI:%d not found", + msg->port_msg.pid, msg->port_msg.reply_port); + + return; + } + + if (nxt_slow_path(nxt_buf_used_size(msg->buf) + < (int) sizeof(nxt_port_msg_get_port_t))) + { + nxt_alert(task, "get_port_handler: message buffer too small (%d)", + (int) nxt_buf_used_size(msg->buf)); + + return; + } + + get_port_msg = (nxt_port_msg_get_port_t *) msg->buf->mem.pos; + + port = nxt_runtime_port_find(rt, get_port_msg->pid, get_port_msg->id); + if (nxt_slow_path(port == NULL)) { + nxt_alert(task, "get_port_handler: port %PI:%d not found", + get_port_msg->pid, get_port_msg->id); + + return; + } + + nxt_debug(task, "get port %PI:%d found", get_port_msg->pid, + get_port_msg->id); + + (void) nxt_port_send_port(task, reply_port, port, msg->port_msg.stream); +} diff --git a/src/nxt_router.h b/src/nxt_router.h index 6004a459..81b3538c 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -48,6 +48,7 @@ typedef struct { nxt_upstreams_t *upstreams; nxt_lvlhsh_t mtypes_hash; + nxt_lvlhsh_t apps_hash; nxt_router_access_log_t *access_log; } nxt_router_conf_t; @@ -66,12 +67,6 @@ typedef struct { typedef struct { - nxt_queue_t creating; /* of nxt_socket_conf_t */ - nxt_queue_t pending; /* of nxt_socket_conf_t */ - nxt_queue_t updating; /* of nxt_socket_conf_t */ - nxt_queue_t keeping; /* of nxt_socket_conf_t */ - nxt_queue_t deleting; /* of nxt_socket_conf_t */ - #if (NXT_TLS) nxt_queue_t tls; /* of nxt_router_tlssock_t */ #endif @@ -107,18 +102,20 @@ typedef struct { struct nxt_app_s { - nxt_thread_mutex_t mutex; /* Protects ports queue. */ - nxt_queue_t ports; /* of nxt_port_t.app_link */ + nxt_thread_mutex_t mutex; /* Protects ports queue. */ + nxt_queue_t ports; /* of nxt_port_t.app_link */ + nxt_lvlhsh_t port_hash; /* of nxt_port_t */ nxt_queue_t spare_ports; /* of nxt_port_t.idle_link */ nxt_queue_t idle_ports; /* of nxt_port_t.idle_link */ nxt_work_t adjust_idle_work; nxt_event_engine_t *engine; - nxt_queue_t requests; /* of nxt_request_app_link_t */ - nxt_queue_t pending; /* of nxt_request_app_link_t */ nxt_str_t name; + uint32_t port_hash_count; + + uint32_t active_requests; uint32_t pending_processes; uint32_t processes; uint32_t idle_processes; @@ -126,7 +123,6 @@ struct nxt_app_s { uint32_t max_processes; uint32_t spare_processes; uint32_t max_pending_processes; - uint32_t max_pending_responses; uint32_t max_requests; nxt_msec_t timeout; @@ -143,8 +139,12 @@ struct nxt_app_s { nxt_str_t conf; nxt_atomic_t use_count; + nxt_queue_t ack_waiting_req; /* of nxt_http_request_t.app_link */ nxt_app_joint_t *joint; + nxt_port_t *shared_port; + + nxt_port_mmaps_t outgoing; }; @@ -219,18 +219,11 @@ struct nxt_router_access_log_s { }; -void nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -void nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -void nxt_router_remove_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -void nxt_router_access_log_reopen_handler(nxt_task_t *task, - nxt_port_recv_msg_t *msg); - void nxt_router_process_http_request(nxt_task_t *task, nxt_http_request_t *r, nxt_app_t *app); void nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port); -void nxt_router_listener_application(nxt_router_temp_conf_t *tmcf, +nxt_int_t nxt_router_listener_application(nxt_router_conf_t *rtcf, nxt_str_t *name, nxt_http_action_t *action); -void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i); void nxt_router_listen_event_release(nxt_task_t *task, nxt_listen_event_t *lev, nxt_socket_conf_joint_t *joint); void nxt_router_conf_release(nxt_task_t *task, nxt_socket_conf_joint_t *joint); diff --git a/src/nxt_router_request.h b/src/nxt_router_request.h index a38980ee..95044dbb 100644 --- a/src/nxt_router_request.h +++ b/src/nxt_router_request.h @@ -7,16 +7,14 @@ #define _NXT_ROUTER_REQUEST_H_INCLUDED_ -typedef struct nxt_msg_info_s { +typedef struct { nxt_buf_t *buf; - nxt_port_mmap_tracking_t tracking; + nxt_fd_t body_fd; + uint32_t tracking_cookie; nxt_work_handler_t completion_handler; } nxt_msg_info_t; -typedef struct nxt_request_app_link_s nxt_request_app_link_t; - - typedef enum { NXT_APR_NEW_PORT, NXT_APR_REQUEST_FAILED, @@ -35,38 +33,9 @@ typedef struct { nxt_http_request_t *request; nxt_msg_info_t msg_info; - nxt_request_app_link_t *req_app_link; -} nxt_request_rpc_data_t; - -struct nxt_request_app_link_s { - uint32_t stream; - nxt_atomic_t use_count; - - nxt_port_t *app_port; - nxt_apr_action_t apr_action; - - nxt_port_t *reply_port; - nxt_http_request_t *request; - nxt_msg_info_t msg_info; - nxt_request_rpc_data_t *req_rpc_data; - nxt_fd_t body_fd; - - nxt_nsec_t res_time; - - nxt_queue_link_t link_app_requests; /* for nxt_app_t.requests */ - /* for nxt_port_t.pending_requests */ - nxt_queue_link_t link_port_pending; - nxt_queue_link_t link_app_pending; /* for nxt_app_t.pending */ - /* for nxt_port_t.active_websockets */ - nxt_queue_link_t link_port_websockets; - - nxt_mp_t *mem_pool; - nxt_work_t work; - - int err_code; - const char *err_str; -}; + nxt_bool_t rpc_cancel; +} nxt_request_rpc_data_t; #endif /* _NXT_ROUTER_REQUEST_H_INCLUDED_ */ diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 5aa061dd..435276a0 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -124,6 +124,14 @@ nxt_runtime_create(nxt_task_t *task) goto fail; } + if (nxt_slow_path(nxt_http_register_variables() != NXT_OK)) { + goto fail; + } + + if (nxt_slow_path(nxt_var_index_init() != NXT_OK)) { + goto fail; + } + nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_runtime_start, task, rt, NULL); @@ -527,7 +535,7 @@ nxt_runtime_stop_all_processes(nxt_task_t *task, nxt_runtime_t *rt) static void nxt_runtime_exit(nxt_task_t *task, void *obj, void *data) { - int status; + int status, engine_count; nxt_runtime_t *rt; nxt_process_t *process; nxt_event_engine_t *engine; @@ -571,14 +579,25 @@ nxt_runtime_exit(nxt_task_t *task, void *obj, void *data) } nxt_runtime_process_loop; - if (rt->port_by_type[rt->type] != NULL) { - nxt_port_use(task, rt->port_by_type[rt->type], -1); - } + status = rt->status; - nxt_thread_mutex_destroy(&rt->processes_mutex); + engine_count = 0; - status = rt->status; - nxt_mp_destroy(rt->mem_pool); + nxt_queue_each(engine, &rt->engines, nxt_event_engine_t, link) { + + engine_count++; + + } nxt_queue_loop; + + if (engine_count <= 1) { + if (rt->port_by_type[rt->type] != NULL) { + nxt_port_use(task, rt->port_by_type[rt->type], -1); + } + + nxt_thread_mutex_destroy(&rt->processes_mutex); + + nxt_mp_destroy(rt->mem_pool); + } nxt_debug(task, "exit: %d", status); @@ -1366,7 +1385,6 @@ nxt_runtime_process_new(nxt_runtime_t *rt) nxt_queue_init(&process->ports); nxt_thread_mutex_create(&process->incoming.mutex); - nxt_thread_mutex_create(&process->outgoing.mutex); nxt_thread_mutex_create(&process->cp_mutex); process->use_count = 1; @@ -1378,8 +1396,6 @@ nxt_runtime_process_new(nxt_runtime_t *rt) void nxt_runtime_process_release(nxt_runtime_t *rt, nxt_process_t *process) { - nxt_port_t *port; - if (process->registered == 1) { nxt_runtime_process_remove(rt, process); } @@ -1388,15 +1404,8 @@ nxt_runtime_process_release(nxt_runtime_t *rt, nxt_process_t *process) nxt_assert(process->registered == 0); nxt_port_mmaps_destroy(&process->incoming, 1); - nxt_port_mmaps_destroy(&process->outgoing, 1); - - do { - port = nxt_port_hash_retrieve(&process->connected_ports); - - } while (port != NULL); nxt_thread_mutex_destroy(&process->incoming.mutex); - nxt_thread_mutex_destroy(&process->outgoing.mutex); nxt_thread_mutex_destroy(&process->cp_mutex); /* processes from nxt_runtime_process_get() have no memory pool */ diff --git a/src/nxt_runtime.h b/src/nxt_runtime.h index d29b6b4d..0fb8c9a1 100644 --- a/src/nxt_runtime.h +++ b/src/nxt_runtime.h @@ -135,6 +135,8 @@ void nxt_cdecl nxt_log_time_handler(nxt_uint_t level, nxt_log_t *log, void nxt_stream_connection_init(nxt_task_t *task, void *obj, void *data); +nxt_int_t nxt_http_register_variables(void); + #define nxt_runtime_process_each(rt, process) \ do { \ diff --git a/src/nxt_socket.c b/src/nxt_socket.c index cc3d7378..a8e0d514 100644 --- a/src/nxt_socket.c +++ b/src/nxt_socket.c @@ -51,18 +51,6 @@ nxt_socket_create(nxt_task_t *task, nxt_uint_t domain, nxt_uint_t type, void -nxt_socket_close(nxt_task_t *task, nxt_socket_t s) -{ - if (nxt_fast_path(close(s) == 0)) { - nxt_debug(task, "socket close(%d)", s); - - } else { - nxt_alert(task, "socket close(%d) failed %E", s, nxt_socket_errno); - } -} - - -void nxt_socket_defer_accept(nxt_task_t *task, nxt_socket_t s, nxt_sockaddr_t *sa) { #if (NXT_HAVE_UNIX_DOMAIN) @@ -291,6 +279,41 @@ nxt_socket_shutdown(nxt_task_t *task, nxt_socket_t s, nxt_uint_t how) } +void +nxt_socket_close(nxt_task_t *task, nxt_socket_t s) +{ + nxt_err_t err; + nxt_uint_t level; + + if (nxt_fast_path(close(s) == 0)) { + nxt_debug(task, "socket close(%d)", s); + return; + } + + err = nxt_socket_errno; + + switch (err) { + + case NXT_ENOTCONN: + level = NXT_LOG_DEBUG; + break; + + case NXT_ECONNRESET: + case NXT_ENETDOWN: + case NXT_ENETUNREACH: + case NXT_EHOSTDOWN: + case NXT_EHOSTUNREACH: + level = NXT_LOG_ERR; + break; + + default: + level = NXT_LOG_ALERT; + } + + nxt_log(task, level, "socket close(%d) failed %E", s, err); +} + + nxt_err_t nxt_socket_error(nxt_socket_t s) { diff --git a/src/nxt_socket.h b/src/nxt_socket.h index e39d8e4d..7403de3d 100644 --- a/src/nxt_socket.h +++ b/src/nxt_socket.h @@ -93,7 +93,6 @@ typedef union { NXT_EXPORT nxt_socket_t nxt_socket_create(nxt_task_t *task, nxt_uint_t family, nxt_uint_t type, nxt_uint_t protocol, nxt_uint_t flags); -NXT_EXPORT void nxt_socket_close(nxt_task_t *task, nxt_socket_t s); NXT_EXPORT void nxt_socket_defer_accept(nxt_task_t *task, nxt_socket_t s, nxt_sockaddr_t *sa); NXT_EXPORT nxt_int_t nxt_socket_getsockopt(nxt_task_t *task, nxt_socket_t s, @@ -106,13 +105,14 @@ NXT_EXPORT nxt_int_t nxt_socket_connect(nxt_task_t *task, nxt_socket_t s, nxt_sockaddr_t *sa); NXT_EXPORT void nxt_socket_shutdown(nxt_task_t *task, nxt_socket_t s, nxt_uint_t how); +NXT_EXPORT void nxt_socket_close(nxt_task_t *task, nxt_socket_t s); nxt_err_t nxt_socket_error(nxt_socket_t s); nxt_uint_t nxt_socket_error_level(nxt_err_t err); NXT_EXPORT nxt_int_t nxt_socketpair_create(nxt_task_t *task, nxt_socket_t *pair); NXT_EXPORT void nxt_socketpair_close(nxt_task_t *task, nxt_socket_t *pair); -NXT_EXPORT ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t fd, +NXT_EXPORT ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob); NXT_EXPORT ssize_t nxt_socketpair_recv(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob); diff --git a/src/nxt_socketpair.c b/src/nxt_socketpair.c index 10ea562e..8b9d12bf 100644 --- a/src/nxt_socketpair.c +++ b/src/nxt_socketpair.c @@ -20,7 +20,7 @@ #endif -static ssize_t nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, +static ssize_t nxt_sendmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob); static ssize_t nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob); @@ -71,7 +71,7 @@ nxt_socketpair_close(nxt_task_t *task, nxt_socket_t *pair) ssize_t -nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t fd, nxt_iobuf_t *iob, +nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) { ssize_t n; @@ -82,7 +82,8 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t fd, nxt_iobuf_t *iob, err = (n == -1) ? nxt_socket_errno : 0; - nxt_debug(ev->task, "sendmsg(%d, %FD, %ui): %z", ev->fd, fd, niob, n); + nxt_debug(ev->task, "sendmsg(%d, %FD, %FD, %ui): %z", ev->fd, fd[0], + fd[1], niob, n); if (n > 0) { return n; @@ -108,8 +109,8 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t fd, nxt_iobuf_t *iob, continue; default: - nxt_alert(ev->task, "sendmsg(%d, %FD, %ui) failed %E", - ev->fd, fd, niob, err); + nxt_alert(ev->task, "sendmsg(%d, %FD, %FD, %ui) failed %E", + ev->fd, fd[0], fd[1], niob, err); return NXT_ERROR; } @@ -133,7 +134,8 @@ nxt_socketpair_recv(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, err = (n == -1) ? nxt_socket_errno : 0; - nxt_debug(ev->task, "recvmsg(%d, %FD, %ui): %z", ev->fd, *fd, niob, n); + nxt_debug(ev->task, "recvmsg(%d, %FD, %FD, %ui): %z", ev->fd, fd[0], + fd[1], niob, n); if (n > 0) { return n; @@ -178,12 +180,13 @@ nxt_socketpair_recv(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, */ static ssize_t -nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, nxt_uint_t niob) +nxt_sendmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) { + size_t csize; struct msghdr msg; union { struct cmsghdr cm; - char space[CMSG_SPACE(sizeof(int))]; + char space[CMSG_SPACE(sizeof(int) * 2)]; } cmsg; msg.msg_name = NULL; @@ -193,15 +196,17 @@ nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, nxt_uint_t niob) /* Flags are cleared just to suppress valgrind warning. */ msg.msg_flags = 0; - if (fd != -1) { + if (fd[0] != -1) { + csize = (fd[1] == -1) ? sizeof(int) : sizeof(int) * 2; + msg.msg_control = (caddr_t) &cmsg; - msg.msg_controllen = sizeof(cmsg); + msg.msg_controllen = CMSG_SPACE(csize); #if (NXT_VALGRIND) nxt_memzero(&cmsg, sizeof(cmsg)); #endif - cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int)); + cmsg.cm.cmsg_len = CMSG_LEN(csize); cmsg.cm.cmsg_level = SOL_SOCKET; cmsg.cm.cmsg_type = SCM_RIGHTS; @@ -214,7 +219,7 @@ nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, nxt_uint_t niob) * Fortunately, GCC with -O1 compiles this nxt_memcpy() * in the same simple assignment as in the code above. */ - nxt_memcpy(CMSG_DATA(&cmsg.cm), &fd, sizeof(int)); + nxt_memcpy(CMSG_DATA(&cmsg.cm), fd, csize); } else { msg.msg_control = NULL; @@ -232,7 +237,7 @@ nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) struct msghdr msg; union { struct cmsghdr cm; - char space[CMSG_SPACE(sizeof(int))]; + char space[CMSG_SPACE(sizeof(int) * 2)]; } cmsg; msg.msg_name = NULL; @@ -242,7 +247,8 @@ nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) msg.msg_control = (caddr_t) &cmsg; msg.msg_controllen = sizeof(cmsg); - *fd = -1; + fd[0] = -1; + fd[1] = -1; #if (NXT_VALGRIND) nxt_memzero(&cmsg, sizeof(cmsg)); @@ -251,12 +257,16 @@ nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) n = recvmsg(s, &msg, 0); if (n > 0 - && cmsg.cm.cmsg_len == CMSG_LEN(sizeof(int)) && cmsg.cm.cmsg_level == SOL_SOCKET && cmsg.cm.cmsg_type == SCM_RIGHTS) { - /* (*fd) = *(int *) CMSG_DATA(&cmsg.cm); */ - nxt_memcpy(fd, CMSG_DATA(&cmsg.cm), sizeof(int)); + if (cmsg.cm.cmsg_len == CMSG_LEN(sizeof(int))) { + nxt_memcpy(fd, CMSG_DATA(&cmsg.cm), sizeof(int)); + } + + if (cmsg.cm.cmsg_len == CMSG_LEN(sizeof(int) * 2)) { + nxt_memcpy(fd, CMSG_DATA(&cmsg.cm), sizeof(int) * 2); + } } return n; @@ -267,7 +277,7 @@ nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) /* Solaris 4.3BSD sockets. */ static ssize_t -nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, nxt_uint_t niob) +nxt_sendmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) { struct msghdr msg; @@ -276,10 +286,14 @@ nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, nxt_uint_t niob) msg.msg_iov = iob; msg.msg_iovlen = niob; - if (fd != -1) { - msg.msg_accrights = (caddr_t) &fd; + if (fd[0] != -1) { + msg.msg_accrights = (caddr_t) fd; msg.msg_accrightslen = sizeof(int); + if (fd[1] != -1) { + msg.msg_accrightslen += sizeof(int); + } + } else { msg.msg_accrights = NULL; msg.msg_accrightslen = 0; @@ -294,14 +308,15 @@ nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) { struct msghdr msg; - *fd = -1; + fd[0] = -1; + fd[1] = -1; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_iov = iob; msg.msg_iovlen = niob; msg.msg_accrights = (caddr_t) fd; - msg.msg_accrightslen = sizeof(int); + msg.msg_accrightslen = sizeof(int) * 2; return recvmsg(s, &msg, 0); } diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 9f6eab95..6b7d631d 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -7,6 +7,8 @@ #include "nxt_main.h" #include "nxt_port_memory_int.h" +#include "nxt_port_queue.h" +#include "nxt_app_queue.h" #include "nxt_unit.h" #include "nxt_unit_request.h" @@ -38,20 +40,30 @@ typedef struct nxt_unit_websocket_frame_impl_s nxt_unit_websocket_frame_impl_t; static nxt_unit_impl_t *nxt_unit_create(nxt_unit_init_t *init); static int nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, void *data); +nxt_inline void nxt_unit_ctx_use(nxt_unit_ctx_t *ctx); +nxt_inline void nxt_unit_ctx_release(nxt_unit_ctx_t *ctx); +nxt_inline void nxt_unit_lib_use(nxt_unit_impl_t *lib); +nxt_inline void nxt_unit_lib_release(nxt_unit_impl_t *lib); nxt_inline void nxt_unit_mmap_buf_insert(nxt_unit_mmap_buf_t **head, nxt_unit_mmap_buf_t *mmap_buf); nxt_inline void nxt_unit_mmap_buf_insert_tail(nxt_unit_mmap_buf_t **prev, nxt_unit_mmap_buf_t *mmap_buf); nxt_inline void nxt_unit_mmap_buf_unlink(nxt_unit_mmap_buf_t *mmap_buf); static int nxt_unit_read_env(nxt_unit_port_t *ready_port, - nxt_unit_port_t *read_port, int *log_fd, uint32_t *stream, - uint32_t *shm_limit); -static int nxt_unit_ready(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, - uint32_t stream); + nxt_unit_port_t *router_port, nxt_unit_port_t *read_port, + int *log_fd, uint32_t *stream, uint32_t *shm_limit); +static int nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, + int queue_fd); +static int nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf); static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); +static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, + nxt_unit_recv_msg_t *recv_msg); +static int nxt_unit_request_check_response_port(nxt_unit_request_info_t *req, + nxt_unit_port_id_t *port_id); +static int nxt_unit_send_req_headers_ack(nxt_unit_request_info_t *req); static int nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_process_shm_ack(nxt_unit_ctx_t *ctx); @@ -63,11 +75,9 @@ static nxt_unit_websocket_frame_impl_t *nxt_unit_websocket_frame_get( nxt_unit_ctx_t *ctx); static void nxt_unit_websocket_frame_release(nxt_unit_websocket_frame_t *ws); static void nxt_unit_websocket_frame_free(nxt_unit_websocket_frame_impl_t *ws); -static nxt_unit_process_t *nxt_unit_msg_get_process(nxt_unit_ctx_t *ctx, - nxt_unit_recv_msg_t *recv_msg); static nxt_unit_mmap_buf_t *nxt_unit_mmap_buf_get(nxt_unit_ctx_t *ctx); static void nxt_unit_mmap_buf_release(nxt_unit_mmap_buf_t *mmap_buf); -static int nxt_unit_mmap_buf_send(nxt_unit_ctx_t *ctx, uint32_t stream, +static int nxt_unit_mmap_buf_send(nxt_unit_request_info_t *req, nxt_unit_mmap_buf_t *mmap_buf, int last); static void nxt_unit_mmap_buf_free(nxt_unit_mmap_buf_t *mmap_buf); static void nxt_unit_free_outgoing_buf(nxt_unit_mmap_buf_t *mmap_buf); @@ -81,70 +91,97 @@ static nxt_unit_mmap_buf_t *nxt_unit_request_preread( static ssize_t nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size); static nxt_port_mmap_header_t *nxt_unit_mmap_get(nxt_unit_ctx_t *ctx, - nxt_unit_process_t *process, nxt_unit_port_id_t *port_id, - nxt_chunk_id_t *c, int *n, int min_n); -static int nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id); + nxt_unit_port_t *port, nxt_chunk_id_t *c, int *n, int min_n); +static int nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); static int nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx); static nxt_unit_mmap_t *nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i); static nxt_port_mmap_header_t *nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, - nxt_unit_process_t *process, nxt_unit_port_id_t *port_id, int n); -static int nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, + nxt_unit_port_t *port, int n); +static int nxt_unit_shm_open(nxt_unit_ctx_t *ctx, size_t size); +static int nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int fd); static int nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, - nxt_unit_process_t *process, nxt_unit_port_id_t *port_id, uint32_t size, + nxt_unit_port_t *port, uint32_t size, uint32_t min_size, nxt_unit_mmap_buf_t *mmap_buf, char *local_buf); static int nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd); static void nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps); -static void nxt_unit_process_use(nxt_unit_ctx_t *ctx, - nxt_unit_process_t *process, int i); +nxt_inline void nxt_unit_process_use(nxt_unit_process_t *process); +nxt_inline void nxt_unit_process_release(nxt_unit_process_t *process); static void nxt_unit_mmaps_destroy(nxt_unit_mmaps_t *mmaps); -static nxt_port_mmap_header_t *nxt_unit_get_incoming_mmap(nxt_unit_ctx_t *ctx, - nxt_unit_process_t *process, uint32_t id); -static int nxt_unit_tracking_read(nxt_unit_ctx_t *ctx, - nxt_unit_recv_msg_t *recv_msg); +static int nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, + nxt_unit_mmaps_t *mmaps, pid_t pid, uint32_t id, + nxt_port_mmap_header_t **hdr, nxt_unit_read_buf_t *rbuf); static int nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, - nxt_unit_recv_msg_t *recv_msg); + nxt_unit_recv_msg_t *recv_msg, nxt_unit_read_buf_t *rbuf); +static int nxt_unit_get_mmap(nxt_unit_ctx_t *ctx, pid_t pid, uint32_t id); static void nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, - nxt_unit_process_t *process, nxt_port_mmap_header_t *hdr, void *start, uint32_t size); static int nxt_unit_send_shm_ack(nxt_unit_ctx_t *ctx, pid_t pid); -static nxt_unit_process_t *nxt_unit_process_get(nxt_unit_ctx_t *ctx, +static nxt_unit_process_t *nxt_unit_process_get(nxt_unit_impl_t *lib, pid_t pid); -static nxt_unit_process_t *nxt_unit_process_find(nxt_unit_ctx_t *ctx, +static nxt_unit_process_t *nxt_unit_process_find(nxt_unit_impl_t *lib, pid_t pid, int remove); static nxt_unit_process_t *nxt_unit_process_pop_first(nxt_unit_impl_t *lib); -static void nxt_unit_read_buf(nxt_unit_ctx_t *ctx, - nxt_unit_read_buf_t *rbuf); -static int nxt_unit_create_port(nxt_unit_ctx_t *ctx, - nxt_unit_port_id_t *port_id, int *fd); - -static int nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *dst, - nxt_unit_port_id_t *new_port, int fd); - -static void nxt_unit_remove_port_unsafe(nxt_unit_ctx_t *ctx, - nxt_unit_port_id_t *port_id, nxt_unit_port_t *r_port, - nxt_unit_process_t **process); -static void nxt_unit_remove_process(nxt_unit_ctx_t *ctx, +static int nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx); +static int nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf); +static int nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx); +static void nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx); +nxt_inline int nxt_unit_is_read_queue(nxt_unit_read_buf_t *rbuf); +nxt_inline int nxt_unit_is_read_socket(nxt_unit_read_buf_t *rbuf); +nxt_inline int nxt_unit_is_shm_ack(nxt_unit_read_buf_t *rbuf); +nxt_inline int nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf); +static int nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, + nxt_unit_port_t *port); +static void nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl); +static nxt_unit_port_t *nxt_unit_create_port(nxt_unit_ctx_t *ctx); + +static int nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *dst, + nxt_unit_port_t *port, int queue_fd); + +nxt_inline void nxt_unit_port_use(nxt_unit_port_t *port); +nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port); +static nxt_unit_port_t *nxt_unit_add_port(nxt_unit_ctx_t *ctx, + nxt_unit_port_t *port, void *queue); +static void nxt_unit_remove_port(nxt_unit_impl_t *lib, + nxt_unit_port_id_t *port_id); +static nxt_unit_port_t *nxt_unit_remove_port_unsafe(nxt_unit_impl_t *lib, + nxt_unit_port_id_t *port_id); +static void nxt_unit_remove_pid(nxt_unit_impl_t *lib, pid_t pid); +static void nxt_unit_remove_process(nxt_unit_impl_t *lib, nxt_unit_process_t *process); - -static ssize_t nxt_unit_port_send_default(nxt_unit_ctx_t *ctx, - nxt_unit_port_id_t *port_id, const void *buf, size_t buf_size, +static void nxt_unit_quit(nxt_unit_ctx_t *ctx); +static int nxt_unit_get_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id); +static ssize_t nxt_unit_port_send(nxt_unit_ctx_t *ctx, + nxt_unit_port_t *port, const void *buf, size_t buf_size, const void *oob, size_t oob_size); -static ssize_t nxt_unit_port_recv_default(nxt_unit_ctx_t *ctx, - nxt_unit_port_id_t *port_id, void *buf, size_t buf_size, - void *oob, size_t oob_size); +static ssize_t nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, + const void *buf, size_t buf_size, const void *oob, size_t oob_size); +static int nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf); +nxt_inline void nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst, + nxt_unit_read_buf_t *src); +static int nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf); +static int nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf); +static int nxt_unit_port_queue_recv(nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf); +static int nxt_unit_app_queue_recv(nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf); +nxt_inline int nxt_unit_close(int fd); +static int nxt_unit_fd_blocking(int fd); static int nxt_unit_port_hash_add(nxt_lvlhsh_t *port_hash, nxt_unit_port_t *port); -static nxt_unit_port_impl_t *nxt_unit_port_hash_find(nxt_lvlhsh_t *port_hash, +static nxt_unit_port_t *nxt_unit_port_hash_find(nxt_lvlhsh_t *port_hash, nxt_unit_port_id_t *port_id, int remove); -static int nxt_unit_request_hash_add(nxt_lvlhsh_t *request_hash, - nxt_unit_request_info_impl_t *req_impl); -static nxt_unit_request_info_impl_t *nxt_unit_request_hash_find( - nxt_lvlhsh_t *request_hash, uint32_t stream, int remove); +static int nxt_unit_request_hash_add(nxt_unit_ctx_t *ctx, + nxt_unit_request_info_t *req); +static nxt_unit_request_info_t *nxt_unit_request_hash_find( + nxt_unit_ctx_t *ctx, uint32_t stream, int remove); static char * nxt_unit_snprint_prefix(char *p, char *end, pid_t pid, int level); @@ -156,10 +193,8 @@ struct nxt_unit_mmap_buf_s { nxt_unit_mmap_buf_t **prev; nxt_port_mmap_header_t *hdr; - nxt_unit_port_id_t port_id; nxt_unit_request_info_t *req; nxt_unit_ctx_impl_t *ctx_impl; - nxt_unit_process_t *process; char *free_ptr; char *plain_ptr; }; @@ -176,8 +211,7 @@ struct nxt_unit_recv_msg_s { void *start; uint32_t size; - int fd; - nxt_unit_process_t *process; + int fd[2]; nxt_unit_mmap_buf_t *incoming_buf; }; @@ -197,15 +231,17 @@ struct nxt_unit_request_info_impl_s { uint32_t stream; - nxt_unit_process_t *process; - nxt_unit_mmap_buf_t *outgoing_buf; nxt_unit_mmap_buf_t *incoming_buf; nxt_unit_req_state_t state; uint8_t websocket; + uint8_t in_hash; + /* for nxt_unit_ctx_impl_t.free_req or active_req */ nxt_queue_link_t link; + /* for nxt_unit_port_impl_t.awaiting_req */ + nxt_queue_link_t port_wait_link; char extra_data[]; }; @@ -223,7 +259,8 @@ struct nxt_unit_websocket_frame_impl_s { struct nxt_unit_read_buf_s { - nxt_unit_read_buf_t *next; + nxt_queue_link_t link; + nxt_unit_ctx_impl_t *ctx_impl; ssize_t size; char buf[16384]; char oob[256]; @@ -233,10 +270,12 @@ struct nxt_unit_read_buf_s { struct nxt_unit_ctx_impl_s { nxt_unit_ctx_t ctx; + nxt_atomic_t use_count; + nxt_atomic_t wait_items; + pthread_mutex_t mutex; - nxt_unit_port_id_t read_port_id; - int read_port_fd; + nxt_unit_port_t *read_port; nxt_queue_link_t link; @@ -254,9 +293,14 @@ struct nxt_unit_ctx_impl_s { /* of nxt_unit_request_info_impl_t */ nxt_lvlhsh_t requests; - nxt_unit_read_buf_t *pending_read_head; - nxt_unit_read_buf_t **pending_read_tail; - nxt_unit_read_buf_t *free_read_buf; + /* of nxt_unit_request_info_impl_t */ + nxt_queue_t ready_req; + + /* of nxt_unit_read_buf_t */ + nxt_queue_t pending_rbuf; + + /* of nxt_unit_read_buf_t */ + nxt_queue_t free_rbuf; nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -265,10 +309,29 @@ struct nxt_unit_ctx_impl_s { }; +struct nxt_unit_mmap_s { + nxt_port_mmap_header_t *hdr; + + /* of nxt_unit_read_buf_t */ + nxt_queue_t awaiting_rbuf; +}; + + +struct nxt_unit_mmaps_s { + pthread_mutex_t mutex; + uint32_t size; + uint32_t cap; + nxt_atomic_t allocated_chunks; + nxt_unit_mmap_t *elts; +}; + + struct nxt_unit_impl_s { nxt_unit_t unit; nxt_unit_callbacks_t callbacks; + nxt_atomic_t use_count; + uint32_t request_data_size; uint32_t shm_mmap_limit; @@ -277,10 +340,14 @@ struct nxt_unit_impl_s { nxt_lvlhsh_t processes; /* of nxt_unit_process_t */ nxt_lvlhsh_t ports; /* of nxt_unit_port_impl_t */ - nxt_unit_port_id_t ready_port_id; + nxt_unit_port_t *router_port; + nxt_unit_port_t *shared_port; nxt_queue_t contexts; /* of nxt_unit_ctx_impl_t */ + nxt_unit_mmaps_t incoming; + nxt_unit_mmaps_t outgoing; + pid_t pid; int log_fd; int online; @@ -292,32 +359,28 @@ struct nxt_unit_impl_s { struct nxt_unit_port_impl_s { nxt_unit_port_t port; + nxt_atomic_t use_count; + + /* for nxt_unit_process_t.ports */ nxt_queue_link_t link; nxt_unit_process_t *process; -}; + /* of nxt_unit_request_info_impl_t */ + nxt_queue_t awaiting_req; -struct nxt_unit_mmap_s { - nxt_port_mmap_header_t *hdr; -}; + int ready; + void *queue; -struct nxt_unit_mmaps_s { - pthread_mutex_t mutex; - uint32_t size; - uint32_t cap; - nxt_atomic_t allocated_chunks; - nxt_unit_mmap_t *elts; + int from_socket; + nxt_unit_read_buf_t *socket_rbuf; }; struct nxt_unit_process_s { pid_t pid; - nxt_queue_t ports; - - nxt_unit_mmaps_t incoming; - nxt_unit_mmaps_t outgoing; + nxt_queue_t ports; /* of nxt_unit_port_impl_t */ nxt_unit_impl_t *lib; @@ -337,34 +400,41 @@ typedef struct { nxt_unit_ctx_t * nxt_unit_init(nxt_unit_init_t *init) { - int rc; + int rc, queue_fd; + void *mem; uint32_t ready_stream, shm_limit; nxt_unit_ctx_t *ctx; nxt_unit_impl_t *lib; - nxt_unit_port_t ready_port, read_port; + nxt_unit_port_t ready_port, router_port, read_port; lib = nxt_unit_create(init); if (nxt_slow_path(lib == NULL)) { return NULL; } + queue_fd = -1; + mem = MAP_FAILED; + if (init->ready_port.id.pid != 0 && init->ready_stream != 0 && init->read_port.id.pid != 0) { ready_port = init->ready_port; ready_stream = init->ready_stream; + router_port = init->router_port; read_port = init->read_port; lib->log_fd = init->log_fd; nxt_unit_port_id_init(&ready_port.id, ready_port.id.pid, ready_port.id.id); + nxt_unit_port_id_init(&router_port.id, router_port.id.pid, + router_port.id.id); nxt_unit_port_id_init(&read_port.id, read_port.id.pid, read_port.id.id); } else { - rc = nxt_unit_read_env(&ready_port, &read_port, &lib->log_fd, - &ready_stream, &shm_limit); + rc = nxt_unit_read_env(&ready_port, &router_port, &read_port, + &lib->log_fd, &ready_stream, &shm_limit); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } @@ -378,37 +448,77 @@ nxt_unit_init(nxt_unit_init_t *init) } lib->pid = read_port.id.pid; + ctx = &lib->main_ctx.ctx; - rc = lib->callbacks.add_port(ctx, &ready_port); - if (rc != NXT_UNIT_OK) { - nxt_unit_alert(NULL, "failed to add ready_port"); + rc = nxt_unit_fd_blocking(router_port.out_fd); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; + } + + lib->router_port = nxt_unit_add_port(ctx, &router_port, NULL); + if (nxt_slow_path(lib->router_port == NULL)) { + nxt_unit_alert(NULL, "failed to add router_port"); + + goto fail; + } + queue_fd = nxt_unit_shm_open(ctx, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue_fd == -1)) { goto fail; } - rc = lib->callbacks.add_port(ctx, &read_port); + mem = mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, MAP_SHARED, queue_fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", queue_fd, + strerror(errno), errno); + + goto fail; + } + + nxt_port_queue_init(mem); + + rc = nxt_unit_fd_blocking(read_port.in_fd); if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; + } + + lib->main_ctx.read_port = nxt_unit_add_port(ctx, &read_port, mem); + if (nxt_slow_path(lib->main_ctx.read_port == NULL)) { nxt_unit_alert(NULL, "failed to add read_port"); goto fail; } - lib->main_ctx.read_port_id = read_port.id; - lib->ready_port_id = ready_port.id; + rc = nxt_unit_fd_blocking(ready_port.out_fd); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; + } - rc = nxt_unit_ready(ctx, &ready_port.id, ready_stream); + rc = nxt_unit_ready(ctx, ready_port.out_fd, ready_stream, queue_fd); if (nxt_slow_path(rc != NXT_UNIT_OK)) { nxt_unit_alert(NULL, "failed to send READY message"); goto fail; } + nxt_unit_close(ready_port.out_fd); + nxt_unit_close(queue_fd); + return ctx; fail: - free(lib); + if (mem != MAP_FAILED) { + munmap(mem, sizeof(nxt_port_queue_t)); + } + + if (queue_fd != -1) { + nxt_unit_close(queue_fd); + } + + nxt_unit_ctx_release(&lib->main_ctx.ctx); return NULL; } @@ -450,8 +560,13 @@ nxt_unit_create(nxt_unit_init_t *init) nxt_queue_init(&lib->contexts); + lib->use_count = 0; + lib->router_port = NULL; + lib->shared_port = NULL; + rc = nxt_unit_ctx_init(lib, &lib->main_ctx, init->ctx_data); if (nxt_slow_path(rc != NXT_UNIT_OK)) { + pthread_mutex_destroy(&lib->mutex); goto fail; } @@ -460,32 +575,12 @@ nxt_unit_create(nxt_unit_init_t *init) if (cb->request_handler == NULL) { nxt_unit_alert(NULL, "request_handler is NULL"); + pthread_mutex_destroy(&lib->mutex); goto fail; } - if (cb->add_port == NULL) { - cb->add_port = nxt_unit_add_port; - } - - if (cb->remove_port == NULL) { - cb->remove_port = nxt_unit_remove_port; - } - - if (cb->remove_pid == NULL) { - cb->remove_pid = nxt_unit_remove_pid; - } - - if (cb->quit == NULL) { - cb->quit = nxt_unit_quit; - } - - if (cb->port_send == NULL) { - cb->port_send = nxt_unit_port_send_default; - } - - if (cb->port_recv == NULL) { - cb->port_recv = nxt_unit_port_recv_default; - } + nxt_unit_mmaps_init(&lib->incoming); + nxt_unit_mmaps_init(&lib->outgoing); return lib; @@ -506,8 +601,6 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, ctx_impl->ctx.data = data; ctx_impl->ctx.unit = &lib->unit; - nxt_queue_insert_tail(&lib->contexts, &ctx_impl->link); - rc = pthread_mutex_init(&ctx_impl->mutex, NULL); if (nxt_slow_path(rc != 0)) { nxt_unit_alert(NULL, "failed to initialize mutex (%d)", rc); @@ -515,25 +608,33 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, return NXT_UNIT_ERROR; } + nxt_unit_lib_use(lib); + + nxt_queue_insert_tail(&lib->contexts, &ctx_impl->link); + + ctx_impl->use_count = 1; + ctx_impl->wait_items = 0; + nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); nxt_queue_init(&ctx_impl->active_req); + nxt_queue_init(&ctx_impl->ready_req); + nxt_queue_init(&ctx_impl->pending_rbuf); + nxt_queue_init(&ctx_impl->free_rbuf); ctx_impl->free_buf = NULL; nxt_unit_mmap_buf_insert(&ctx_impl->free_buf, &ctx_impl->ctx_buf[1]); nxt_unit_mmap_buf_insert(&ctx_impl->free_buf, &ctx_impl->ctx_buf[0]); nxt_queue_insert_tail(&ctx_impl->free_req, &ctx_impl->req.link); + nxt_queue_insert_tail(&ctx_impl->free_rbuf, &ctx_impl->ctx_read_buf.link); - ctx_impl->pending_read_head = NULL; - ctx_impl->pending_read_tail = &ctx_impl->pending_read_head; - ctx_impl->free_read_buf = &ctx_impl->ctx_read_buf; - ctx_impl->ctx_read_buf.next = NULL; + ctx_impl->ctx_read_buf.ctx_impl = ctx_impl; ctx_impl->req.req.ctx = &ctx_impl->ctx; ctx_impl->req.req.unit = &lib->unit; - ctx_impl->read_port_fd = -1; + ctx_impl->read_port = NULL; ctx_impl->requests.slot = 0; return NXT_UNIT_OK; @@ -541,6 +642,80 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, nxt_inline void +nxt_unit_ctx_use(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + nxt_atomic_fetch_add(&ctx_impl->use_count, 1); +} + + +nxt_inline void +nxt_unit_ctx_release(nxt_unit_ctx_t *ctx) +{ + long c; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + c = nxt_atomic_fetch_add(&ctx_impl->use_count, -1); + + if (c == 1) { + nxt_unit_ctx_free(ctx_impl); + } +} + + +nxt_inline void +nxt_unit_lib_use(nxt_unit_impl_t *lib) +{ + nxt_atomic_fetch_add(&lib->use_count, 1); +} + + +nxt_inline void +nxt_unit_lib_release(nxt_unit_impl_t *lib) +{ + long c; + nxt_unit_process_t *process; + + c = nxt_atomic_fetch_add(&lib->use_count, -1); + + if (c == 1) { + for ( ;; ) { + pthread_mutex_lock(&lib->mutex); + + process = nxt_unit_process_pop_first(lib); + if (process == NULL) { + pthread_mutex_unlock(&lib->mutex); + + break; + } + + nxt_unit_remove_process(lib, process); + } + + pthread_mutex_destroy(&lib->mutex); + + if (nxt_fast_path(lib->router_port != NULL)) { + nxt_unit_port_release(lib->router_port); + } + + if (nxt_fast_path(lib->shared_port != NULL)) { + nxt_unit_port_release(lib->shared_port); + } + + nxt_unit_mmaps_destroy(&lib->incoming); + nxt_unit_mmaps_destroy(&lib->outgoing); + + free(lib); + } +} + + +nxt_inline void nxt_unit_mmap_buf_insert(nxt_unit_mmap_buf_t **head, nxt_unit_mmap_buf_t *mmap_buf) { @@ -585,15 +760,16 @@ nxt_unit_mmap_buf_unlink(nxt_unit_mmap_buf_t *mmap_buf) static int -nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *read_port, - int *log_fd, uint32_t *stream, uint32_t *shm_limit) +nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, + nxt_unit_port_t *read_port, int *log_fd, uint32_t *stream, + uint32_t *shm_limit) { int rc; - int ready_fd, read_fd; + int ready_fd, router_fd, read_fd; char *unit_init, *version_end; long version_length; - int64_t ready_pid, read_pid; - uint32_t ready_stream, ready_id, read_id; + int64_t ready_pid, router_pid, read_pid; + uint32_t ready_stream, router_id, ready_id, read_id; unit_init = getenv(NXT_UNIT_INIT_ENV); if (nxt_slow_path(unit_init == NULL)) { @@ -621,13 +797,15 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *read_port, "%"PRIu32";" "%"PRId64",%"PRIu32",%d;" "%"PRId64",%"PRIu32",%d;" + "%"PRId64",%"PRIu32",%d;" "%d,%"PRIu32, &ready_stream, &ready_pid, &ready_id, &ready_fd, + &router_pid, &router_id, &router_fd, &read_pid, &read_id, &read_fd, log_fd, shm_limit); - if (nxt_slow_path(rc != 9)) { + if (nxt_slow_path(rc != 12)) { nxt_unit_alert(NULL, "failed to scan variables: %d", rc); return NXT_UNIT_ERROR; @@ -639,6 +817,12 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *read_port, ready_port->out_fd = ready_fd; ready_port->data = NULL; + nxt_unit_port_id_init(&router_port->id, (pid_t) router_pid, router_id); + + router_port->in_fd = -1; + router_port->out_fd = router_fd; + router_port->data = NULL; + nxt_unit_port_id_init(&read_port->id, (pid_t) read_pid, read_id); read_port->in_fd = read_fd; @@ -652,13 +836,17 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *read_port, static int -nxt_unit_ready(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, - uint32_t stream) +nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd) { ssize_t res; nxt_port_msg_t msg; nxt_unit_impl_t *lib; + union { + struct cmsghdr cm; + char space[CMSG_SPACE(sizeof(int))]; + } cmsg; + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); msg.stream = stream; @@ -671,7 +859,25 @@ nxt_unit_ready(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, msg.mf = 0; msg.tracking = 0; - res = lib->callbacks.port_send(ctx, port_id, &msg, sizeof(msg), NULL, 0); + memset(&cmsg, 0, sizeof(cmsg)); + + cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int)); + cmsg.cm.cmsg_level = SOL_SOCKET; + cmsg.cm.cmsg_type = SCM_RIGHTS; + + /* + * memcpy() is used instead of simple + * *(int *) CMSG_DATA(&cmsg.cm) = fd; + * because GCC 4.4 with -O2/3/s optimization may issue a warning: + * dereferencing type-punned pointer will break strict-aliasing rules + * + * Fortunately, GCC with -O1 compiles this nxt_memcpy() + * in the same simple assignment as in the code above. + */ + memcpy(CMSG_DATA(&cmsg.cm), &queue_fd, sizeof(int)); + + res = nxt_unit_sendmsg(ctx, ready_fd, &msg, sizeof(msg), + &cmsg, sizeof(cmsg)); if (res != sizeof(msg)) { return NXT_UNIT_ERROR; } @@ -680,41 +886,56 @@ nxt_unit_ready(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, } -int -nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, - void *buf, size_t buf_size, void *oob, size_t oob_size) +static int +nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) { - int rc; - pid_t pid; - struct cmsghdr *cm; - nxt_port_msg_t *port_msg; - nxt_unit_impl_t *lib; - nxt_unit_recv_msg_t recv_msg; - nxt_unit_callbacks_t *cb; + int rc; + pid_t pid; + struct cmsghdr *cm; + nxt_port_msg_t *port_msg; + nxt_unit_impl_t *lib; + nxt_unit_recv_msg_t recv_msg; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - rc = NXT_UNIT_ERROR; - recv_msg.fd = -1; - recv_msg.process = NULL; - port_msg = buf; - cm = oob; - - if (oob_size >= CMSG_SPACE(sizeof(int)) - && cm->cmsg_len == CMSG_LEN(sizeof(int)) - && cm->cmsg_level == SOL_SOCKET + recv_msg.fd[0] = -1; + recv_msg.fd[1] = -1; + port_msg = (nxt_port_msg_t *) rbuf->buf; + cm = (struct cmsghdr *) rbuf->oob; + + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_RIGHTS) { - memcpy(&recv_msg.fd, CMSG_DATA(cm), sizeof(int)); + if (cm->cmsg_len == CMSG_LEN(sizeof(int))) { + memcpy(recv_msg.fd, CMSG_DATA(cm), sizeof(int)); + } + + if (cm->cmsg_len == CMSG_LEN(sizeof(int) * 2)) { + memcpy(recv_msg.fd, CMSG_DATA(cm), sizeof(int) * 2); + } } recv_msg.incoming_buf = NULL; - if (nxt_slow_path(buf_size < sizeof(nxt_port_msg_t))) { - nxt_unit_warn(ctx, "message too small (%d bytes)", (int) buf_size); - goto fail; + if (nxt_slow_path(rbuf->size < (ssize_t) sizeof(nxt_port_msg_t))) { + if (nxt_slow_path(rbuf->size == 0)) { + nxt_unit_debug(ctx, "read port closed"); + + nxt_unit_quit(ctx); + rc = NXT_UNIT_OK; + goto done; + } + + nxt_unit_alert(ctx, "message too small (%d bytes)", (int) rbuf->size); + + rc = NXT_UNIT_ERROR; + goto done; } + nxt_unit_debug(ctx, "#%"PRIu32": process message %d fd[0] %d fd[1] %d", + port_msg->stream, (int) port_msg->type, + recv_msg.fd[0], recv_msg.fd[1]); + recv_msg.stream = port_msg->stream; recv_msg.pid = port_msg->pid; recv_msg.reply_port = port_msg->reply_port; @@ -722,41 +943,42 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, recv_msg.mmap = port_msg->mmap; recv_msg.start = port_msg + 1; - recv_msg.size = buf_size - sizeof(nxt_port_msg_t); + recv_msg.size = rbuf->size - sizeof(nxt_port_msg_t); if (nxt_slow_path(port_msg->type >= NXT_PORT_MSG_MAX)) { - nxt_unit_warn(ctx, "#%"PRIu32": unknown message type (%d)", - port_msg->stream, (int) port_msg->type); - goto fail; - } - - if (port_msg->tracking && nxt_unit_tracking_read(ctx, &recv_msg) == 0) { - rc = NXT_UNIT_OK; - - goto fail; + nxt_unit_alert(ctx, "#%"PRIu32": unknown message type (%d)", + port_msg->stream, (int) port_msg->type); + rc = NXT_UNIT_ERROR; + goto done; } /* Fragmentation is unsupported. */ if (nxt_slow_path(port_msg->nf != 0 || port_msg->mf != 0)) { - nxt_unit_warn(ctx, "#%"PRIu32": fragmented message type (%d)", - port_msg->stream, (int) port_msg->type); - goto fail; + nxt_unit_alert(ctx, "#%"PRIu32": fragmented message type (%d)", + port_msg->stream, (int) port_msg->type); + rc = NXT_UNIT_ERROR; + goto done; } if (port_msg->mmap) { - if (nxt_unit_mmap_read(ctx, &recv_msg) != NXT_UNIT_OK) { - goto fail; + rc = nxt_unit_mmap_read(ctx, &recv_msg, rbuf); + + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + if (rc == NXT_UNIT_AGAIN) { + recv_msg.fd[0] = -1; + recv_msg.fd[1] = -1; + } + + goto done; } } - cb = &lib->callbacks; - switch (port_msg->type) { case _NXT_PORT_MSG_QUIT: nxt_unit_debug(ctx, "#%"PRIu32": quit", port_msg->stream); - cb->quit(ctx); + nxt_unit_quit(ctx); rc = NXT_UNIT_OK; break; @@ -766,45 +988,52 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, case _NXT_PORT_MSG_CHANGE_FILE: nxt_unit_debug(ctx, "#%"PRIu32": change_file: fd %d", - port_msg->stream, recv_msg.fd); + port_msg->stream, recv_msg.fd[0]); - if (dup2(recv_msg.fd, lib->log_fd) == -1) { + if (dup2(recv_msg.fd[0], lib->log_fd) == -1) { nxt_unit_alert(ctx, "#%"PRIu32": dup2(%d, %d) failed: %s (%d)", - port_msg->stream, recv_msg.fd, lib->log_fd, + port_msg->stream, recv_msg.fd[0], lib->log_fd, strerror(errno), errno); - goto fail; + rc = NXT_UNIT_ERROR; + goto done; } rc = NXT_UNIT_OK; break; case _NXT_PORT_MSG_MMAP: - if (nxt_slow_path(recv_msg.fd < 0)) { + if (nxt_slow_path(recv_msg.fd[0] < 0)) { nxt_unit_alert(ctx, "#%"PRIu32": invalid fd %d for mmap", - port_msg->stream, recv_msg.fd); + port_msg->stream, recv_msg.fd[0]); - goto fail; + rc = NXT_UNIT_ERROR; + goto done; } - rc = nxt_unit_incoming_mmap(ctx, port_msg->pid, recv_msg.fd); + rc = nxt_unit_incoming_mmap(ctx, port_msg->pid, recv_msg.fd[0]); break; case _NXT_PORT_MSG_REQ_HEADERS: rc = nxt_unit_process_req_headers(ctx, &recv_msg); break; + case _NXT_PORT_MSG_REQ_BODY: + rc = nxt_unit_process_req_body(ctx, &recv_msg); + break; + case _NXT_PORT_MSG_WEBSOCKET: rc = nxt_unit_process_websocket(ctx, &recv_msg); break; case _NXT_PORT_MSG_REMOVE_PID: if (nxt_slow_path(recv_msg.size != sizeof(pid))) { - nxt_unit_warn(ctx, "#%"PRIu32": remove_pid: invalid message size " - "(%d != %d)", port_msg->stream, (int) recv_msg.size, - (int) sizeof(pid)); + nxt_unit_alert(ctx, "#%"PRIu32": remove_pid: invalid message size " + "(%d != %d)", port_msg->stream, (int) recv_msg.size, + (int) sizeof(pid)); - goto fail; + rc = NXT_UNIT_ERROR; + goto done; } memcpy(&pid, recv_msg.start, sizeof(pid)); @@ -812,7 +1041,7 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, nxt_unit_debug(ctx, "#%"PRIu32": remove_pid: %d", port_msg->stream, (int) pid); - cb->remove_pid(ctx, pid); + nxt_unit_remove_pid(lib, pid); rc = NXT_UNIT_OK; break; @@ -825,21 +1054,29 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, nxt_unit_debug(ctx, "#%"PRIu32": ignore message type: %d", port_msg->stream, (int) port_msg->type); - goto fail; + rc = NXT_UNIT_ERROR; + goto done; } -fail: +done: - if (recv_msg.fd != -1) { - close(recv_msg.fd); + if (recv_msg.fd[0] != -1) { + nxt_unit_close(recv_msg.fd[0]); + } + + if (recv_msg.fd[1] != -1) { + nxt_unit_close(recv_msg.fd[1]); } while (recv_msg.incoming_buf != NULL) { nxt_unit_mmap_buf_free(recv_msg.incoming_buf); } - if (recv_msg.process != NULL) { - nxt_unit_process_use(ctx, recv_msg.process, -1); + if (nxt_fast_path(rc != NXT_UNIT_AGAIN)) { +#if (NXT_DEBUG) + memset(rbuf->buf, 0xAC, rbuf->size); +#endif + nxt_unit_read_buf_release(ctx, rbuf); } return rc; @@ -849,9 +1086,9 @@ fail: static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) { - int nb; + void *mem; nxt_unit_impl_t *lib; - nxt_unit_port_t new_port; + nxt_unit_port_t new_port, *port; nxt_port_msg_new_port_t *new_port_msg; if (nxt_slow_path(recv_msg->size != sizeof(nxt_port_msg_new_port_t))) { @@ -862,48 +1099,80 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) return NXT_UNIT_ERROR; } - if (nxt_slow_path(recv_msg->fd < 0)) { + if (nxt_slow_path(recv_msg->fd[0] < 0)) { nxt_unit_alert(ctx, "#%"PRIu32": invalid fd %d for new port", - recv_msg->stream, recv_msg->fd); + recv_msg->stream, recv_msg->fd[0]); return NXT_UNIT_ERROR; } new_port_msg = recv_msg->start; - nxt_unit_debug(ctx, "#%"PRIu32": new_port: %d,%d fd %d", + nxt_unit_debug(ctx, "#%"PRIu32": new_port: port{%d,%d} fd[0] %d fd[1] %d", recv_msg->stream, (int) new_port_msg->pid, - (int) new_port_msg->id, recv_msg->fd); + (int) new_port_msg->id, recv_msg->fd[0], recv_msg->fd[1]); - nb = 0; + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - if (nxt_slow_path(ioctl(recv_msg->fd, FIONBIO, &nb) == -1)) { - nxt_unit_alert(ctx, "#%"PRIu32": new_port: ioctl(%d, FIONBIO, 0) " - "failed: %s (%d)", - recv_msg->stream, recv_msg->fd, strerror(errno), errno); + if (new_port_msg->id == (nxt_port_id_t) -1) { + nxt_unit_port_id_init(&new_port.id, lib->pid, new_port_msg->id); - return NXT_UNIT_ERROR; + new_port.in_fd = recv_msg->fd[0]; + new_port.out_fd = -1; + + mem = mmap(NULL, sizeof(nxt_app_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED, recv_msg->fd[1], 0); + + } else { + if (nxt_slow_path(nxt_unit_fd_blocking(recv_msg->fd[0]) + != NXT_UNIT_OK)) + { + return NXT_UNIT_ERROR; + } + + nxt_unit_port_id_init(&new_port.id, new_port_msg->pid, + new_port_msg->id); + + new_port.in_fd = -1; + new_port.out_fd = recv_msg->fd[0]; + + mem = mmap(NULL, sizeof(nxt_port_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED, recv_msg->fd[1], 0); } - nxt_unit_port_id_init(&new_port.id, new_port_msg->pid, - new_port_msg->id); + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", recv_msg->fd[1], + strerror(errno), errno); + + return NXT_UNIT_ERROR; + } - new_port.in_fd = -1; - new_port.out_fd = recv_msg->fd; new_port.data = NULL; - recv_msg->fd = -1; + recv_msg->fd[0] = -1; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + port = nxt_unit_add_port(ctx, &new_port, mem); + if (nxt_slow_path(port == NULL)) { + return NXT_UNIT_ERROR; + } + + if (new_port_msg->id == (nxt_port_id_t) -1) { + lib->shared_port = port; + + } else { + nxt_unit_port_release(port); + } - return lib->callbacks.add_port(ctx, &new_port); + return NXT_UNIT_OK; } static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) { + int res; nxt_unit_impl_t *lib; + nxt_unit_port_id_t port_id; nxt_unit_request_t *r; nxt_unit_mmap_buf_t *b; nxt_unit_request_info_t *req; @@ -934,9 +1203,6 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) req = &req_impl->req; - nxt_unit_port_id_init(&req->response_port, recv_msg->pid, - recv_msg->reply_port); - req->request = recv_msg->start; b = recv_msg->incoming_buf; @@ -952,14 +1218,6 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) req->content_buf = req->request_buf; req->content_buf->free = nxt_unit_sptr_get(&r->preread_content); - /* "Move" process reference to req_impl. */ - req_impl->process = nxt_unit_msg_get_process(ctx, recv_msg); - if (nxt_slow_path(req_impl->process == NULL)) { - return NXT_UNIT_ERROR; - } - - recv_msg->process = NULL; - req_impl->stream = recv_msg->stream; req_impl->outgoing_buf = NULL; @@ -973,12 +1231,13 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) req_impl->incoming_buf->prev = &req_impl->incoming_buf; recv_msg->incoming_buf = NULL; - req->content_fd = recv_msg->fd; - recv_msg->fd = -1; + req->content_fd = recv_msg->fd[0]; + recv_msg->fd[0] = -1; req->response_max_fields = 0; req_impl->state = NXT_UNIT_RS_START; req_impl->websocket = 0; + req_impl->in_hash = 0; nxt_unit_debug(ctx, "#%"PRIu32": %.*s %.*s (%d)", recv_msg->stream, (int) r->method_length, @@ -987,9 +1246,247 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) (char *) nxt_unit_sptr_get(&r->target), (int) r->content_length); + nxt_unit_port_id_init(&port_id, recv_msg->pid, recv_msg->reply_port); + + res = nxt_unit_request_check_response_port(req, &port_id); + if (nxt_slow_path(res == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } + + if (nxt_fast_path(res == NXT_UNIT_OK)) { + res = nxt_unit_send_req_headers_ack(req); + if (nxt_slow_path(res == NXT_UNIT_ERROR)) { + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + return NXT_UNIT_ERROR; + } + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + if (req->content_length + > (uint64_t) (req->content_buf->end - req->content_buf->free)) + { + res = nxt_unit_request_hash_add(ctx, req); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + nxt_unit_req_warn(req, "failed to add request to hash"); + + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + return NXT_UNIT_ERROR; + } + + /* + * If application have separate data handler, we may start + * request processing and process data when it is arrived. + */ + if (lib->callbacks.data_handler == NULL) { + return NXT_UNIT_OK; + } + } + + lib->callbacks.request_handler(req); + } + + return NXT_UNIT_OK; +} + + +static int +nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) +{ + uint64_t l; + nxt_unit_impl_t *lib; + nxt_unit_mmap_buf_t *b; + nxt_unit_request_info_t *req; + + req = nxt_unit_request_hash_find(ctx, recv_msg->stream, recv_msg->last); + if (req == NULL) { + return NXT_UNIT_OK; + } + + l = req->content_buf->end - req->content_buf->free; + + for (b = recv_msg->incoming_buf; b != NULL; b = b->next) { + b->req = req; + l += b->buf.end - b->buf.free; + } + + if (recv_msg->incoming_buf != NULL) { + b = nxt_container_of(req->content_buf, nxt_unit_mmap_buf_t, buf); + + /* "Move" incoming buffer list to req_impl. */ + nxt_unit_mmap_buf_insert_tail(&b->next, recv_msg->incoming_buf); + recv_msg->incoming_buf = NULL; + } + + req->content_fd = recv_msg->fd[0]; + recv_msg->fd[0] = -1; + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - lib->callbacks.request_handler(req); + if (lib->callbacks.data_handler != NULL) { + lib->callbacks.data_handler(req); + + return NXT_UNIT_OK; + } + + if (req->content_fd != -1 || l == req->content_length) { + lib->callbacks.request_handler(req); + } + + return NXT_UNIT_OK; +} + + +static int +nxt_unit_request_check_response_port(nxt_unit_request_info_t *req, + nxt_unit_port_id_t *port_id) +{ + int res; + nxt_unit_ctx_t *ctx; + nxt_unit_impl_t *lib; + nxt_unit_port_t *port; + nxt_unit_process_t *process; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_port_impl_t *port_impl; + nxt_unit_request_info_impl_t *req_impl; + + ctx = req->ctx; + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + pthread_mutex_lock(&lib->mutex); + + port = nxt_unit_port_hash_find(&lib->ports, port_id, 0); + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + + if (nxt_fast_path(port != NULL)) { + req->response_port = port; + + if (nxt_fast_path(port_impl->ready)) { + pthread_mutex_unlock(&lib->mutex); + + nxt_unit_debug(ctx, "check_response_port: found port{%d,%d}", + (int) port->id.pid, (int) port->id.id); + + return NXT_UNIT_OK; + } + + nxt_unit_debug(ctx, "check_response_port: " + "port{%d,%d} already requested", + (int) port->id.pid, (int) port->id.id); + + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); + + nxt_queue_insert_tail(&port_impl->awaiting_req, + &req_impl->port_wait_link); + + pthread_mutex_unlock(&lib->mutex); + + nxt_atomic_fetch_add(&ctx_impl->wait_items, 1); + + return NXT_UNIT_AGAIN; + } + + port_impl = malloc(sizeof(nxt_unit_port_impl_t)); + if (nxt_slow_path(port_impl == NULL)) { + nxt_unit_alert(ctx, "check_response_port: malloc(%d) failed", + (int) sizeof(nxt_unit_port_impl_t)); + + pthread_mutex_unlock(&lib->mutex); + + return NXT_UNIT_ERROR; + } + + port = &port_impl->port; + + port->id = *port_id; + port->in_fd = -1; + port->out_fd = -1; + port->data = NULL; + + res = nxt_unit_port_hash_add(&lib->ports, port); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + nxt_unit_alert(ctx, "check_response_port: %d,%d hash_add failed", + port->id.pid, port->id.id); + + pthread_mutex_unlock(&lib->mutex); + + free(port); + + return NXT_UNIT_ERROR; + } + + process = nxt_unit_process_find(lib, port_id->pid, 0); + if (nxt_slow_path(process == NULL)) { + nxt_unit_alert(ctx, "check_response_port: process %d not found", + port->id.pid); + + nxt_unit_port_hash_find(&lib->ports, port_id, 1); + + pthread_mutex_unlock(&lib->mutex); + + free(port); + + return NXT_UNIT_ERROR; + } + + nxt_queue_insert_tail(&process->ports, &port_impl->link); + + port_impl->process = process; + port_impl->queue = NULL; + port_impl->from_socket = 0; + port_impl->socket_rbuf = NULL; + + nxt_queue_init(&port_impl->awaiting_req); + + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); + + nxt_queue_insert_tail(&port_impl->awaiting_req, &req_impl->port_wait_link); + + port_impl->use_count = 2; + port_impl->ready = 0; + + req->response_port = port; + + pthread_mutex_unlock(&lib->mutex); + + res = nxt_unit_get_port(ctx, port_id); + if (nxt_slow_path(res == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } + + nxt_atomic_fetch_add(&ctx_impl->wait_items, 1); + + return NXT_UNIT_AGAIN; +} + + +static int +nxt_unit_send_req_headers_ack(nxt_unit_request_info_t *req) +{ + ssize_t res; + nxt_port_msg_t msg; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_impl_t *req_impl; + + lib = nxt_container_of(req->ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(req->ctx, nxt_unit_ctx_impl_t, ctx); + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); + + memset(&msg, 0, sizeof(nxt_port_msg_t)); + + msg.stream = req_impl->stream; + msg.pid = lib->pid; + msg.reply_port = ctx_impl->read_port->id.id; + msg.type = _NXT_PORT_MSG_REQ_HEADERS_ACK; + + res = nxt_unit_port_send(req->ctx, req->response_port, + &msg, sizeof(msg), NULL, 0); + if (nxt_slow_path(res != sizeof(msg))) { + return NXT_UNIT_ERROR; + } return NXT_UNIT_OK; } @@ -1001,21 +1498,17 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) size_t hsize; nxt_unit_impl_t *lib; nxt_unit_mmap_buf_t *b; - nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_callbacks_t *cb; nxt_unit_request_info_t *req; nxt_unit_request_info_impl_t *req_impl; nxt_unit_websocket_frame_impl_t *ws_impl; - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - - req_impl = nxt_unit_request_hash_find(&ctx_impl->requests, recv_msg->stream, - recv_msg->last); - if (req_impl == NULL) { + req = nxt_unit_request_hash_find(ctx, recv_msg->stream, recv_msg->last); + if (nxt_slow_path(req == NULL)) { return NXT_UNIT_OK; } - req = &req_impl->req; + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); cb = &lib->callbacks; @@ -1181,12 +1674,12 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) req->response = NULL; req->response_buf = NULL; - if (req_impl->websocket) { - nxt_unit_request_hash_find(&ctx_impl->requests, req_impl->stream, 1); - - req_impl->websocket = 0; + if (req_impl->in_hash) { + nxt_unit_request_hash_find(req->ctx, req_impl->stream, 1); } + req_impl->websocket = 0; + while (req_impl->outgoing_buf != NULL) { nxt_unit_mmap_buf_free(req_impl->outgoing_buf); } @@ -1196,19 +1689,15 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) } if (req->content_fd != -1) { - close(req->content_fd); + nxt_unit_close(req->content_fd); req->content_fd = -1; } - /* - * Process release should go after buffers release to guarantee mmap - * existence. - */ - if (req_impl->process != NULL) { - nxt_unit_process_use(req->ctx, req_impl->process, -1); + if (req->response_port != NULL) { + nxt_unit_port_release(req->response_port); - req_impl->process = NULL; + req->response_port = NULL; } pthread_mutex_lock(&ctx_impl->mutex); @@ -1729,7 +2218,7 @@ nxt_unit_response_send(nxt_unit_request_info_t *req) mmap_buf = nxt_container_of(req->response_buf, nxt_unit_mmap_buf_t, buf); - rc = nxt_unit_mmap_buf_send(req->ctx, req_impl->stream, mmap_buf, 0); + rc = nxt_unit_mmap_buf_send(req, mmap_buf, 0); if (nxt_fast_path(rc == NXT_UNIT_OK)) { req->response = NULL; req->response_buf = NULL; @@ -1782,8 +2271,8 @@ nxt_unit_response_buf_alloc(nxt_unit_request_info_t *req, uint32_t size) nxt_unit_mmap_buf_insert_tail(&req_impl->outgoing_buf, mmap_buf); - rc = nxt_unit_get_outgoing_buf(req->ctx, req_impl->process, - &req->response_port, size, size, mmap_buf, + rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, + size, size, mmap_buf, NULL); if (nxt_slow_path(rc != NXT_UNIT_OK)) { nxt_unit_mmap_buf_release(mmap_buf); @@ -1795,32 +2284,6 @@ nxt_unit_response_buf_alloc(nxt_unit_request_info_t *req, uint32_t size) } -static nxt_unit_process_t * -nxt_unit_msg_get_process(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) -{ - nxt_unit_impl_t *lib; - - if (recv_msg->process != NULL) { - return recv_msg->process; - } - - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - - pthread_mutex_lock(&lib->mutex); - - recv_msg->process = nxt_unit_process_find(ctx, recv_msg->pid, 0); - - pthread_mutex_unlock(&lib->mutex); - - if (recv_msg->process == NULL) { - nxt_unit_warn(ctx, "#%"PRIu32": process %d not found", - recv_msg->stream, (int) recv_msg->pid); - } - - return recv_msg->process; -} - - static nxt_unit_mmap_buf_t * nxt_unit_mmap_buf_get(nxt_unit_ctx_t *ctx) { @@ -1869,15 +2332,6 @@ nxt_unit_mmap_buf_release(nxt_unit_mmap_buf_t *mmap_buf) } -typedef struct { - size_t len; - const char *str; -} nxt_unit_str_t; - - -#define nxt_unit_str(str) { nxt_length(str), str } - - int nxt_unit_request_is_websocket_handshake(nxt_unit_request_info_t *req) { @@ -1889,7 +2343,6 @@ int nxt_unit_response_upgrade(nxt_unit_request_info_t *req) { int rc; - nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_request_info_impl_t *req_impl; req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); @@ -1912,9 +2365,7 @@ nxt_unit_response_upgrade(nxt_unit_request_info_t *req) return NXT_UNIT_ERROR; } - ctx_impl = nxt_container_of(req->ctx, nxt_unit_ctx_impl_t, ctx); - - rc = nxt_unit_request_hash_add(&ctx_impl->requests, req_impl); + rc = nxt_unit_request_hash_add(req->ctx, req); if (nxt_slow_path(rc != NXT_UNIT_OK)) { nxt_unit_req_warn(req, "upgrade: failed to add request to hash"); @@ -1980,7 +2431,7 @@ nxt_unit_buf_send(nxt_unit_buf_t *buf) } if (nxt_fast_path(buf->free > buf->start)) { - rc = nxt_unit_mmap_buf_send(req->ctx, req_impl->stream, mmap_buf, 0); + rc = nxt_unit_mmap_buf_send(req, mmap_buf, 0); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return rc; } @@ -1995,17 +2446,15 @@ nxt_unit_buf_send(nxt_unit_buf_t *buf) static void nxt_unit_buf_send_done(nxt_unit_buf_t *buf) { - int rc; - nxt_unit_mmap_buf_t *mmap_buf; - nxt_unit_request_info_t *req; - nxt_unit_request_info_impl_t *req_impl; + int rc; + nxt_unit_mmap_buf_t *mmap_buf; + nxt_unit_request_info_t *req; mmap_buf = nxt_container_of(buf, nxt_unit_mmap_buf_t, buf); req = mmap_buf->req; - req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); - rc = nxt_unit_mmap_buf_send(req->ctx, req_impl->stream, mmap_buf, 1); + rc = nxt_unit_mmap_buf_send(req, mmap_buf, 1); if (nxt_slow_path(rc == NXT_UNIT_OK)) { nxt_unit_mmap_buf_free(mmap_buf); @@ -2018,7 +2467,7 @@ nxt_unit_buf_send_done(nxt_unit_buf_t *buf) static int -nxt_unit_mmap_buf_send(nxt_unit_ctx_t *ctx, uint32_t stream, +nxt_unit_mmap_buf_send(nxt_unit_request_info_t *req, nxt_unit_mmap_buf_t *mmap_buf, int last) { struct { @@ -2026,22 +2475,24 @@ nxt_unit_mmap_buf_send(nxt_unit_ctx_t *ctx, uint32_t stream, nxt_port_mmap_msg_t mmap_msg; } m; - int rc; - u_char *last_used, *first_free; - ssize_t res; - nxt_chunk_id_t first_free_chunk; - nxt_unit_buf_t *buf; - nxt_unit_impl_t *lib; - nxt_port_mmap_header_t *hdr; + int rc; + u_char *last_used, *first_free; + ssize_t res; + nxt_chunk_id_t first_free_chunk; + nxt_unit_buf_t *buf; + nxt_unit_impl_t *lib; + nxt_port_mmap_header_t *hdr; + nxt_unit_request_info_impl_t *req_impl; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + lib = nxt_container_of(req->ctx->unit, nxt_unit_impl_t, unit); + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); buf = &mmap_buf->buf; hdr = mmap_buf->hdr; m.mmap_msg.size = buf->free - buf->start; - m.msg.stream = stream; + m.msg.stream = req_impl->stream; m.msg.pid = lib->pid; m.msg.reply_port = 0; m.msg.type = _NXT_PORT_MSG_DATA; @@ -2058,14 +2509,14 @@ nxt_unit_mmap_buf_send(nxt_unit_ctx_t *ctx, uint32_t stream, m.mmap_msg.chunk_id = nxt_port_mmap_chunk_id(hdr, (u_char *) buf->start); - nxt_unit_debug(ctx, "#%"PRIu32": send mmap: (%d,%d,%d)", - stream, + nxt_unit_debug(req->ctx, "#%"PRIu32": send mmap: (%d,%d,%d)", + req_impl->stream, (int) m.mmap_msg.mmap_id, (int) m.mmap_msg.chunk_id, (int) m.mmap_msg.size); - res = lib->callbacks.port_send(ctx, &mmap_buf->port_id, &m, sizeof(m), - NULL, 0); + res = nxt_unit_port_send(req->ctx, req->response_port, &m, sizeof(m), + NULL, 0); if (nxt_slow_path(res != sizeof(m))) { goto free_buf; } @@ -2091,33 +2542,34 @@ nxt_unit_mmap_buf_send(nxt_unit_ctx_t *ctx, uint32_t stream, mmap_buf->hdr = NULL; } - nxt_atomic_fetch_add(&mmap_buf->process->outgoing.allocated_chunks, + nxt_atomic_fetch_add(&lib->outgoing.allocated_chunks, (int) m.mmap_msg.chunk_id - (int) first_free_chunk); - nxt_unit_debug(ctx, "process %d allocated_chunks %d", - mmap_buf->process->pid, - (int) mmap_buf->process->outgoing.allocated_chunks); + nxt_unit_debug(req->ctx, "allocated_chunks %d", + (int) lib->outgoing.allocated_chunks); } else { if (nxt_slow_path(mmap_buf->plain_ptr == NULL || mmap_buf->plain_ptr > buf->start - sizeof(m.msg))) { - nxt_unit_warn(ctx, "#%"PRIu32": failed to send plain memory buffer" - ": no space reserved for message header", stream); + nxt_unit_alert(req->ctx, + "#%"PRIu32": failed to send plain memory buffer" + ": no space reserved for message header", + req_impl->stream); goto free_buf; } memcpy(buf->start - sizeof(m.msg), &m.msg, sizeof(m.msg)); - nxt_unit_debug(ctx, "#%"PRIu32": send plain: %d", - stream, + nxt_unit_debug(req->ctx, "#%"PRIu32": send plain: %d", + req_impl->stream, (int) (sizeof(m.msg) + m.mmap_msg.size)); - res = lib->callbacks.port_send(ctx, &mmap_buf->port_id, - buf->start - sizeof(m.msg), - m.mmap_msg.size + sizeof(m.msg), - NULL, 0); + res = nxt_unit_port_send(req->ctx, req->response_port, + buf->start - sizeof(m.msg), + m.mmap_msg.size + sizeof(m.msg), + NULL, 0); if (nxt_slow_path(res != (ssize_t) (m.mmap_msg.size + sizeof(m.msg)))) { goto free_buf; } @@ -2154,7 +2606,6 @@ nxt_unit_free_outgoing_buf(nxt_unit_mmap_buf_t *mmap_buf) { if (mmap_buf->hdr != NULL) { nxt_unit_mmap_release(&mmap_buf->ctx_impl->ctx, - mmap_buf->process, mmap_buf->hdr, mmap_buf->buf.start, mmap_buf->buf.end - mmap_buf->buf.start); @@ -2175,33 +2626,43 @@ static nxt_unit_read_buf_t * nxt_unit_read_buf_get(nxt_unit_ctx_t *ctx) { nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_read_buf_t *rbuf; ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); pthread_mutex_lock(&ctx_impl->mutex); - return nxt_unit_read_buf_get_impl(ctx_impl); + rbuf = nxt_unit_read_buf_get_impl(ctx_impl); + + pthread_mutex_unlock(&ctx_impl->mutex); + + memset(rbuf->oob, 0, sizeof(struct cmsghdr)); + + return rbuf; } static nxt_unit_read_buf_t * nxt_unit_read_buf_get_impl(nxt_unit_ctx_impl_t *ctx_impl) { + nxt_queue_link_t *link; nxt_unit_read_buf_t *rbuf; - if (ctx_impl->free_read_buf != NULL) { - rbuf = ctx_impl->free_read_buf; - ctx_impl->free_read_buf = rbuf->next; + if (!nxt_queue_is_empty(&ctx_impl->free_rbuf)) { + link = nxt_queue_first(&ctx_impl->free_rbuf); + nxt_queue_remove(link); - pthread_mutex_unlock(&ctx_impl->mutex); + rbuf = nxt_container_of(link, nxt_unit_read_buf_t, link); return rbuf; } - pthread_mutex_unlock(&ctx_impl->mutex); - rbuf = malloc(sizeof(nxt_unit_read_buf_t)); + if (nxt_fast_path(rbuf != NULL)) { + rbuf->ctx_impl = ctx_impl; + } + return rbuf; } @@ -2216,8 +2677,7 @@ nxt_unit_read_buf_release(nxt_unit_ctx_t *ctx, pthread_mutex_lock(&ctx_impl->mutex); - rbuf->next = ctx_impl->free_read_buf; - ctx_impl->free_read_buf = rbuf; + nxt_queue_insert_head(&ctx_impl->free_rbuf, &rbuf->link); pthread_mutex_unlock(&ctx_impl->mutex); } @@ -2276,13 +2736,15 @@ nxt_unit_response_write_nb(nxt_unit_request_info_t *req, const void *start, nxt_unit_request_info_impl_t *req_impl; char local_buf[NXT_UNIT_LOCAL_BUF_SIZE]; + nxt_unit_req_debug(req, "write: %d", (int) size); + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); part_start = start; sent = 0; if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) { - nxt_unit_req_warn(req, "write: response not initialized yet"); + nxt_unit_req_alert(req, "write: response not initialized yet"); return -NXT_UNIT_ERROR; } @@ -2314,8 +2776,7 @@ nxt_unit_response_write_nb(nxt_unit_request_info_t *req, const void *start, min_part_size = nxt_min(min_size, part_size); min_part_size = nxt_min(min_part_size, PORT_MMAP_CHUNK_SIZE); - rc = nxt_unit_get_outgoing_buf(req->ctx, req_impl->process, - &req->response_port, part_size, + rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, part_size, min_part_size, &mmap_buf, local_buf); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return -rc; @@ -2330,7 +2791,7 @@ nxt_unit_response_write_nb(nxt_unit_request_info_t *req, const void *start, mmap_buf.buf.free = nxt_cpymem(mmap_buf.buf.free, part_start, part_size); - rc = nxt_unit_mmap_buf_send(req->ctx, req_impl->stream, &mmap_buf, 0); + rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return -rc; } @@ -2360,8 +2821,14 @@ nxt_unit_response_write_cb(nxt_unit_request_info_t *req, req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); + if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) { + nxt_unit_req_alert(req, "write: response not initialized yet"); + + return NXT_UNIT_ERROR; + } + /* Check if response is not send yet. */ - if (nxt_slow_path(req->response_buf)) { + if (nxt_slow_path(req->response_buf != NULL)) { /* Enable content in headers buf. */ rc = nxt_unit_response_add_content(req, "", 0); @@ -2408,8 +2875,7 @@ nxt_unit_response_write_cb(nxt_unit_request_info_t *req, buf_size = nxt_min(read_info->buf_size, PORT_MMAP_DATA_SIZE); - rc = nxt_unit_get_outgoing_buf(req->ctx, req_impl->process, - &req->response_port, + rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, buf_size, buf_size, &mmap_buf, local_buf); if (nxt_slow_path(rc != NXT_UNIT_OK)) { @@ -2431,7 +2897,7 @@ nxt_unit_response_write_cb(nxt_unit_request_info_t *req, buf->free += n; } - rc = nxt_unit_mmap_buf_send(req->ctx, req_impl->stream, &mmap_buf, 0); + rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0); if (nxt_slow_path(rc != NXT_UNIT_OK)) { nxt_unit_req_error(req, "Failed to send content"); @@ -2451,9 +2917,11 @@ nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size) buf_res = nxt_unit_buf_read(&req->content_buf, &req->content_length, dst, size); + nxt_unit_req_debug(req, "read: %d", (int) buf_res); + if (buf_res < (ssize_t) size && req->content_fd != -1) { res = read(req->content_fd, dst, size); - if (res < 0) { + if (nxt_slow_path(res < 0)) { nxt_unit_req_alert(req, "failed to read content: %s (%d)", strerror(errno), errno); @@ -2461,7 +2929,7 @@ nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size) } if (res < (ssize_t) size) { - close(req->content_fd); + nxt_unit_close(req->content_fd); req->content_fd = -1; } @@ -2561,7 +3029,6 @@ nxt_unit_request_preread(nxt_unit_request_info_t *req, size_t size) mmap_buf->buf.start = mmap_buf->free_ptr; mmap_buf->buf.free = mmap_buf->buf.start; mmap_buf->buf.end = mmap_buf->buf.start + size; - mmap_buf->process = NULL; res = read(req->content_fd, mmap_buf->free_ptr, size); if (res < 0) { @@ -2574,7 +3041,7 @@ nxt_unit_request_preread(nxt_unit_request_info_t *req, size_t size) } if (res < (ssize_t) size) { - close(req->content_fd); + nxt_unit_close(req->content_fd); req->content_fd = -1; } @@ -2689,8 +3156,8 @@ skip_response_send: msg.mf = 0; msg.tracking = 0; - (void) lib->callbacks.port_send(req->ctx, &req->response_port, - &msg, sizeof(msg), NULL, 0); + (void) nxt_unit_port_send(req->ctx, req->response_port, + &msg, sizeof(msg), NULL, 0); nxt_unit_request_info_release(req); } @@ -2710,17 +3177,14 @@ int nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode, uint8_t last, const struct iovec *iov, int iovcnt) { - int i, rc; - size_t l, copy; - uint32_t payload_len, buf_size, alloc_size; - const uint8_t *b; - nxt_unit_buf_t *buf; - nxt_unit_mmap_buf_t mmap_buf; - nxt_websocket_header_t *wh; - nxt_unit_request_info_impl_t *req_impl; - char local_buf[NXT_UNIT_LOCAL_BUF_SIZE]; - - req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); + int i, rc; + size_t l, copy; + uint32_t payload_len, buf_size, alloc_size; + const uint8_t *b; + nxt_unit_buf_t *buf; + nxt_unit_mmap_buf_t mmap_buf; + nxt_websocket_header_t *wh; + char local_buf[NXT_UNIT_LOCAL_BUF_SIZE]; payload_len = 0; @@ -2731,8 +3195,7 @@ nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode, buf_size = 10 + payload_len; alloc_size = nxt_min(buf_size, PORT_MMAP_DATA_SIZE); - rc = nxt_unit_get_outgoing_buf(req->ctx, req_impl->process, - &req->response_port, + rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, alloc_size, alloc_size, &mmap_buf, local_buf); if (nxt_slow_path(rc != NXT_UNIT_OK)) { @@ -2766,8 +3229,7 @@ nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode, if (l > 0) { if (nxt_fast_path(buf->free > buf->start)) { - rc = nxt_unit_mmap_buf_send(req->ctx, req_impl->stream, - &mmap_buf, 0); + rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return rc; @@ -2776,8 +3238,7 @@ nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode, alloc_size = nxt_min(buf_size, PORT_MMAP_DATA_SIZE); - rc = nxt_unit_get_outgoing_buf(req->ctx, req_impl->process, - &req->response_port, + rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, alloc_size, alloc_size, &mmap_buf, local_buf); if (nxt_slow_path(rc != NXT_UNIT_OK)) { @@ -2790,8 +3251,7 @@ nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode, } if (buf->free > buf->start) { - rc = nxt_unit_mmap_buf_send(req->ctx, req_impl->stream, - &mmap_buf, 0); + rc = nxt_unit_mmap_buf_send(req, &mmap_buf, 0); } return rc; @@ -2864,8 +3324,8 @@ nxt_unit_websocket_done(nxt_unit_websocket_frame_t *ws) static nxt_port_mmap_header_t * -nxt_unit_mmap_get(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, - nxt_unit_port_id_t *port_id, nxt_chunk_id_t *c, int *n, int min_n) +nxt_unit_mmap_get(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + nxt_chunk_id_t *c, int *n, int min_n) { int res, nchunks, i; uint32_t outgoing_size; @@ -2875,18 +3335,18 @@ nxt_unit_mmap_get(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - pthread_mutex_lock(&process->outgoing.mutex); + pthread_mutex_lock(&lib->outgoing.mutex); retry: - outgoing_size = process->outgoing.size; + outgoing_size = lib->outgoing.size; - mm_end = process->outgoing.elts + outgoing_size; + mm_end = lib->outgoing.elts + outgoing_size; - for (mm = process->outgoing.elts; mm < mm_end; mm++) { + for (mm = lib->outgoing.elts; mm < mm_end; mm++) { hdr = mm->hdr; - if (hdr->sent_over != 0xFFFFu && hdr->sent_over != port_id->id) { + if (hdr->sent_over != 0xFFFFu && hdr->sent_over != port->id.id) { continue; } @@ -2930,13 +3390,13 @@ retry: if (outgoing_size >= lib->shm_mmap_limit) { /* Cannot allocate more shared memory. */ - pthread_mutex_unlock(&process->outgoing.mutex); + pthread_mutex_unlock(&lib->outgoing.mutex); if (min_n == 0) { *n = 0; } - if (nxt_slow_path(process->outgoing.allocated_chunks + min_n + if (nxt_slow_path(lib->outgoing.allocated_chunks + min_n >= lib->shm_mmap_limit * PORT_MMAP_CHUNK_COUNT)) { /* Memory allocated by application, but not send to router. */ @@ -2945,7 +3405,7 @@ retry: /* Notify router about OOSM condition. */ - res = nxt_unit_send_oosm(ctx, port_id); + res = nxt_unit_send_oosm(ctx, port); if (nxt_slow_path(res != NXT_UNIT_OK)) { return NULL; } @@ -2965,30 +3425,29 @@ retry: nxt_unit_debug(ctx, "oosm: retry"); - pthread_mutex_lock(&process->outgoing.mutex); + pthread_mutex_lock(&lib->outgoing.mutex); goto retry; } *c = 0; - hdr = nxt_unit_new_mmap(ctx, process, port_id, *n); + hdr = nxt_unit_new_mmap(ctx, port, *n); unlock: - nxt_atomic_fetch_add(&process->outgoing.allocated_chunks, *n); + nxt_atomic_fetch_add(&lib->outgoing.allocated_chunks, *n); - nxt_unit_debug(ctx, "process %d allocated_chunks %d", - process->pid, - (int) process->outgoing.allocated_chunks); + nxt_unit_debug(ctx, "allocated_chunks %d", + (int) lib->outgoing.allocated_chunks); - pthread_mutex_unlock(&process->outgoing.mutex); + pthread_mutex_unlock(&lib->outgoing.mutex); return hdr; } static int -nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id) +nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { ssize_t res; nxt_port_msg_t msg; @@ -3006,7 +3465,7 @@ nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id) msg.mf = 0; msg.tracking = 0; - res = lib->callbacks.port_send(ctx, port_id, &msg, sizeof(msg), NULL, 0); + res = nxt_unit_port_send(ctx, lib->router_port, &msg, sizeof(msg), NULL, 0); if (nxt_slow_path(res != sizeof(msg))) { return NXT_UNIT_ERROR; } @@ -3018,7 +3477,7 @@ nxt_unit_send_oosm(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id) static int nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx) { - nxt_port_msg_t *port_msg; + int res; nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_read_buf_t *rbuf; @@ -3030,31 +3489,25 @@ nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx) return NXT_UNIT_ERROR; } - nxt_unit_read_buf(ctx, rbuf); - - if (nxt_slow_path(rbuf->size < (ssize_t) sizeof(nxt_port_msg_t))) { + res = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); + if (res == NXT_UNIT_ERROR) { nxt_unit_read_buf_release(ctx, rbuf); return NXT_UNIT_ERROR; } - port_msg = (nxt_port_msg_t *) rbuf->buf; - - if (port_msg->type == _NXT_PORT_MSG_SHM_ACK) { + if (nxt_unit_is_shm_ack(rbuf)) { nxt_unit_read_buf_release(ctx, rbuf); - break; } pthread_mutex_lock(&ctx_impl->mutex); - *ctx_impl->pending_read_tail = rbuf; - ctx_impl->pending_read_tail = &rbuf->next; - rbuf->next = NULL; + nxt_queue_insert_tail(&ctx_impl->pending_rbuf, &rbuf->link); pthread_mutex_unlock(&ctx_impl->mutex); - if (port_msg->type == _NXT_PORT_MSG_QUIT) { + if (nxt_unit_is_quit(rbuf)) { nxt_unit_debug(ctx, "oosm: quit received"); return NXT_UNIT_ERROR; @@ -3068,7 +3521,12 @@ nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx) static nxt_unit_mmap_t * nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) { - uint32_t cap; + uint32_t cap, n; + nxt_unit_mmap_t *e; + + if (nxt_fast_path(mmaps->size > i)) { + return mmaps->elts + i; + } cap = mmaps->cap; @@ -3088,13 +3546,19 @@ nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) if (cap != mmaps->cap) { - mmaps->elts = realloc(mmaps->elts, cap * sizeof(*mmaps->elts)); - if (nxt_slow_path(mmaps->elts == NULL)) { + e = realloc(mmaps->elts, cap * sizeof(nxt_unit_mmap_t)); + if (nxt_slow_path(e == NULL)) { return NULL; } - memset(mmaps->elts + mmaps->cap, 0, - sizeof(*mmaps->elts) * (cap - mmaps->cap)); + mmaps->elts = e; + + for (n = mmaps->cap; n < cap; n++) { + e = mmaps->elts + n; + + e->hdr = NULL; + nxt_queue_init(&e->awaiting_rbuf); + } mmaps->cap = cap; } @@ -3108,27 +3572,100 @@ nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) static nxt_port_mmap_header_t * -nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, - nxt_unit_port_id_t *port_id, int n) +nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n) { int i, fd, rc; void *mem; - char name[64]; nxt_unit_mmap_t *mm; nxt_unit_impl_t *lib; nxt_port_mmap_header_t *hdr; - lib = process->lib; + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - mm = nxt_unit_mmap_at(&process->outgoing, process->outgoing.size); + mm = nxt_unit_mmap_at(&lib->outgoing, lib->outgoing.size); if (nxt_slow_path(mm == NULL)) { - nxt_unit_warn(ctx, "failed to add mmap to outgoing array"); + nxt_unit_alert(ctx, "failed to add mmap to outgoing array"); return NULL; } + fd = nxt_unit_shm_open(ctx, PORT_MMAP_SIZE); + if (nxt_slow_path(fd == -1)) { + goto remove_fail; + } + + mem = mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", fd, + strerror(errno), errno); + + nxt_unit_close(fd); + + goto remove_fail; + } + + mm->hdr = mem; + hdr = mem; + + memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); + memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); + + hdr->id = lib->outgoing.size - 1; + hdr->src_pid = lib->pid; + hdr->dst_pid = port->id.pid; + hdr->sent_over = port->id.id; + + /* Mark first n chunk(s) as busy */ + for (i = 0; i < n; i++) { + nxt_port_mmap_set_chunk_busy(hdr->free_map, i); + } + + /* Mark as busy chunk followed the last available chunk. */ + nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); + nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); + + pthread_mutex_unlock(&lib->outgoing.mutex); + + rc = nxt_unit_send_mmap(ctx, port, fd); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + munmap(mem, PORT_MMAP_SIZE); + hdr = NULL; + + } else { + nxt_unit_debug(ctx, "new mmap #%"PRIu32" created for %d -> %d", + hdr->id, (int) lib->pid, (int) port->id.pid); + } + + nxt_unit_close(fd); + + pthread_mutex_lock(&lib->outgoing.mutex); + + if (nxt_fast_path(hdr != NULL)) { + return hdr; + } + +remove_fail: + + lib->outgoing.size--; + + return NULL; +} + + +static int +nxt_unit_shm_open(nxt_unit_ctx_t *ctx, size_t size) +{ + int fd; + nxt_unit_impl_t *lib; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + +#if (NXT_HAVE_MEMFD_CREATE || NXT_HAVE_SHM_OPEN) + char name[64]; + snprintf(name, sizeof(name), NXT_SHM_PREFIX "unit.%d.%p", lib->pid, (void *) pthread_self()); +#endif #if (NXT_HAVE_MEMFD_CREATE) @@ -3137,7 +3674,7 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, nxt_unit_alert(ctx, "memfd_create(%s) failed: %s (%d)", name, strerror(errno), errno); - goto remove_fail; + return -1; } nxt_unit_debug(ctx, "memfd_create(%s): %d", name, fd); @@ -3149,7 +3686,7 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, nxt_unit_alert(ctx, "shm_open(SHM_ANON) failed: %s (%d)", strerror(errno), errno); - goto remove_fail; + return -1; } #elif (NXT_HAVE_SHM_OPEN) @@ -3162,12 +3699,12 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, nxt_unit_alert(ctx, "shm_open(%s) failed: %s (%d)", name, strerror(errno), errno); - goto remove_fail; + return -1; } if (nxt_slow_path(shm_unlink(name) == -1)) { - nxt_unit_warn(ctx, "shm_unlink(%s) failed: %s (%d)", name, - strerror(errno), errno); + nxt_unit_alert(ctx, "shm_unlink(%s) failed: %s (%d)", name, + strerror(errno), errno); } #else @@ -3176,71 +3713,21 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, #endif - if (nxt_slow_path(ftruncate(fd, PORT_MMAP_SIZE) == -1)) { + if (nxt_slow_path(ftruncate(fd, size) == -1)) { nxt_unit_alert(ctx, "ftruncate(%d) failed: %s (%d)", fd, strerror(errno), errno); - goto remove_fail; - } - - mem = mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (nxt_slow_path(mem == MAP_FAILED)) { - nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", fd, - strerror(errno), errno); - - goto remove_fail; - } - - mm->hdr = mem; - hdr = mem; - - memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); - memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); - - hdr->id = process->outgoing.size - 1; - hdr->src_pid = lib->pid; - hdr->dst_pid = process->pid; - hdr->sent_over = port_id->id; + nxt_unit_close(fd); - /* Mark first n chunk(s) as busy */ - for (i = 0; i < n; i++) { - nxt_port_mmap_set_chunk_busy(hdr->free_map, i); - } - - /* Mark as busy chunk followed the last available chunk. */ - nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); - nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); - - pthread_mutex_unlock(&process->outgoing.mutex); - - rc = nxt_unit_send_mmap(ctx, port_id, fd); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - munmap(mem, PORT_MMAP_SIZE); - hdr = NULL; - - } else { - nxt_unit_debug(ctx, "new mmap #%"PRIu32" created for %d -> %d", - hdr->id, (int) lib->pid, (int) process->pid); - } - - close(fd); - - pthread_mutex_lock(&process->outgoing.mutex); - - if (nxt_fast_path(hdr != NULL)) { - return hdr; + return -1; } -remove_fail: - - process->outgoing.size--; - - return NULL; + return fd; } static int -nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, int fd) +nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int fd) { ssize_t res; nxt_port_msg_t msg; @@ -3284,8 +3771,8 @@ nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, int fd) */ memcpy(CMSG_DATA(&cmsg.cm), &fd, sizeof(int)); - res = lib->callbacks.port_send(ctx, port_id, &msg, sizeof(msg), - &cmsg, sizeof(cmsg)); + res = nxt_unit_port_send(ctx, port, &msg, sizeof(msg), + &cmsg, sizeof(cmsg)); if (nxt_slow_path(res != sizeof(msg))) { return NXT_UNIT_ERROR; } @@ -3295,8 +3782,8 @@ nxt_unit_send_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, int fd) static int -nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, - nxt_unit_port_id_t *port_id, uint32_t size, uint32_t min_size, +nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + uint32_t size, uint32_t min_size, nxt_unit_mmap_buf_t *mmap_buf, char *local_buf) { int nchunks, min_nchunks; @@ -3321,8 +3808,6 @@ nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, mmap_buf->buf.start = mmap_buf->plain_ptr + sizeof(nxt_port_msg_t); mmap_buf->buf.free = mmap_buf->buf.start; mmap_buf->buf.end = mmap_buf->buf.start + size; - mmap_buf->port_id = *port_id; - mmap_buf->process = process; nxt_unit_debug(ctx, "outgoing plain buffer allocation: (%p, %d)", mmap_buf->buf.start, (int) size); @@ -3333,7 +3818,7 @@ nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, nchunks = (size + PORT_MMAP_CHUNK_SIZE - 1) / PORT_MMAP_CHUNK_SIZE; min_nchunks = (min_size + PORT_MMAP_CHUNK_SIZE - 1) / PORT_MMAP_CHUNK_SIZE; - hdr = nxt_unit_mmap_get(ctx, process, port_id, &c, &nchunks, min_nchunks); + hdr = nxt_unit_mmap_get(ctx, port, &c, &nchunks, min_nchunks); if (nxt_slow_path(hdr == NULL)) { if (nxt_fast_path(min_nchunks == 0 && nchunks == 0)) { mmap_buf->hdr = NULL; @@ -3352,8 +3837,6 @@ nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, mmap_buf->buf.start = (char *) nxt_port_mmap_chunk_start(hdr, c); mmap_buf->buf.free = mmap_buf->buf.start; mmap_buf->buf.end = mmap_buf->buf.start + nchunks * PORT_MMAP_CHUNK_SIZE; - mmap_buf->port_id = *port_id; - mmap_buf->process = process; mmap_buf->free_ptr = NULL; mmap_buf->ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); @@ -3368,83 +3851,87 @@ nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, static int nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) { - int rc; - void *mem; - struct stat mmap_stat; - nxt_unit_mmap_t *mm; - nxt_unit_impl_t *lib; - nxt_unit_process_t *process; - nxt_port_mmap_header_t *hdr; + int rc; + void *mem; + nxt_queue_t awaiting_rbuf; + struct stat mmap_stat; + nxt_unit_mmap_t *mm; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_read_buf_t *rbuf; + nxt_port_mmap_header_t *hdr; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); nxt_unit_debug(ctx, "incoming_mmap: fd %d from process %d", fd, (int) pid); - pthread_mutex_lock(&lib->mutex); - - process = nxt_unit_process_find(ctx, pid, 0); - - pthread_mutex_unlock(&lib->mutex); - - if (nxt_slow_path(process == NULL)) { - nxt_unit_warn(ctx, "incoming_mmap: process %d not found, fd %d", - (int) pid, fd); - - return NXT_UNIT_ERROR; - } - - rc = NXT_UNIT_ERROR; - if (fstat(fd, &mmap_stat) == -1) { - nxt_unit_warn(ctx, "incoming_mmap: fstat(%d) failed: %s (%d)", fd, - strerror(errno), errno); + nxt_unit_alert(ctx, "incoming_mmap: fstat(%d) failed: %s (%d)", fd, + strerror(errno), errno); - goto fail; + return NXT_UNIT_ERROR; } mem = mmap(NULL, mmap_stat.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (nxt_slow_path(mem == MAP_FAILED)) { - nxt_unit_warn(ctx, "incoming_mmap: mmap() failed: %s (%d)", - strerror(errno), errno); + nxt_unit_alert(ctx, "incoming_mmap: mmap() failed: %s (%d)", + strerror(errno), errno); - goto fail; + return NXT_UNIT_ERROR; } hdr = mem; - if (nxt_slow_path(hdr->src_pid != pid || hdr->dst_pid != lib->pid)) { + if (nxt_slow_path(hdr->src_pid != pid)) { - nxt_unit_warn(ctx, "incoming_mmap: unexpected pid in mmap header " - "detected: %d != %d or %d != %d", (int) hdr->src_pid, - (int) pid, (int) hdr->dst_pid, (int) lib->pid); + nxt_unit_alert(ctx, "incoming_mmap: unexpected pid in mmap header " + "detected: %d != %d or %d != %d", (int) hdr->src_pid, + (int) pid, (int) hdr->dst_pid, (int) lib->pid); munmap(mem, PORT_MMAP_SIZE); - goto fail; + return NXT_UNIT_ERROR; } - pthread_mutex_lock(&process->incoming.mutex); + nxt_queue_init(&awaiting_rbuf); + + pthread_mutex_lock(&lib->incoming.mutex); - mm = nxt_unit_mmap_at(&process->incoming, hdr->id); + mm = nxt_unit_mmap_at(&lib->incoming, hdr->id); if (nxt_slow_path(mm == NULL)) { - nxt_unit_warn(ctx, "incoming_mmap: failed to add to incoming array"); + nxt_unit_alert(ctx, "incoming_mmap: failed to add to incoming array"); munmap(mem, PORT_MMAP_SIZE); + rc = NXT_UNIT_ERROR; + } else { mm->hdr = hdr; hdr->sent_over = 0xFFFFu; + nxt_queue_add(&awaiting_rbuf, &mm->awaiting_rbuf); + nxt_queue_init(&mm->awaiting_rbuf); + rc = NXT_UNIT_OK; } - pthread_mutex_unlock(&process->incoming.mutex); + pthread_mutex_unlock(&lib->incoming.mutex); -fail: + nxt_queue_each(rbuf, &awaiting_rbuf, nxt_unit_read_buf_t, link) { - nxt_unit_process_use(ctx, process, -1); + ctx_impl = rbuf->ctx_impl; + + pthread_mutex_lock(&ctx_impl->mutex); + + nxt_queue_insert_head(&ctx_impl->pending_rbuf, &rbuf->link); + + pthread_mutex_unlock(&ctx_impl->mutex); + + nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + + } nxt_queue_loop; return rc; } @@ -3462,18 +3949,22 @@ nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps) } -static void -nxt_unit_process_use(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, int i) +nxt_inline void +nxt_unit_process_use(nxt_unit_process_t *process) { - long c; + nxt_atomic_fetch_add(&process->use_count, 1); +} + - c = nxt_atomic_fetch_add(&process->use_count, i); +nxt_inline void +nxt_unit_process_release(nxt_unit_process_t *process) +{ + long c; - if (i < 0 && c == -i) { - nxt_unit_debug(ctx, "destroy process #%d", (int) process->pid); + c = nxt_atomic_fetch_add(&process->use_count, -1); - nxt_unit_mmaps_destroy(&process->incoming); - nxt_unit_mmaps_destroy(&process->outgoing); + if (c == 1) { + nxt_unit_debug(NULL, "destroy process #%d", (int) process->pid); free(process); } @@ -3499,85 +3990,62 @@ nxt_unit_mmaps_destroy(nxt_unit_mmaps_t *mmaps) } -static nxt_port_mmap_header_t * -nxt_unit_get_incoming_mmap(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process, - uint32_t id) +static int +nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, + pid_t pid, uint32_t id, nxt_port_mmap_header_t **hdr, + nxt_unit_read_buf_t *rbuf) { - nxt_port_mmap_header_t *hdr; - - if (nxt_fast_path(process->incoming.size > id)) { - hdr = process->incoming.elts[id].hdr; - - } else { - hdr = NULL; - } + int res, need_rbuf; + nxt_unit_mmap_t *mm; + nxt_unit_ctx_impl_t *ctx_impl; - return hdr; -} + mm = nxt_unit_mmap_at(mmaps, id); + if (nxt_slow_path(mm == NULL)) { + nxt_unit_alert(ctx, "failed to allocate mmap"); + pthread_mutex_unlock(&mmaps->mutex); -static int -nxt_unit_tracking_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) -{ - int rc; - nxt_chunk_id_t c; - nxt_unit_process_t *process; - nxt_port_mmap_header_t *hdr; - nxt_port_mmap_tracking_msg_t *tracking_msg; + *hdr = NULL; - if (recv_msg->size < (int) sizeof(nxt_port_mmap_tracking_msg_t)) { - nxt_unit_warn(ctx, "#%"PRIu32": tracking_read: too small message (%d)", - recv_msg->stream, (int) recv_msg->size); - - return 0; + return NXT_UNIT_ERROR; } - tracking_msg = recv_msg->start; + *hdr = mm->hdr; - recv_msg->start = tracking_msg + 1; - recv_msg->size -= sizeof(nxt_port_mmap_tracking_msg_t); - - process = nxt_unit_msg_get_process(ctx, recv_msg); - if (nxt_slow_path(process == NULL)) { - return 0; + if (nxt_fast_path(*hdr != NULL)) { + return NXT_UNIT_OK; } - pthread_mutex_lock(&process->incoming.mutex); - - hdr = nxt_unit_get_incoming_mmap(ctx, process, tracking_msg->mmap_id); - if (nxt_slow_path(hdr == NULL)) { - pthread_mutex_unlock(&process->incoming.mutex); + need_rbuf = nxt_queue_is_empty(&mm->awaiting_rbuf); - nxt_unit_warn(ctx, "#%"PRIu32": tracking_read: " - "invalid mmap id %d,%"PRIu32, - recv_msg->stream, (int) process->pid, - tracking_msg->mmap_id); + nxt_queue_insert_tail(&mm->awaiting_rbuf, &rbuf->link); - return 0; - } + pthread_mutex_unlock(&mmaps->mutex); - c = tracking_msg->tracking_id; - rc = nxt_atomic_cmp_set(hdr->tracking + c, recv_msg->stream, 0); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - if (rc == 0) { - nxt_unit_debug(ctx, "#%"PRIu32": tracking cancelled", - recv_msg->stream); + nxt_atomic_fetch_add(&ctx_impl->wait_items, 1); - nxt_port_mmap_set_chunk_free(hdr->free_tracking_map, c); + if (need_rbuf) { + res = nxt_unit_get_mmap(ctx, pid, id); + if (nxt_slow_path(res == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } } - pthread_mutex_unlock(&process->incoming.mutex); - - return rc; + return NXT_UNIT_AGAIN; } static int -nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) +nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, + nxt_unit_read_buf_t *rbuf) { + int res; void *start; uint32_t size; - nxt_unit_process_t *process; + nxt_unit_impl_t *lib; + nxt_unit_mmaps_t *mmaps; nxt_unit_mmap_buf_t *b, **incoming_tail; nxt_port_mmap_msg_t *mmap_msg, *end; nxt_port_mmap_header_t *hdr; @@ -3589,22 +4057,22 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) return NXT_UNIT_ERROR; } - process = nxt_unit_msg_get_process(ctx, recv_msg); - if (nxt_slow_path(process == NULL)) { - return NXT_UNIT_ERROR; - } - mmap_msg = recv_msg->start; end = nxt_pointer_to(recv_msg->start, recv_msg->size); incoming_tail = &recv_msg->incoming_buf; + /* Allocating buffer structures. */ for (; mmap_msg < end; mmap_msg++) { b = nxt_unit_mmap_buf_get(ctx); if (nxt_slow_path(b == NULL)) { nxt_unit_warn(ctx, "#%"PRIu32": mmap_read: failed to allocate buf", recv_msg->stream); + while (recv_msg->incoming_buf != NULL) { + nxt_unit_mmap_buf_release(recv_msg->incoming_buf); + } + return NXT_UNIT_ERROR; } @@ -3615,19 +4083,23 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) b = recv_msg->incoming_buf; mmap_msg = recv_msg->start; - pthread_mutex_lock(&process->incoming.mutex); + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + mmaps = &lib->incoming; + + pthread_mutex_lock(&mmaps->mutex); for (; mmap_msg < end; mmap_msg++) { - hdr = nxt_unit_get_incoming_mmap(ctx, process, mmap_msg->mmap_id); - if (nxt_slow_path(hdr == NULL)) { - pthread_mutex_unlock(&process->incoming.mutex); + res = nxt_unit_check_rbuf_mmap(ctx, mmaps, + recv_msg->pid, mmap_msg->mmap_id, + &hdr, rbuf); - nxt_unit_warn(ctx, "#%"PRIu32": mmap_read: " - "invalid mmap id %d,%"PRIu32, - recv_msg->stream, (int) process->pid, - mmap_msg->mmap_id); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + while (recv_msg->incoming_buf != NULL) { + nxt_unit_mmap_buf_release(recv_msg->incoming_buf); + } - return NXT_UNIT_ERROR; + return res; } start = nxt_port_mmap_chunk_start(hdr, mmap_msg->chunk_id); @@ -3642,7 +4114,6 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) b->buf.free = start; b->buf.end = b->buf.start + size; b->hdr = hdr; - b->process = process; b = b->next; @@ -3654,15 +4125,48 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) (int) mmap_msg->size); } - pthread_mutex_unlock(&process->incoming.mutex); + pthread_mutex_unlock(&mmaps->mutex); + + return NXT_UNIT_OK; +} + + +static int +nxt_unit_get_mmap(nxt_unit_ctx_t *ctx, pid_t pid, uint32_t id) +{ + ssize_t res; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + + struct { + nxt_port_msg_t msg; + nxt_port_msg_get_mmap_t get_mmap; + } m; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + memset(&m.msg, 0, sizeof(nxt_port_msg_t)); + + m.msg.pid = lib->pid; + m.msg.reply_port = ctx_impl->read_port->id.id; + m.msg.type = _NXT_PORT_MSG_GET_MMAP; + + m.get_mmap.id = id; + + nxt_unit_debug(ctx, "get_mmap: %d %d", (int) pid, (int) id); + + res = nxt_unit_port_send(ctx, lib->router_port, &m, sizeof(m), NULL, 0); + if (nxt_slow_path(res != sizeof(m))) { + return NXT_UNIT_ERROR; + } return NXT_UNIT_OK; } static void -nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, - nxt_unit_process_t *process, nxt_port_mmap_header_t *hdr, +nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, nxt_port_mmap_header_t *hdr, void *start, uint32_t size) { int freed_chunks; @@ -3688,12 +4192,10 @@ nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); if (hdr->src_pid == lib->pid && freed_chunks != 0) { - nxt_atomic_fetch_add(&process->outgoing.allocated_chunks, - -freed_chunks); + nxt_atomic_fetch_add(&lib->outgoing.allocated_chunks, -freed_chunks); - nxt_unit_debug(ctx, "process %d allocated_chunks %d", - process->pid, - (int) process->outgoing.allocated_chunks); + nxt_unit_debug(ctx, "allocated_chunks %d", + (int) lib->outgoing.allocated_chunks); } if (hdr->dst_pid == lib->pid @@ -3708,15 +4210,12 @@ nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, static int nxt_unit_send_shm_ack(nxt_unit_ctx_t *ctx, pid_t pid) { - ssize_t res; - nxt_port_msg_t msg; - nxt_unit_impl_t *lib; - nxt_unit_port_id_t port_id; + ssize_t res; + nxt_port_msg_t msg; + nxt_unit_impl_t *lib; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - nxt_unit_port_id_init(&port_id, pid, 0); - msg.stream = 0; msg.pid = lib->pid; msg.reply_port = 0; @@ -3727,7 +4226,7 @@ nxt_unit_send_shm_ack(nxt_unit_ctx_t *ctx, pid_t pid) msg.mf = 0; msg.tracking = 0; - res = lib->callbacks.port_send(ctx, &port_id, &msg, sizeof(msg), NULL, 0); + res = nxt_unit_port_send(ctx, lib->router_port, &msg, sizeof(msg), NULL, 0); if (nxt_slow_path(res != sizeof(msg))) { return NXT_UNIT_ERROR; } @@ -3772,40 +4271,34 @@ nxt_unit_process_lhq_pid(nxt_lvlhsh_query_t *lhq, pid_t *pid) static nxt_unit_process_t * -nxt_unit_process_get(nxt_unit_ctx_t *ctx, pid_t pid) +nxt_unit_process_get(nxt_unit_impl_t *lib, pid_t pid) { - nxt_unit_impl_t *lib; nxt_unit_process_t *process; nxt_lvlhsh_query_t lhq; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - nxt_unit_process_lhq_pid(&lhq, &pid); if (nxt_lvlhsh_find(&lib->processes, &lhq) == NXT_OK) { process = lhq.value; - nxt_unit_process_use(ctx, process, 1); + nxt_unit_process_use(process); return process; } process = malloc(sizeof(nxt_unit_process_t)); if (nxt_slow_path(process == NULL)) { - nxt_unit_warn(ctx, "failed to allocate process for #%d", (int) pid); + nxt_unit_alert(NULL, "failed to allocate process for #%d", (int) pid); return NULL; } process->pid = pid; - process->use_count = 1; + process->use_count = 2; process->next_port_id = 0; process->lib = lib; nxt_queue_init(&process->ports); - nxt_unit_mmaps_init(&process->incoming); - nxt_unit_mmaps_init(&process->outgoing); - lhq.replace = 0; lhq.value = process; @@ -3815,31 +4308,23 @@ nxt_unit_process_get(nxt_unit_ctx_t *ctx, pid_t pid) break; default: - nxt_unit_warn(ctx, "process %d insert failed", (int) pid); + nxt_unit_alert(NULL, "process %d insert failed", (int) pid); - pthread_mutex_destroy(&process->outgoing.mutex); - pthread_mutex_destroy(&process->incoming.mutex); free(process); process = NULL; break; } - nxt_unit_process_use(ctx, process, 1); - return process; } static nxt_unit_process_t * -nxt_unit_process_find(nxt_unit_ctx_t *ctx, pid_t pid, int remove) +nxt_unit_process_find(nxt_unit_impl_t *lib, pid_t pid, int remove) { int rc; - nxt_unit_impl_t *lib; - nxt_unit_process_t *process; nxt_lvlhsh_query_t lhq; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - nxt_unit_process_lhq_pid(&lhq, &pid); if (remove) { @@ -3850,13 +4335,11 @@ nxt_unit_process_find(nxt_unit_ctx_t *ctx, pid_t pid, int remove) } if (rc == NXT_OK) { - process = lhq.value; - if (!remove) { - nxt_unit_process_use(ctx, process, 1); + nxt_unit_process_use(lhq.value); } - return process; + return lhq.value; } return NULL; @@ -3876,17 +4359,21 @@ nxt_unit_run(nxt_unit_ctx_t *ctx) int rc; nxt_unit_impl_t *lib; + nxt_unit_ctx_use(ctx); + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); rc = NXT_UNIT_OK; while (nxt_fast_path(lib->online)) { - rc = nxt_unit_run_once(ctx); + rc = nxt_unit_run_once_impl(ctx); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { break; } } + nxt_unit_ctx_release(ctx); + return rc; } @@ -3894,174 +4381,578 @@ nxt_unit_run(nxt_unit_ctx_t *ctx) int nxt_unit_run_once(nxt_unit_ctx_t *ctx) { + int rc; + + nxt_unit_ctx_use(ctx); + + rc = nxt_unit_run_once_impl(ctx); + + nxt_unit_ctx_release(ctx); + + return rc; +} + + +static int +nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx) +{ int rc; - nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_read_buf_t *rbuf; + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + return NXT_UNIT_ERROR; + } + + rc = nxt_unit_read_buf(ctx, rbuf); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + nxt_unit_read_buf_release(ctx, rbuf); + + return rc; + } + + rc = nxt_unit_process_msg(ctx, rbuf); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } + + rc = nxt_unit_process_pending_rbuf(ctx); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } + + nxt_unit_process_ready_req(ctx); + + return rc; +} + + +static int +nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) +{ + int nevents, res, err; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_port_impl_t *port_impl; + struct pollfd fds[2]; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - pthread_mutex_lock(&ctx_impl->mutex); + if (ctx_impl->wait_items > 0 || lib->shared_port == NULL) { - if (ctx_impl->pending_read_head != NULL) { - rbuf = ctx_impl->pending_read_head; - ctx_impl->pending_read_head = rbuf->next; + return nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); + } + + port_impl = nxt_container_of(ctx_impl->read_port, nxt_unit_port_impl_t, + port); - if (ctx_impl->pending_read_tail == &rbuf->next) { - ctx_impl->pending_read_tail = &ctx_impl->pending_read_head; +retry: + + if (port_impl->from_socket == 0) { + res = nxt_unit_port_queue_recv(ctx_impl->read_port, rbuf); + if (res == NXT_UNIT_OK) { + if (nxt_unit_is_read_socket(rbuf)) { + port_impl->from_socket++; + + nxt_unit_debug(ctx, "port{%d,%d} dequeue 1 read_socket %d", + (int) ctx_impl->read_port->id.pid, + (int) ctx_impl->read_port->id.id, + port_impl->from_socket); + + } else { + nxt_unit_debug(ctx, "port{%d,%d} dequeue %d", + (int) ctx_impl->read_port->id.pid, + (int) ctx_impl->read_port->id.id, + (int) rbuf->size); + + return NXT_UNIT_OK; + } } + } - pthread_mutex_unlock(&ctx_impl->mutex); + res = nxt_unit_app_queue_recv(lib->shared_port, rbuf); + if (res == NXT_UNIT_OK) { + return NXT_UNIT_OK; + } - } else { - rbuf = nxt_unit_read_buf_get_impl(ctx_impl); - if (nxt_slow_path(rbuf == NULL)) { - return NXT_UNIT_ERROR; + fds[0].fd = ctx_impl->read_port->in_fd; + fds[0].events = POLLIN; + fds[0].revents = 0; + + fds[1].fd = lib->shared_port->in_fd; + fds[1].events = POLLIN; + fds[1].revents = 0; + + nevents = poll(fds, 2, -1); + if (nxt_slow_path(nevents == -1)) { + err = errno; + + if (err == EINTR) { + goto retry; } - nxt_unit_read_buf(ctx, rbuf); + nxt_unit_alert(ctx, "poll(%d,%d) failed: %s (%d)", + fds[0].fd, fds[1].fd, strerror(err), err); + + rbuf->size = -1; + + return (err == EAGAIN) ? NXT_UNIT_AGAIN : NXT_UNIT_ERROR; } - if (nxt_fast_path(rbuf->size > 0)) { - rc = nxt_unit_process_msg(ctx, &ctx_impl->read_port_id, - rbuf->buf, rbuf->size, - rbuf->oob, sizeof(rbuf->oob)); + nxt_unit_debug(ctx, "poll(%d,%d): %d, revents [%04uXi, %04uXi]", + fds[0].fd, fds[1].fd, nevents, fds[0].revents, + fds[1].revents); -#if (NXT_DEBUG) - memset(rbuf->buf, 0xAC, rbuf->size); -#endif + if ((fds[0].revents & POLLIN) != 0) { + res = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); + if (res == NXT_UNIT_AGAIN) { + goto retry; + } - } else { - rc = NXT_UNIT_ERROR; + return res; + } + + if ((fds[1].revents & POLLIN) != 0) { + res = nxt_unit_shared_port_recv(ctx, lib->shared_port, rbuf); + if (res == NXT_UNIT_AGAIN) { + goto retry; + } + + return res; } - nxt_unit_read_buf_release(ctx, rbuf); + nxt_unit_alert(ctx, "poll(%d,%d): %d unexpected revents [%04uXi, %04uXi]", + fds[0].fd, fds[1].fd, nevents, fds[0].revents, + fds[1].revents); + + return NXT_UNIT_ERROR; +} + + +static int +nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx) +{ + int rc; + nxt_queue_t pending_rbuf; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_read_buf_t *rbuf; + + nxt_queue_init(&pending_rbuf); + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + pthread_mutex_lock(&ctx_impl->mutex); + + if (nxt_queue_is_empty(&ctx_impl->pending_rbuf)) { + pthread_mutex_unlock(&ctx_impl->mutex); + + return NXT_UNIT_OK; + } + + nxt_queue_add(&pending_rbuf, &ctx_impl->pending_rbuf); + nxt_queue_init(&ctx_impl->pending_rbuf); + + pthread_mutex_unlock(&ctx_impl->mutex); + + rc = NXT_UNIT_OK; + + nxt_queue_each(rbuf, &pending_rbuf, nxt_unit_read_buf_t, link) { + + if (nxt_fast_path(rc != NXT_UNIT_ERROR)) { + rc = nxt_unit_process_msg(&ctx_impl->ctx, rbuf); + + } else { + nxt_unit_read_buf_release(ctx, rbuf); + } + + } nxt_queue_loop; return rc; } static void -nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) +nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) +{ + int res; + nxt_queue_t ready_req; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_t *req; + nxt_unit_request_info_impl_t *req_impl; + + nxt_queue_init(&ready_req); + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + pthread_mutex_lock(&ctx_impl->mutex); + + if (nxt_queue_is_empty(&ctx_impl->ready_req)) { + pthread_mutex_unlock(&ctx_impl->mutex); + + return; + } + + nxt_queue_add(&ready_req, &ctx_impl->ready_req); + nxt_queue_init(&ctx_impl->ready_req); + + pthread_mutex_unlock(&ctx_impl->mutex); + + nxt_queue_each(req_impl, &ready_req, + nxt_unit_request_info_impl_t, port_wait_link) + { + lib = nxt_container_of(ctx_impl->ctx.unit, nxt_unit_impl_t, unit); + + req = &req_impl->req; + + res = nxt_unit_send_req_headers_ack(req); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + continue; + } + + if (req->content_length + > (uint64_t) (req->content_buf->end - req->content_buf->free)) + { + res = nxt_unit_request_hash_add(ctx, req); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + nxt_unit_req_warn(req, "failed to add request to hash"); + + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + continue; + } + + /* + * If application have separate data handler, we may start + * request processing and process data when it is arrived. + */ + if (lib->callbacks.data_handler == NULL) { + continue; + } + } + + lib->callbacks.request_handler(&req_impl->req); + + } nxt_queue_loop; +} + + +int +nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) { + int rc; nxt_unit_impl_t *lib; + nxt_unit_read_buf_t *rbuf; nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_ctx_use(ctx); + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - memset(rbuf->oob, 0, sizeof(struct cmsghdr)); + rc = NXT_UNIT_OK; - if (ctx_impl->read_port_fd != -1) { - rbuf->size = nxt_unit_port_recv(ctx, ctx_impl->read_port_fd, - rbuf->buf, sizeof(rbuf->buf), - rbuf->oob, sizeof(rbuf->oob)); + while (nxt_fast_path(lib->online)) { + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + rc = NXT_UNIT_ERROR; + break; + } - } else { - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + retry: - rbuf->size = lib->callbacks.port_recv(ctx, &ctx_impl->read_port_id, - rbuf->buf, sizeof(rbuf->buf), - rbuf->oob, sizeof(rbuf->oob)); + rc = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); + if (rc == NXT_UNIT_AGAIN) { + goto retry; + } + + rc = nxt_unit_process_msg(ctx, rbuf); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + break; + } + + rc = nxt_unit_process_pending_rbuf(ctx); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + break; + } + + nxt_unit_process_ready_req(ctx); } + + nxt_unit_ctx_release(ctx); + + return rc; } -void -nxt_unit_done(nxt_unit_ctx_t *ctx) +nxt_inline int +nxt_unit_is_read_queue(nxt_unit_read_buf_t *rbuf) { + nxt_port_msg_t *port_msg; + + if (nxt_fast_path(rbuf->size == (ssize_t) sizeof(nxt_port_msg_t))) { + port_msg = (nxt_port_msg_t *) rbuf->buf; + + return port_msg->type == _NXT_PORT_MSG_READ_QUEUE; + } + + return 0; +} + + +nxt_inline int +nxt_unit_is_read_socket(nxt_unit_read_buf_t *rbuf) +{ + if (nxt_fast_path(rbuf->size == 1)) { + return rbuf->buf[0] == _NXT_PORT_MSG_READ_SOCKET; + } + + return 0; +} + + +nxt_inline int +nxt_unit_is_shm_ack(nxt_unit_read_buf_t *rbuf) +{ + nxt_port_msg_t *port_msg; + + if (nxt_fast_path(rbuf->size == (ssize_t) sizeof(nxt_port_msg_t))) { + port_msg = (nxt_port_msg_t *) rbuf->buf; + + return port_msg->type == _NXT_PORT_MSG_SHM_ACK; + } + + return 0; +} + + +nxt_inline int +nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf) +{ + nxt_port_msg_t *port_msg; + + if (nxt_fast_path(rbuf->size == (ssize_t) sizeof(nxt_port_msg_t))) { + port_msg = (nxt_port_msg_t *) rbuf->buf; + + return port_msg->type == _NXT_PORT_MSG_QUIT; + } + + return 0; +} + + +int +nxt_unit_run_shared(nxt_unit_ctx_t *ctx) +{ + int rc; nxt_unit_impl_t *lib; - nxt_unit_process_t *process; - nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_read_buf_t *rbuf; + + nxt_unit_ctx_use(ctx); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + rc = NXT_UNIT_OK; - nxt_queue_each(ctx_impl, &lib->contexts, nxt_unit_ctx_impl_t, link) { + while (nxt_fast_path(lib->online)) { + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + rc = NXT_UNIT_ERROR; + break; + } - nxt_unit_ctx_free(&ctx_impl->ctx); + retry: - } nxt_queue_loop; + rc = nxt_unit_shared_port_recv(ctx, lib->shared_port, rbuf); + if (rc == NXT_UNIT_AGAIN) { + goto retry; + } - for ( ;; ) { - pthread_mutex_lock(&lib->mutex); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + nxt_unit_read_buf_release(ctx, rbuf); + break; + } - process = nxt_unit_process_pop_first(lib); - if (process == NULL) { - pthread_mutex_unlock(&lib->mutex); + rc = nxt_unit_process_msg(ctx, rbuf); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + break; + } + rc = nxt_unit_process_pending_rbuf(ctx); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { break; } - nxt_unit_remove_process(ctx, process); + nxt_unit_process_ready_req(ctx); } - pthread_mutex_destroy(&lib->mutex); + nxt_unit_ctx_release(ctx); - free(lib); + return rc; +} + + +int +nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) +{ + int rc; + + nxt_unit_ctx_use(ctx); + + rc = nxt_unit_process_port_msg_impl(ctx, port); + + nxt_unit_ctx_release(ctx); + + return rc; +} + + +static int +nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) +{ + int rc; + nxt_unit_impl_t *lib; + nxt_unit_read_buf_t *rbuf; + + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + return NXT_UNIT_ERROR; + } + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + +retry: + + if (port == lib->shared_port) { + rc = nxt_unit_shared_port_recv(ctx, port, rbuf); + + } else { + rc = nxt_unit_ctx_port_recv(ctx, port, rbuf); + } + + if (rc != NXT_UNIT_OK) { + nxt_unit_read_buf_release(ctx, rbuf); + return rc; + } + + rc = nxt_unit_process_msg(ctx, rbuf); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } + + rc = nxt_unit_process_pending_rbuf(ctx); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } + + nxt_unit_process_ready_req(ctx); + + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + return NXT_UNIT_ERROR; + } + + if (lib->online) { + goto retry; + } + + return rc; +} + + +void +nxt_unit_done(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_release(ctx); } nxt_unit_ctx_t * nxt_unit_ctx_alloc(nxt_unit_ctx_t *ctx, void *data) { - int rc, fd; - nxt_unit_impl_t *lib; - nxt_unit_port_id_t new_port_id; - nxt_unit_ctx_impl_t *new_ctx; + int rc, queue_fd; + void *mem; + nxt_unit_impl_t *lib; + nxt_unit_port_t *port; + nxt_unit_ctx_impl_t *new_ctx; + nxt_unit_port_impl_t *port_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); new_ctx = malloc(sizeof(nxt_unit_ctx_impl_t) + lib->request_data_size); if (nxt_slow_path(new_ctx == NULL)) { - nxt_unit_warn(ctx, "failed to allocate context"); + nxt_unit_alert(ctx, "failed to allocate context"); return NULL; } - rc = nxt_unit_create_port(ctx, &new_port_id, &fd); + rc = nxt_unit_ctx_init(lib, new_ctx, data); if (nxt_slow_path(rc != NXT_UNIT_OK)) { - free(new_ctx); + free(new_ctx); - return NULL; + return NULL; } - rc = nxt_unit_send_port(ctx, &lib->ready_port_id, &new_port_id, fd); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - lib->callbacks.remove_port(ctx, &new_port_id); + queue_fd = -1; - close(fd); + port = nxt_unit_create_port(ctx); + if (nxt_slow_path(port == NULL)) { + goto fail; + } - free(new_ctx); + new_ctx->read_port = port; - return NULL; + queue_fd = nxt_unit_shm_open(ctx, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue_fd == -1)) { + goto fail; } - close(fd); + mem = mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, MAP_SHARED, queue_fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_unit_alert(ctx, "mmap(%d) failed: %s (%d)", queue_fd, + strerror(errno), errno); - rc = nxt_unit_ctx_init(lib, new_ctx, data); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - lib->callbacks.remove_port(ctx, &new_port_id); + goto fail; + } - free(new_ctx); + nxt_port_queue_init(mem); - return NULL; + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + port_impl->queue = mem; + + rc = nxt_unit_send_port(ctx, lib->router_port, port, queue_fd); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; } - new_ctx->read_port_id = new_port_id; + nxt_unit_close(queue_fd); return &new_ctx->ctx; + +fail: + + if (queue_fd != -1) { + nxt_unit_close(queue_fd); + } + + nxt_unit_ctx_release(&new_ctx->ctx); + + return NULL; } -void -nxt_unit_ctx_free(nxt_unit_ctx_t *ctx) +static void +nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl) { nxt_unit_impl_t *lib; - nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_mmap_buf_t *mmap_buf; nxt_unit_request_info_impl_t *req_impl; nxt_unit_websocket_frame_impl_t *ws_impl; - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + lib = nxt_container_of(ctx_impl->ctx.unit, nxt_unit_impl_t, unit); nxt_queue_each(req_impl, &ctx_impl->active_req, nxt_unit_request_info_impl_t, link) @@ -4099,9 +4990,16 @@ nxt_unit_ctx_free(nxt_unit_ctx_t *ctx) nxt_queue_remove(&ctx_impl->link); + if (nxt_fast_path(ctx_impl->read_port != NULL)) { + nxt_unit_remove_port(lib, &ctx_impl->read_port->id); + nxt_unit_port_release(ctx_impl->read_port); + } + if (ctx_impl != &lib->main_ctx) { free(ctx_impl); } + + nxt_unit_lib_release(lib); } @@ -4127,42 +5025,12 @@ nxt_unit_port_id_init(nxt_unit_port_id_t *port_id, pid_t pid, uint16_t id) } -int -nxt_unit_create_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *dst, - nxt_unit_port_id_t *port_id) -{ - int rc, fd; - nxt_unit_impl_t *lib; - nxt_unit_port_id_t new_port_id; - - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - - rc = nxt_unit_create_port(ctx, &new_port_id, &fd); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - return rc; - } - - rc = nxt_unit_send_port(ctx, dst, &new_port_id, fd); - - if (nxt_fast_path(rc == NXT_UNIT_OK)) { - *port_id = new_port_id; - - } else { - lib->callbacks.remove_port(ctx, &new_port_id); - } - - close(fd); - - return rc; -} - - -static int -nxt_unit_create_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, int *fd) +static nxt_unit_port_t * +nxt_unit_create_port(nxt_unit_ctx_t *ctx) { int rc, port_sockets[2]; nxt_unit_impl_t *lib; - nxt_unit_port_t new_port; + nxt_unit_port_t new_port, *port; nxt_unit_process_t *process; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -4172,7 +5040,7 @@ nxt_unit_create_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, int *fd) nxt_unit_warn(ctx, "create_port: socketpair() failed: %s (%d)", strerror(errno), errno); - return NXT_UNIT_ERROR; + return NULL; } nxt_unit_debug(ctx, "create_port: new socketpair: %d->%d", @@ -4180,49 +5048,43 @@ nxt_unit_create_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, int *fd) pthread_mutex_lock(&lib->mutex); - process = nxt_unit_process_get(ctx, lib->pid); + process = nxt_unit_process_get(lib, lib->pid); if (nxt_slow_path(process == NULL)) { pthread_mutex_unlock(&lib->mutex); - close(port_sockets[0]); - close(port_sockets[1]); + nxt_unit_close(port_sockets[0]); + nxt_unit_close(port_sockets[1]); - return NXT_UNIT_ERROR; + return NULL; } nxt_unit_port_id_init(&new_port.id, lib->pid, process->next_port_id++); new_port.in_fd = port_sockets[0]; - new_port.out_fd = -1; + new_port.out_fd = port_sockets[1]; new_port.data = NULL; pthread_mutex_unlock(&lib->mutex); - nxt_unit_process_use(ctx, process, -1); - - rc = lib->callbacks.add_port(ctx, &new_port); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_unit_warn(ctx, "create_port: add_port() failed"); - - close(port_sockets[0]); - close(port_sockets[1]); + nxt_unit_process_release(process); - return rc; + port = nxt_unit_add_port(ctx, &new_port, NULL); + if (nxt_slow_path(port == NULL)) { + nxt_unit_close(port_sockets[0]); + nxt_unit_close(port_sockets[1]); } - *port_id = new_port.id; - *fd = port_sockets[1]; - - return rc; + return port; } static int -nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *dst, - nxt_unit_port_id_t *new_port, int fd) +nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *dst, + nxt_unit_port_t *port, int queue_fd) { ssize_t res; nxt_unit_impl_t *lib; + int fds[2] = { port->out_fd, queue_fd }; struct { nxt_port_msg_t msg; @@ -4231,7 +5093,7 @@ nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *dst, union { struct cmsghdr cm; - char space[CMSG_SPACE(sizeof(int))]; + char space[CMSG_SPACE(sizeof(int) * 2)]; } cmsg; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -4246,15 +5108,15 @@ nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *dst, m.msg.mf = 0; m.msg.tracking = 0; - m.new_port.id = new_port->id; - m.new_port.pid = new_port->pid; + m.new_port.id = port->id.id; + m.new_port.pid = port->id.pid; m.new_port.type = NXT_PROCESS_APP; m.new_port.max_size = 16 * 1024; m.new_port.max_share = 64 * 1024; memset(&cmsg, 0, sizeof(cmsg)); - cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int)); + cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int) * 2); cmsg.cm.cmsg_level = SOL_SOCKET; cmsg.cm.cmsg_type = SCM_RIGHTS; @@ -4267,22 +5129,74 @@ nxt_unit_send_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *dst, * Fortunately, GCC with -O1 compiles this nxt_memcpy() * in the same simple assignment as in the code above. */ - memcpy(CMSG_DATA(&cmsg.cm), &fd, sizeof(int)); + memcpy(CMSG_DATA(&cmsg.cm), fds, sizeof(int) * 2); - res = lib->callbacks.port_send(ctx, dst, &m, sizeof(m), - &cmsg, sizeof(cmsg)); + res = nxt_unit_port_send(ctx, dst, &m, sizeof(m), &cmsg, sizeof(cmsg)); - return res == sizeof(m) ? NXT_UNIT_OK : NXT_UNIT_ERROR; + return (res == sizeof(m)) ? NXT_UNIT_OK : NXT_UNIT_ERROR; } -int -nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) +nxt_inline void nxt_unit_port_use(nxt_unit_port_t *port) { - int rc; - nxt_unit_impl_t *lib; - nxt_unit_process_t *process; - nxt_unit_port_impl_t *new_port, *old_port; + nxt_unit_port_impl_t *port_impl; + + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + + nxt_atomic_fetch_add(&port_impl->use_count, 1); +} + + +nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port) +{ + long c; + nxt_unit_port_impl_t *port_impl; + + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + + c = nxt_atomic_fetch_add(&port_impl->use_count, -1); + + if (c == 1) { + nxt_unit_debug(NULL, "destroy port{%d,%d} in_fd %d out_fd %d", + (int) port->id.pid, (int) port->id.id, + port->in_fd, port->out_fd); + + nxt_unit_process_release(port_impl->process); + + if (port->in_fd != -1) { + nxt_unit_close(port->in_fd); + + port->in_fd = -1; + } + + if (port->out_fd != -1) { + nxt_unit_close(port->out_fd); + + port->out_fd = -1; + } + + if (port_impl->queue != NULL) { + munmap(port_impl->queue, (port->id.id == (nxt_port_id_t) -1) + ? sizeof(nxt_app_queue_t) + : sizeof(nxt_port_queue_t)); + } + + free(port_impl); + } +} + + +static nxt_unit_port_t * +nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) +{ + int rc; + nxt_queue_t awaiting_req; + nxt_unit_impl_t *lib; + nxt_unit_port_t *old_port; + nxt_unit_process_t *process; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_port_impl_t *new_port, *old_port_impl; + nxt_unit_request_info_impl_t *req_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -4291,32 +5205,91 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) old_port = nxt_unit_port_hash_find(&lib->ports, &port->id, 0); if (nxt_slow_path(old_port != NULL)) { - nxt_unit_debug(ctx, "add_port: duplicate %d,%d in_fd %d out_fd %d", - port->id.pid, port->id.id, - port->in_fd, port->out_fd); + nxt_unit_debug(ctx, "add_port: duplicate port{%d,%d} " + "in_fd %d out_fd %d queue %p", + port->id.pid, port->id.id, + port->in_fd, port->out_fd, queue); + + if (old_port->data == NULL) { + old_port->data = port->data; + port->data = NULL; + } + + if (old_port->in_fd == -1) { + old_port->in_fd = port->in_fd; + port->in_fd = -1; + } if (port->in_fd != -1) { - close(port->in_fd); + nxt_unit_close(port->in_fd); port->in_fd = -1; } + if (old_port->out_fd == -1) { + old_port->out_fd = port->out_fd; + port->out_fd = -1; + } + if (port->out_fd != -1) { - close(port->out_fd); + nxt_unit_close(port->out_fd); port->out_fd = -1; } + *port = *old_port; + + nxt_queue_init(&awaiting_req); + + old_port_impl = nxt_container_of(old_port, nxt_unit_port_impl_t, port); + + if (old_port_impl->queue == NULL) { + old_port_impl->queue = queue; + } + + if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) { + nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req); + nxt_queue_init(&old_port_impl->awaiting_req); + } + + old_port_impl->ready = (port->in_fd != -1 || port->out_fd != -1); + pthread_mutex_unlock(&lib->mutex); - return NXT_UNIT_OK; + if (lib->callbacks.add_port != NULL + && (port->in_fd != -1 || port->out_fd != -1)) + { + lib->callbacks.add_port(ctx, old_port); + } + + nxt_queue_each(req_impl, &awaiting_req, + nxt_unit_request_info_impl_t, port_wait_link) + { + nxt_queue_remove(&req_impl->port_wait_link); + + ctx_impl = nxt_container_of(req_impl->req.ctx, nxt_unit_ctx_impl_t, + ctx); + + pthread_mutex_lock(&ctx_impl->mutex); + + nxt_queue_insert_tail(&ctx_impl->ready_req, + &req_impl->port_wait_link); + + pthread_mutex_unlock(&ctx_impl->mutex); + + nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + + } nxt_queue_loop; + + return old_port; } - nxt_unit_debug(ctx, "add_port: %d,%d in_fd %d out_fd %d", + new_port = NULL; + + nxt_unit_debug(ctx, "add_port: port{%d,%d} in_fd %d out_fd %d queue %p", port->id.pid, port->id.id, - port->in_fd, port->out_fd); + port->in_fd, port->out_fd, queue); - process = nxt_unit_process_get(ctx, port->id.pid); + process = nxt_unit_process_get(lib, port->id.pid); if (nxt_slow_path(process == NULL)) { - rc = NXT_UNIT_ERROR; goto unlock; } @@ -4326,7 +5299,9 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) new_port = malloc(sizeof(nxt_unit_port_impl_t)); if (nxt_slow_path(new_port == NULL)) { - rc = NXT_UNIT_ERROR; + nxt_unit_alert(ctx, "add_port: %d,%d malloc() failed", + port->id.pid, port->id.id); + goto unlock; } @@ -4337,149 +5312,131 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) nxt_unit_alert(ctx, "add_port: %d,%d hash_add failed", port->id.pid, port->id.id); + free(new_port); + + new_port = NULL; + goto unlock; } nxt_queue_insert_tail(&process->ports, &new_port->link); - rc = NXT_UNIT_OK; - + new_port->use_count = 2; new_port->process = process; + new_port->ready = (port->in_fd != -1 || port->out_fd != -1); + new_port->queue = queue; + new_port->from_socket = 0; + new_port->socket_rbuf = NULL; + + nxt_queue_init(&new_port->awaiting_req); + + process = NULL; unlock: pthread_mutex_unlock(&lib->mutex); - if (nxt_slow_path(process != NULL && rc != NXT_UNIT_OK)) { - nxt_unit_process_use(ctx, process, -1); + if (nxt_slow_path(process != NULL)) { + nxt_unit_process_release(process); } - return rc; -} - + if (lib->callbacks.add_port != NULL + && new_port != NULL + && (port->in_fd != -1 || port->out_fd != -1)) + { + lib->callbacks.add_port(ctx, &new_port->port); + } -void -nxt_unit_remove_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id) -{ - nxt_unit_find_remove_port(ctx, port_id, NULL); + return &new_port->port; } -void -nxt_unit_find_remove_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, - nxt_unit_port_t *r_port) +static void +nxt_unit_remove_port(nxt_unit_impl_t *lib, nxt_unit_port_id_t *port_id) { - nxt_unit_impl_t *lib; - nxt_unit_process_t *process; - - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + nxt_unit_port_t *port; + nxt_unit_port_impl_t *port_impl; pthread_mutex_lock(&lib->mutex); - process = NULL; + port = nxt_unit_remove_port_unsafe(lib, port_id); - nxt_unit_remove_port_unsafe(ctx, port_id, r_port, &process); + if (nxt_fast_path(port != NULL)) { + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + + nxt_queue_remove(&port_impl->link); + } pthread_mutex_unlock(&lib->mutex); - if (nxt_slow_path(process != NULL)) { - nxt_unit_process_use(ctx, process, -1); + if (lib->callbacks.remove_port != NULL && port != NULL) { + lib->callbacks.remove_port(&lib->unit, port); + } + + if (nxt_fast_path(port != NULL)) { + nxt_unit_port_release(port); } } -static void -nxt_unit_remove_port_unsafe(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, - nxt_unit_port_t *r_port, nxt_unit_process_t **process) +static nxt_unit_port_t * +nxt_unit_remove_port_unsafe(nxt_unit_impl_t *lib, nxt_unit_port_id_t *port_id) { - nxt_unit_impl_t *lib; - nxt_unit_port_impl_t *port; - - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + nxt_unit_port_t *port; port = nxt_unit_port_hash_find(&lib->ports, port_id, 1); if (nxt_slow_path(port == NULL)) { - nxt_unit_debug(ctx, "remove_port: port %d,%d not found", + nxt_unit_debug(NULL, "remove_port: port{%d,%d} not found", (int) port_id->pid, (int) port_id->id); - return; + return NULL; } - nxt_unit_debug(ctx, "remove_port: port %d,%d, fds %d,%d, data %p", + nxt_unit_debug(NULL, "remove_port: port{%d,%d}, fds %d,%d, data %p", (int) port_id->pid, (int) port_id->id, - port->port.in_fd, port->port.out_fd, port->port.data); - - if (port->port.in_fd != -1) { - close(port->port.in_fd); - } - - if (port->port.out_fd != -1) { - close(port->port.out_fd); - } - - if (port->process != NULL) { - nxt_queue_remove(&port->link); - } - - if (process != NULL) { - *process = port->process; - } + port->in_fd, port->out_fd, port->data); - if (r_port != NULL) { - *r_port = port->port; - } - - free(port); + return port; } -void -nxt_unit_remove_pid(nxt_unit_ctx_t *ctx, pid_t pid) +static void +nxt_unit_remove_pid(nxt_unit_impl_t *lib, pid_t pid) { - nxt_unit_impl_t *lib; nxt_unit_process_t *process; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - pthread_mutex_lock(&lib->mutex); - process = nxt_unit_process_find(ctx, pid, 1); + process = nxt_unit_process_find(lib, pid, 1); if (nxt_slow_path(process == NULL)) { - nxt_unit_debug(ctx, "remove_pid: process %d not found", (int) pid); + nxt_unit_debug(NULL, "remove_pid: process %d not found", (int) pid); pthread_mutex_unlock(&lib->mutex); return; } - nxt_unit_remove_process(ctx, process); + nxt_unit_remove_process(lib, process); + + if (lib->callbacks.remove_pid != NULL) { + lib->callbacks.remove_pid(&lib->unit, pid); + } } static void -nxt_unit_remove_process(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process) +nxt_unit_remove_process(nxt_unit_impl_t *lib, nxt_unit_process_t *process) { nxt_queue_t ports; - nxt_unit_impl_t *lib; nxt_unit_port_impl_t *port; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - nxt_queue_init(&ports); nxt_queue_add(&ports, &process->ports); nxt_queue_each(port, &ports, nxt_unit_port_impl_t, link) { - nxt_unit_process_use(ctx, process, -1); - port->process = NULL; - - /* Shortcut for default callback. */ - if (lib->callbacks.remove_port == nxt_unit_remove_port) { - nxt_queue_remove(&port->link); - - nxt_unit_remove_port_unsafe(ctx, &port->port.id, NULL, NULL); - } + nxt_unit_remove_port_unsafe(lib, &port->port.id); } nxt_queue_loop; @@ -4489,70 +5446,168 @@ nxt_unit_remove_process(nxt_unit_ctx_t *ctx, nxt_unit_process_t *process) nxt_queue_remove(&port->link); - lib->callbacks.remove_port(ctx, &port->port.id); + if (lib->callbacks.remove_port != NULL) { + lib->callbacks.remove_port(&lib->unit, &port->port); + } + + nxt_unit_port_release(&port->port); } nxt_queue_loop; - nxt_unit_process_use(ctx, process, -1); + nxt_unit_process_release(process); } -void +static void nxt_unit_quit(nxt_unit_ctx_t *ctx) { nxt_unit_impl_t *lib; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - lib->online = 0; + if (lib->online) { + lib->online = 0; + + if (lib->callbacks.quit != NULL) { + lib->callbacks.quit(ctx); + } + } +} + + +static int +nxt_unit_get_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id) +{ + ssize_t res; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + + struct { + nxt_port_msg_t msg; + nxt_port_msg_get_port_t get_port; + } m; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + memset(&m.msg, 0, sizeof(nxt_port_msg_t)); + + m.msg.pid = lib->pid; + m.msg.reply_port = ctx_impl->read_port->id.id; + m.msg.type = _NXT_PORT_MSG_GET_PORT; + + m.get_port.id = port_id->id; + m.get_port.pid = port_id->pid; + + nxt_unit_debug(ctx, "get_port: %d %d", (int) port_id->pid, + (int) port_id->id); + + res = nxt_unit_port_send(ctx, lib->router_port, &m, sizeof(m), NULL, 0); + if (nxt_slow_path(res != sizeof(m))) { + return NXT_UNIT_ERROR; + } + + return NXT_UNIT_OK; } static ssize_t -nxt_unit_port_send_default(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, +nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, const void *buf, size_t buf_size, const void *oob, size_t oob_size) { - int fd; + int notify; + ssize_t ret; + nxt_int_t rc; + nxt_port_msg_t msg; nxt_unit_impl_t *lib; - nxt_unit_port_impl_t *port; + nxt_unit_port_impl_t *port_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - pthread_mutex_lock(&lib->mutex); + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + if (port_impl->queue != NULL && oob_size == 0 + && buf_size <= NXT_PORT_QUEUE_MSG_SIZE) + { + rc = nxt_port_queue_send(port_impl->queue, buf, buf_size, ¬ify); + if (nxt_slow_path(rc != NXT_OK)) { + nxt_unit_alert(ctx, "port_send: port %d,%d queue overflow", + (int) port->id.pid, (int) port->id.id); - port = nxt_unit_port_hash_find(&lib->ports, port_id, 0); + return -1; + } - if (nxt_fast_path(port != NULL)) { - fd = port->port.out_fd; + nxt_unit_debug(ctx, "port{%d,%d} enqueue %d notify %d", + (int) port->id.pid, (int) port->id.id, + (int) buf_size, notify); - } else { - nxt_unit_warn(ctx, "port_send: port %d,%d not found", - (int) port_id->pid, (int) port_id->id); - fd = -1; + if (notify) { + memcpy(&msg, buf, sizeof(nxt_port_msg_t)); + + msg.type = _NXT_PORT_MSG_READ_QUEUE; + + if (lib->callbacks.port_send == NULL) { + ret = nxt_unit_sendmsg(ctx, port->out_fd, &msg, + sizeof(nxt_port_msg_t), NULL, 0); + + nxt_unit_debug(ctx, "port{%d,%d} send %d read_queue", + (int) port->id.pid, (int) port->id.id, + (int) ret); + + } else { + ret = lib->callbacks.port_send(ctx, port, &msg, + sizeof(nxt_port_msg_t), NULL, 0); + + nxt_unit_debug(ctx, "port{%d,%d} sendcb %d read_queue", + (int) port->id.pid, (int) port->id.id, + (int) ret); + } + + } + + return buf_size; } - pthread_mutex_unlock(&lib->mutex); + if (port_impl->queue != NULL) { + msg.type = _NXT_PORT_MSG_READ_SOCKET; - if (nxt_slow_path(fd == -1)) { - if (port != NULL) { - nxt_unit_warn(ctx, "port_send: port %d,%d: fd == -1", - (int) port_id->pid, (int) port_id->id); + rc = nxt_port_queue_send(port_impl->queue, &msg.type, 1, ¬ify); + if (nxt_slow_path(rc != NXT_OK)) { + nxt_unit_alert(ctx, "port_send: port %d,%d queue overflow", + (int) port->id.pid, (int) port->id.id); + + return -1; } - return -1; + nxt_unit_debug(ctx, "port{%d,%d} enqueue 1 read_socket notify %d", + (int) port->id.pid, (int) port->id.id, notify); } - nxt_unit_debug(ctx, "port_send: found port %d,%d fd %d", - (int) port_id->pid, (int) port_id->id, fd); + if (lib->callbacks.port_send != NULL) { + ret = lib->callbacks.port_send(ctx, port, buf, buf_size, + oob, oob_size); + + nxt_unit_debug(ctx, "port{%d,%d} sendcb %d", + (int) port->id.pid, (int) port->id.id, + (int) ret); + + } else { + ret = nxt_unit_sendmsg(ctx, port->out_fd, buf, buf_size, + oob, oob_size); + + nxt_unit_debug(ctx, "port{%d,%d} sendmsg %d", + (int) port->id.pid, (int) port->id.id, + (int) ret); + } - return nxt_unit_port_send(ctx, fd, buf, buf_size, oob, oob_size); + return ret; } -ssize_t -nxt_unit_port_send(nxt_unit_ctx_t *ctx, int fd, +static ssize_t +nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, const void *buf, size_t buf_size, const void *oob, size_t oob_size) { + int err; ssize_t res; struct iovec iov[1]; struct msghdr msg; @@ -4573,7 +5628,9 @@ retry: res = sendmsg(fd, &msg, 0); if (nxt_slow_path(res == -1)) { - if (errno == EINTR) { + err = errno; + + if (err == EINTR) { goto retry; } @@ -4582,7 +5639,7 @@ retry: * implementation. */ nxt_unit_warn(ctx, "sendmsg(%d, %d) failed: %s (%d)", - fd, (int) buf_size, strerror(errno), errno); + fd, (int) buf_size, strerror(err), err); } else { nxt_unit_debug(ctx, "sendmsg(%d, %d): %d", fd, (int) buf_size, @@ -4593,88 +5650,310 @@ retry: } -static ssize_t -nxt_unit_port_recv_default(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id, - void *buf, size_t buf_size, void *oob, size_t oob_size) +static int +nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf) { - int fd; - nxt_unit_impl_t *lib; - nxt_unit_ctx_impl_t *ctx_impl; - nxt_unit_port_impl_t *port; + int res, read; + nxt_unit_port_impl_t *port_impl; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); - pthread_mutex_lock(&lib->mutex); + read = 0; - port = nxt_unit_port_hash_find(&lib->ports, port_id, 0); +retry: - if (nxt_fast_path(port != NULL)) { - fd = port->port.in_fd; + if (port_impl->from_socket > 0) { + if (port_impl->socket_rbuf != NULL + && port_impl->socket_rbuf->size > 0) + { + port_impl->from_socket--; + + nxt_unit_rbuf_cpy(rbuf, port_impl->socket_rbuf); + port_impl->socket_rbuf->size = 0; + + nxt_unit_debug(ctx, "port{%d,%d} use suspended message %d", + (int) port->id.pid, (int) port->id.id, + (int) rbuf->size); + + return NXT_UNIT_OK; + } } else { - nxt_unit_debug(ctx, "port_recv: port %d,%d not found", - (int) port_id->pid, (int) port_id->id); - fd = -1; + res = nxt_unit_port_queue_recv(port, rbuf); + + if (res == NXT_UNIT_OK) { + if (nxt_unit_is_read_socket(rbuf)) { + port_impl->from_socket++; + + nxt_unit_debug(ctx, "port{%d,%d} dequeue 1 read_socket %d", + (int) port->id.pid, (int) port->id.id, + port_impl->from_socket); + + goto retry; + } + + nxt_unit_debug(ctx, "port{%d,%d} dequeue %d", + (int) port->id.pid, (int) port->id.id, + (int) rbuf->size); + + return NXT_UNIT_OK; + } } - pthread_mutex_unlock(&lib->mutex); + if (read) { + return NXT_UNIT_AGAIN; + } - if (nxt_slow_path(fd == -1)) { - return -1; + res = nxt_unit_port_recv(ctx, port, rbuf); + if (nxt_slow_path(res == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; } - nxt_unit_debug(ctx, "port_recv: found port %d,%d, fd %d", - (int) port_id->pid, (int) port_id->id, fd); + read = 1; - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + if (nxt_unit_is_read_queue(rbuf)) { + nxt_unit_debug(ctx, "port{%d,%d} recv %d read_queue", + (int) port->id.pid, (int) port->id.id, (int) rbuf->size); + + if (port_impl->from_socket) { + nxt_unit_warn(ctx, "port protocol warning: READ_QUEUE after READ_SOCKET"); + } + + goto retry; + } + + nxt_unit_debug(ctx, "port{%d,%d} recvmsg %d", + (int) port->id.pid, (int) port->id.id, + (int) rbuf->size); + + if (res == NXT_UNIT_AGAIN) { + return NXT_UNIT_AGAIN; + } + + if (port_impl->from_socket > 0) { + port_impl->from_socket--; + + return NXT_UNIT_OK; + } + + nxt_unit_debug(ctx, "port{%d,%d} suspend message %d", + (int) port->id.pid, (int) port->id.id, + (int) rbuf->size); + + if (port_impl->socket_rbuf == NULL) { + port_impl->socket_rbuf = nxt_unit_read_buf_get(ctx); + + if (nxt_slow_path(port_impl->socket_rbuf == NULL)) { + return NXT_UNIT_ERROR; + } + + port_impl->socket_rbuf->size = 0; + } + + if (port_impl->socket_rbuf->size > 0) { + nxt_unit_alert(ctx, "too many port socket messages"); + + return NXT_UNIT_ERROR; + } - if (nxt_fast_path(port_id == &ctx_impl->read_port_id)) { - ctx_impl->read_port_fd = fd; + nxt_unit_rbuf_cpy(port_impl->socket_rbuf, rbuf); + + memset(rbuf->oob, 0, sizeof(struct cmsghdr)); + + goto retry; +} + + +nxt_inline void +nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst, nxt_unit_read_buf_t *src) +{ + memcpy(dst->buf, src->buf, src->size); + dst->size = src->size; + memcpy(dst->oob, src->oob, sizeof(src->oob)); +} + + +static int +nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf) +{ + int res; + +retry: + + res = nxt_unit_app_queue_recv(port, rbuf); + + if (res == NXT_UNIT_AGAIN) { + res = nxt_unit_port_recv(ctx, port, rbuf); + if (nxt_slow_path(res == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; + } + + if (nxt_unit_is_read_queue(rbuf)) { + nxt_unit_debug(ctx, "port{%d,%d} recv %d read_queue", + (int) port->id.pid, (int) port->id.id, (int) rbuf->size); + + goto retry; + } } - return nxt_unit_port_recv(ctx, fd, buf, buf_size, oob, oob_size); + return res; } -ssize_t -nxt_unit_port_recv(nxt_unit_ctx_t *ctx, int fd, void *buf, size_t buf_size, - void *oob, size_t oob_size) +static int +nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + nxt_unit_read_buf_t *rbuf) { - ssize_t res; - struct iovec iov[1]; - struct msghdr msg; + int fd, err; + struct iovec iov[1]; + struct msghdr msg; + nxt_unit_impl_t *lib; - iov[0].iov_base = buf; - iov[0].iov_len = buf_size; + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + if (lib->callbacks.port_recv != NULL) { + rbuf->size = lib->callbacks.port_recv(ctx, port, + rbuf->buf, sizeof(rbuf->buf), + rbuf->oob, sizeof(rbuf->oob)); + + nxt_unit_debug(ctx, "port{%d,%d} recvcb %d", + (int) port->id.pid, (int) port->id.id, (int) rbuf->size); + + if (nxt_slow_path(rbuf->size < 0)) { + return NXT_UNIT_ERROR; + } + + return NXT_UNIT_OK; + } + + iov[0].iov_base = rbuf->buf; + iov[0].iov_len = sizeof(rbuf->buf); msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_iov = iov; msg.msg_iovlen = 1; msg.msg_flags = 0; - msg.msg_control = oob; - msg.msg_controllen = oob_size; + msg.msg_control = rbuf->oob; + msg.msg_controllen = sizeof(rbuf->oob); + + fd = port->in_fd; retry: - res = recvmsg(fd, &msg, 0); + rbuf->size = recvmsg(fd, &msg, 0); - if (nxt_slow_path(res == -1)) { - if (errno == EINTR) { + if (nxt_slow_path(rbuf->size == -1)) { + err = errno; + + if (err == EINTR) { goto retry; } + if (err == EAGAIN) { + nxt_unit_debug(ctx, "recvmsg(%d) failed: %s (%d)", + fd, strerror(err), err); + + return NXT_UNIT_AGAIN; + } + nxt_unit_alert(ctx, "recvmsg(%d) failed: %s (%d)", + fd, strerror(err), err); + + return NXT_UNIT_ERROR; + } + + nxt_unit_debug(ctx, "recvmsg(%d): %d", fd, (int) rbuf->size); + + return NXT_UNIT_OK; +} + + +static int +nxt_unit_port_queue_recv(nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf) +{ + nxt_unit_port_impl_t *port_impl; + + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + + rbuf->size = nxt_port_queue_recv(port_impl->queue, rbuf->buf); + + return (rbuf->size == -1) ? NXT_UNIT_AGAIN : NXT_UNIT_OK; +} + + +static int +nxt_unit_app_queue_recv(nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf) +{ + uint32_t cookie; + nxt_port_msg_t *port_msg; + nxt_app_queue_t *queue; + nxt_unit_port_impl_t *port_impl; + + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + queue = port_impl->queue; + +retry: + + rbuf->size = nxt_app_queue_recv(queue, rbuf->buf, &cookie); + + nxt_unit_debug(NULL, "app_queue_recv: %d", (int) rbuf->size); + + if (rbuf->size >= (ssize_t) sizeof(nxt_port_msg_t)) { + port_msg = (nxt_port_msg_t *) rbuf->buf; + + if (nxt_app_queue_cancel(queue, cookie, port_msg->stream)) { + return NXT_UNIT_OK; + } + + nxt_unit_debug(NULL, "app_queue_recv: message cancelled"); + + goto retry; + } + + return (rbuf->size == -1) ? NXT_UNIT_AGAIN : NXT_UNIT_OK; +} + + +nxt_inline int +nxt_unit_close(int fd) +{ + int res; + + res = close(fd); + + if (nxt_slow_path(res == -1)) { + nxt_unit_alert(NULL, "close(%d) failed: %s (%d)", fd, strerror(errno), errno); } else { - nxt_unit_debug(ctx, "recvmsg(%d): %d", fd, (int) res); + nxt_unit_debug(NULL, "close(%d): %d", fd, res); } return res; } +static int +nxt_unit_fd_blocking(int fd) +{ + int nb; + + nb = 0; + + if (nxt_slow_path(ioctl(fd, FIONBIO, &nb) == -1)) { + nxt_unit_alert(NULL, "ioctl(%d, FIONBIO, 0) failed: %s (%d)", + fd, strerror(errno), errno); + + return NXT_UNIT_ERROR; + } + + return NXT_UNIT_OK; +} + + static nxt_int_t nxt_unit_port_hash_test(nxt_lvlhsh_query_t *lhq, void *data) { @@ -4755,7 +6034,7 @@ nxt_unit_port_hash_add(nxt_lvlhsh_t *port_hash, nxt_unit_port_t *port) } -static nxt_unit_port_impl_t * +static nxt_unit_port_t * nxt_unit_port_hash_find(nxt_lvlhsh_t *port_hash, nxt_unit_port_id_t *port_id, int remove) { @@ -4775,6 +6054,10 @@ nxt_unit_port_hash_find(nxt_lvlhsh_t *port_hash, nxt_unit_port_id_t *port_id, switch (res) { case NXT_OK: + if (!remove) { + nxt_unit_port_use(lhq.value); + } + return lhq.value; default: @@ -4799,12 +6082,19 @@ static const nxt_lvlhsh_proto_t lvlhsh_requests_proto nxt_aligned(64) = { static int -nxt_unit_request_hash_add(nxt_lvlhsh_t *request_hash, - nxt_unit_request_info_impl_t *req_impl) +nxt_unit_request_hash_add(nxt_unit_ctx_t *ctx, + nxt_unit_request_info_t *req) { - uint32_t *stream; - nxt_int_t res; - nxt_lvlhsh_query_t lhq; + uint32_t *stream; + nxt_int_t res; + nxt_lvlhsh_query_t lhq; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_impl_t *req_impl; + + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); + if (req_impl->in_hash) { + return NXT_UNIT_OK; + } stream = &req_impl->stream; @@ -4816,11 +6106,18 @@ nxt_unit_request_hash_add(nxt_lvlhsh_t *request_hash, lhq.replace = 0; lhq.value = req_impl; - res = nxt_lvlhsh_insert(request_hash, &lhq); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + pthread_mutex_lock(&ctx_impl->mutex); + + res = nxt_lvlhsh_insert(&ctx_impl->requests, &lhq); + + pthread_mutex_unlock(&ctx_impl->mutex); switch (res) { case NXT_OK: + req_impl->in_hash = 1; return NXT_UNIT_OK; default: @@ -4829,12 +6126,13 @@ nxt_unit_request_hash_add(nxt_lvlhsh_t *request_hash, } -static nxt_unit_request_info_impl_t * -nxt_unit_request_hash_find(nxt_lvlhsh_t *request_hash, uint32_t stream, - int remove) +static nxt_unit_request_info_t * +nxt_unit_request_hash_find(nxt_unit_ctx_t *ctx, uint32_t stream, int remove) { - nxt_int_t res; - nxt_lvlhsh_query_t lhq; + nxt_int_t res; + nxt_lvlhsh_query_t lhq; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_impl_t *req_impl; lhq.key_hash = nxt_murmur_hash2(&stream, sizeof(stream)); lhq.key.length = sizeof(stream); @@ -4842,16 +6140,26 @@ nxt_unit_request_hash_find(nxt_lvlhsh_t *request_hash, uint32_t stream, lhq.proto = &lvlhsh_requests_proto; lhq.pool = NULL; + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + pthread_mutex_lock(&ctx_impl->mutex); + if (remove) { - res = nxt_lvlhsh_delete(request_hash, &lhq); + res = nxt_lvlhsh_delete(&ctx_impl->requests, &lhq); } else { - res = nxt_lvlhsh_find(request_hash, &lhq); + res = nxt_lvlhsh_find(&ctx_impl->requests, &lhq); } + pthread_mutex_unlock(&ctx_impl->mutex); + switch (res) { case NXT_OK: + req_impl = nxt_container_of(lhq.value, nxt_unit_request_info_impl_t, + req); + req_impl->in_hash = 0; + return lhq.value; default: @@ -4977,11 +6285,18 @@ nxt_unit_snprint_prefix(char *p, char *end, pid_t pid, int level) tm = *localtime(&ts.tv_sec); #endif +#if (NXT_DEBUG) p += snprintf(p, end - p, "%4d/%02d/%02d %02d:%02d:%02d.%03d ", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, (int) ts.tv_nsec / 1000000); +#else + p += snprintf(p, end - p, + "%4d/%02d/%02d %02d:%02d:%02d ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec); +#endif p += snprintf(p, end - p, "[%s] %d#%"PRIu64" [unit] ", nxt_unit_log_levels[level], diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 596dd8b6..67244cf4 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -19,6 +19,8 @@ enum { NXT_UNIT_OK = 0, NXT_UNIT_ERROR = 1, + NXT_UNIT_AGAIN = 2, + NXT_UNIT_CANCELLED = 3, }; enum { @@ -91,8 +93,7 @@ struct nxt_unit_request_info_s { nxt_unit_t *unit; nxt_unit_ctx_t *ctx; - nxt_unit_port_id_t request_port; - nxt_unit_port_id_t response_port; + nxt_unit_port_t *response_port; nxt_unit_request_t *request; nxt_unit_buf_t *request_buf; @@ -120,6 +121,8 @@ struct nxt_unit_callbacks_s { */ void (*request_handler)(nxt_unit_request_info_t *req); + void (*data_handler)(nxt_unit_request_info_t *req); + /* Process websocket frame. */ void (*websocket_handler)(nxt_unit_websocket_frame_t *ws); @@ -130,26 +133,25 @@ struct nxt_unit_callbacks_s { int (*add_port)(nxt_unit_ctx_t *, nxt_unit_port_t *port); /* Remove previously added port. Optional. */ - void (*remove_port)(nxt_unit_ctx_t *, nxt_unit_port_id_t *port_id); + void (*remove_port)(nxt_unit_t *, nxt_unit_port_t *port); /* Remove all data associated with process pid including ports. Optional. */ - void (*remove_pid)(nxt_unit_ctx_t *, pid_t pid); + void (*remove_pid)(nxt_unit_t *, pid_t pid); /* Gracefully quit the application. Optional. */ void (*quit)(nxt_unit_ctx_t *); - /* Shared memory release acknowledgement. */ + /* Shared memory release acknowledgement. Optional. */ void (*shm_ack_handler)(nxt_unit_ctx_t *); /* Send data and control to process pid using port id. Optional. */ - ssize_t (*port_send)(nxt_unit_ctx_t *, nxt_unit_port_id_t *port_id, + ssize_t (*port_send)(nxt_unit_ctx_t *, nxt_unit_port_t *port, const void *buf, size_t buf_size, const void *oob, size_t oob_size); /* Receive data on port id. Optional. */ - ssize_t (*port_recv)(nxt_unit_ctx_t *, nxt_unit_port_id_t *port_id, + ssize_t (*port_recv)(nxt_unit_ctx_t *, nxt_unit_port_t *port, void *buf, size_t buf_size, void *oob, size_t oob_size); - }; @@ -165,6 +167,7 @@ struct nxt_unit_init_s { nxt_unit_port_t ready_port; uint32_t ready_stream; + nxt_unit_port_t router_port; nxt_unit_port_t read_port; int log_fd; }; @@ -189,16 +192,6 @@ struct nxt_unit_read_info_s { nxt_unit_ctx_t *nxt_unit_init(nxt_unit_init_t *); /* - * Process received message, invoke configured callbacks. - * - * If application implements it's own event loop, each datagram received - * from port socket should be initially processed by unit. This function - * may invoke other application-defined callback for message processing. - */ -int nxt_unit_process_msg(nxt_unit_ctx_t *, nxt_unit_port_id_t *port_id, - void *buf, size_t buf_size, void *oob, size_t oob_size); - -/* * Main function useful in case when application does not have it's own * event loop. nxt_unit_run() starts infinite message wait and process loop. * @@ -211,8 +204,21 @@ int nxt_unit_process_msg(nxt_unit_ctx_t *, nxt_unit_port_id_t *port_id, */ int nxt_unit_run(nxt_unit_ctx_t *); +int nxt_unit_run_ctx(nxt_unit_ctx_t *ctx); + +int nxt_unit_run_shared(nxt_unit_ctx_t *ctx); + +/* + * Receive and process one message, invoke configured callbacks. + * + * If application implements it's own event loop, each datagram received + * from port socket should be initially processed by unit. This function + * may invoke other application-defined callback for message processing. + */ int nxt_unit_run_once(nxt_unit_ctx_t *ctx); +int nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); + /* Destroy application library object. */ void nxt_unit_done(nxt_unit_ctx_t *); @@ -222,49 +228,9 @@ void nxt_unit_done(nxt_unit_ctx_t *); */ nxt_unit_ctx_t *nxt_unit_ctx_alloc(nxt_unit_ctx_t *, void *); -/* Free unused context. It is not required to free main context. */ -void nxt_unit_ctx_free(nxt_unit_ctx_t *); - /* Initialize port_id, calculate hash. */ void nxt_unit_port_id_init(nxt_unit_port_id_t *port_id, pid_t pid, uint16_t id); -/* - * Create extra incoming port, perform all required actions to propogate - * the port to Unit server. Fills structure referenced by port_id with - * current pid and new port id. - */ -int nxt_unit_create_send_port(nxt_unit_ctx_t *, nxt_unit_port_id_t *dst, - nxt_unit_port_id_t *port_id); - -/* Default 'add_port' implementation. */ -int nxt_unit_add_port(nxt_unit_ctx_t *, nxt_unit_port_t *port); - -/* Find previously added port. */ -nxt_unit_port_t *nxt_unit_find_port(nxt_unit_ctx_t *, - nxt_unit_port_id_t *port_id); - -/* Find, fill output 'port' and remove port from storage. */ -void nxt_unit_find_remove_port(nxt_unit_ctx_t *, nxt_unit_port_id_t *port_id, - nxt_unit_port_t *port); - -/* Default 'remove_port' implementation. */ -void nxt_unit_remove_port(nxt_unit_ctx_t *, nxt_unit_port_id_t *port_id); - -/* Default 'remove_pid' implementation. */ -void nxt_unit_remove_pid(nxt_unit_ctx_t *, pid_t pid); - -/* Default 'quit' implementation. */ -void nxt_unit_quit(nxt_unit_ctx_t *); - -/* Default 'port_send' implementation. */ -ssize_t nxt_unit_port_send(nxt_unit_ctx_t *, int fd, - const void *buf, size_t buf_size, - const void *oob, size_t oob_size); - -/* Default 'port_recv' implementation. */ -ssize_t nxt_unit_port_recv(nxt_unit_ctx_t *, int fd, void *buf, size_t buf_size, - void *oob, size_t oob_size); - /* Calculates hash for given field name. */ uint16_t nxt_unit_field_hash(const char* name, size_t name_length); diff --git a/src/nxt_upstream.c b/src/nxt_upstream.c index 66b6619a..c8ecbbe6 100644 --- a/src/nxt_upstream.c +++ b/src/nxt_upstream.c @@ -71,7 +71,7 @@ nxt_upstreams_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, } -void +nxt_int_t nxt_upstream_find(nxt_upstreams_t *upstreams, nxt_str_t *name, nxt_http_action_t *action) { @@ -86,9 +86,11 @@ nxt_upstream_find(nxt_upstreams_t *upstreams, nxt_str_t *name, action->u.upstream_number = i; action->handler = nxt_upstream_handler; - return; + return NXT_DECLINED; } } + + return NXT_OK; } diff --git a/src/nxt_var.c b/src/nxt_var.c new file mode 100644 index 00000000..2731fd09 --- /dev/null +++ b/src/nxt_var.c @@ -0,0 +1,616 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +struct nxt_var_s { + size_t plain; + nxt_uint_t vars; + u_char data[]; + +/* + uint32_t indexes[vars]; + size_t positions[vars]; + u_char chars[plain]; +*/ +}; + + +typedef struct { + nxt_var_t *var; + nxt_str_t *value; +} nxt_var_value_t; + + +struct nxt_var_query_s { + nxt_array_t values; /* of nxt_var_value_t */ + nxt_array_t parts; /* of nxt_str_t * */ + + nxt_lvlhsh_t cache; + + nxt_str_t *spare; + nxt_uint_t waiting; + nxt_uint_t failed; /* 1 bit */ + + void *ctx; + void *data; + + nxt_work_handler_t ready; + nxt_work_handler_t error; +}; + + +#define nxt_var_indexes(var) ((uint32_t *) (var)->data) + +#define nxt_var_positions(var) \ + ((size_t *) ((var)->data + (var)->vars * sizeof(uint32_t))) + +#define nxt_var_plain_start(var) \ + ((var)->data + (var)->vars * (sizeof(uint32_t) + sizeof(size_t))) + + +static nxt_int_t nxt_var_hash_test(nxt_lvlhsh_query_t *lhq, void *data); +static nxt_var_decl_t *nxt_var_hash_find(nxt_str_t *name); + +static nxt_int_t nxt_var_cache_test(nxt_lvlhsh_query_t *lhq, void *data); +static nxt_str_t *nxt_var_cache_find(nxt_lvlhsh_t *lh, uint32_t index); +static nxt_int_t nxt_var_cache_add(nxt_lvlhsh_t *lh, uint32_t index, + nxt_str_t *value, nxt_mp_t *mp); + +static u_char *nxt_var_next_part(u_char *start, size_t length, nxt_str_t *part, + nxt_bool_t *is_var); + +static void nxt_var_query_finish(nxt_task_t *task, nxt_var_query_t *query); + + +static const nxt_lvlhsh_proto_t nxt_var_hash_proto nxt_aligned(64) = { + NXT_LVLHSH_DEFAULT, + nxt_var_hash_test, + nxt_lvlhsh_alloc, + nxt_lvlhsh_free, +}; + +static const nxt_lvlhsh_proto_t nxt_var_cache_proto nxt_aligned(64) = { + NXT_LVLHSH_DEFAULT, + nxt_var_cache_test, + nxt_mp_lvlhsh_alloc, + nxt_mp_lvlhsh_free, +}; + + +static nxt_lvlhsh_t nxt_var_hash; +static uint32_t nxt_var_count; + +static nxt_var_handler_t *nxt_var_index; + + +static nxt_int_t +nxt_var_hash_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + nxt_var_decl_t *decl; + + decl = data; + + return nxt_strstr_eq(&lhq->key, &decl->name) ? NXT_OK : NXT_DECLINED; +} + + +static nxt_var_decl_t * +nxt_var_hash_find(nxt_str_t *name) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(name->start, name->length); + lhq.key = *name; + lhq.proto = &nxt_var_hash_proto; + + if (nxt_lvlhsh_find(&nxt_var_hash, &lhq) != NXT_OK) { + return NULL; + } + + return lhq.value; +} + + +static nxt_int_t +nxt_var_cache_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + return NXT_OK; +} + + +static nxt_str_t * +nxt_var_cache_find(nxt_lvlhsh_t *lh, uint32_t index) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_murmur_hash2_uint32(&index); + lhq.key.length = sizeof(uint32_t); + lhq.key.start = (u_char *) &index; + lhq.proto = &nxt_var_cache_proto; + + if (nxt_lvlhsh_find(lh, &lhq) != NXT_OK) { + return NULL; + } + + return lhq.value; +} + + +static nxt_int_t +nxt_var_cache_add(nxt_lvlhsh_t *lh, uint32_t index, nxt_str_t *value, + nxt_mp_t *mp) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_murmur_hash2_uint32(&index); + lhq.replace = 0; + lhq.key.length = sizeof(uint32_t); + lhq.key.start = (u_char *) &index; + lhq.value = value; + lhq.proto = &nxt_var_cache_proto; + lhq.pool = mp; + + return nxt_lvlhsh_insert(lh, &lhq); +} + + +nxt_int_t +nxt_var_register(nxt_var_decl_t *decl, size_t n) +{ + nxt_uint_t i; + nxt_lvlhsh_query_t lhq; + + lhq.replace = 0; + lhq.proto = &nxt_var_hash_proto; + + for (i = 0; i < n; i++) { + lhq.key = decl[i].name; + lhq.key_hash = nxt_djb_hash(lhq.key.start, lhq.key.length); + lhq.value = &decl[i]; + + if (nxt_slow_path(nxt_lvlhsh_insert(&nxt_var_hash, &lhq) != NXT_OK)) { + return NXT_ERROR; + } + } + + nxt_var_count += n; + + return NXT_OK; +} + + +nxt_int_t +nxt_var_index_init(void) +{ + nxt_uint_t i; + nxt_var_decl_t *decl; + nxt_var_handler_t *index; + nxt_lvlhsh_each_t lhe; + + index = nxt_memalign(64, nxt_var_count * sizeof(nxt_var_handler_t)); + if (index == NULL) { + return NXT_ERROR; + } + + nxt_lvlhsh_each_init(&lhe, &nxt_var_hash_proto); + + for (i = 0; i < nxt_var_count; i++) { + decl = nxt_lvlhsh_each(&nxt_var_hash, &lhe); + decl->index = i; + index[i] = decl->handler; + } + + nxt_var_index = index; + + return NXT_OK; +} + + +nxt_var_t * +nxt_var_compile(nxt_str_t *str, nxt_mp_t *mp) +{ + u_char *p, *end, *plain_pos; + size_t plain, size, *positions; + uint32_t *indexes; + nxt_var_t *var; + nxt_str_t part; + nxt_uint_t n; + nxt_bool_t is_var; + nxt_var_decl_t *decl; + + plain = 0; + n = 0; + + p = str->start; + end = p + str->length; + + while (p < end) { + p = nxt_var_next_part(p, end - p, &part, &is_var); + + if (nxt_slow_path(p == NULL)) { + return NULL; + } + + if (is_var) { + n++; + + } else { + plain += part.length; + } + } + + size = sizeof(nxt_var_t) + + n * (sizeof(nxt_var_handler_t) + sizeof (size_t)) + + plain; + + var = nxt_mp_get(mp, size); + if (nxt_slow_path(var == NULL)) { + return NULL; + } + + var->plain = plain; + var->vars = n; + + indexes = nxt_var_indexes(var); + positions = nxt_var_positions(var); + plain_pos = nxt_var_plain_start(var); + + plain = 0; + n = 0; + + p = str->start; + + while (p < end) { + p = nxt_var_next_part(p, end - p, &part, &is_var); + + if (is_var) { + decl = nxt_var_hash_find(&part); + + if (nxt_slow_path(decl == NULL)) { + return NULL; + } + + indexes[n] = decl->index; + positions[n] = plain; + + n++; + + } else { + plain_pos = nxt_cpymem(plain_pos, part.start, part.length); + plain += part.length; + } + } + + return var; +} + + +nxt_int_t +nxt_var_test(nxt_str_t *str, u_char *error) +{ + u_char *p, *end, *next; + nxt_str_t part; + nxt_bool_t is_var; + nxt_var_decl_t *decl; + + p = str->start; + end = p + str->length; + + while (p < end) { + next = nxt_var_next_part(p, end - p, &part, &is_var); + + if (next == NULL) { + nxt_sprintf(error, error + NXT_MAX_ERROR_STR, + "Invalid variable at position %uz%Z", p - str->start); + + return NXT_ERROR; + } + + if (is_var) { + decl = nxt_var_hash_find(&part); + + if (decl == NULL) { + nxt_sprintf(error, error + NXT_MAX_ERROR_STR, + "Unknown variable \"%V\"%Z", &part); + + return NXT_ERROR; + } + } + + p = next; + } + + return NXT_OK; +} + + +static u_char * +nxt_var_next_part(u_char *start, size_t length, nxt_str_t *part, + nxt_bool_t *is_var) +{ + u_char *p, *end, ch, c; + nxt_bool_t bracket; + + end = start + length; + + p = nxt_memchr(start, '$', length); + + if (p == start) { + *is_var = 1; + + p++; + + if (p == end) { + return NULL; + } + + if (*p == '{') { + bracket = 1; + + if (end - p < 2) { + return NULL; + } + + p++; + + } else { + bracket = 0; + } + + start = p; + + for ( ;; ) { + ch = *p; + + c = (u_char) (ch | 0x20); + if ((c < 'a' || c > 'z') && ch != '_') { + + if (bracket && ch != '}') { + return NULL; + } + + break; + } + + p++; + + if (p == end) { + if (bracket) { + return NULL; + } + + break; + } + } + + length = p - start; + end = p + bracket; + + } else { + *is_var = 0; + + if (p != NULL) { + length = p - start; + end = p; + } + } + + part->length = length; + part->start = start; + + return end; +} + + +nxt_int_t +nxt_var_query_init(nxt_var_query_t **query_p, void *ctx, nxt_mp_t *mp) +{ + nxt_var_query_t *query; + + query = *query_p; + + if (*query_p == NULL) { + query = nxt_mp_zget(mp, sizeof(nxt_var_query_t)); + if (nxt_slow_path(query == NULL)) { + return NXT_ERROR; + } + + nxt_array_init(&query->values, mp, sizeof(nxt_var_value_t)); + nxt_array_init(&query->parts, mp, sizeof(nxt_str_t *)); + + } else { + nxt_array_reset(&query->values); + } + + query->ctx = ctx; + + *query_p = query; + + return NXT_OK; +} + + +void +nxt_var_query(nxt_task_t *task, nxt_var_query_t *query, nxt_var_t *var, + nxt_str_t *str) +{ + uint32_t *indexes; + nxt_mp_t *mp; + nxt_str_t *value; + nxt_int_t ret; + nxt_uint_t i; + nxt_var_value_t *val; + + if (var->vars == 0) { + str->length = var->plain; + str->start = nxt_var_plain_start(var); + return; + } + + if (nxt_slow_path(query->failed)) { + return; + } + + mp = query->values.mem_pool; + indexes = nxt_var_indexes(var); + value = query->spare; + + for (i = 0; i < var->vars; i++) { + + if (value == NULL) { + value = nxt_mp_zget(mp, sizeof(nxt_str_t)); + if (nxt_slow_path(value == NULL)) { + goto fail; + } + } + + ret = nxt_var_cache_add(&query->cache, indexes[i], value, mp); + + if (ret != NXT_OK) { + if (nxt_slow_path(ret == NXT_ERROR)) { + goto fail; + } + + continue; /* NXT_DECLINED */ + } + + ret = nxt_var_index[indexes[i]](task, query, value, query->ctx); + + value = NULL; + + if (ret != NXT_OK) { + if (nxt_slow_path(ret != NXT_AGAIN)) { + goto fail; + } + + query->waiting++; + } + } + + query->spare = value; + + val = nxt_array_add(&query->values); + if (nxt_slow_path(val == NULL)) { + goto fail; + } + + val->var = var; + val->value = str; + + return; + +fail: + + query->failed = 1; +} + + +void +nxt_var_query_resolve(nxt_task_t *task, nxt_var_query_t *query, void *data, + nxt_work_handler_t ready, nxt_work_handler_t error) +{ + query->data = data; + query->ready = ready; + query->error = error; + + if (query->waiting == 0) { + nxt_var_query_finish(task, query); + } +} + + +void +nxt_var_query_handle(nxt_task_t *task, nxt_var_query_t *query, + nxt_bool_t failed) +{ + query->failed |= failed; + + if (--query->waiting == 0) { + nxt_var_query_finish(task, query); + } +} + + +static void +nxt_var_query_finish(nxt_task_t *task, nxt_var_query_t *query) +{ + u_char *p, *src; + size_t length, plain, next, *positions; + uint32_t *indexes; + nxt_str_t *str, **part; + nxt_var_t *var; + nxt_uint_t i, j; + nxt_var_value_t *val; + + if (query->failed) { + goto done; + } + + val = query->values.elts; + + for (i = 0; i < query->values.nelts; i++) { + var = val[i].var; + + length = var->plain; + indexes = nxt_var_indexes(var); + + for (j = 0; j < var->vars; j++) { + str = nxt_var_cache_find(&query->cache, indexes[j]); + + nxt_assert(str != NULL); + + part = nxt_array_add(&query->parts); + + if (nxt_slow_path(part == NULL)) { + query->failed = 1; + goto done; + } + + *part = str; + + length += str->length; + } + + p = nxt_mp_nget(query->values.mem_pool, length); + if (nxt_slow_path(p == NULL)) { + query->failed = 1; + goto done; + } + + val[i].value->length = length; + val[i].value->start = p; + + part = query->parts.elts; + positions = nxt_var_positions(var); + src = nxt_var_plain_start(var); + + plain = 0; + + for (j = 0; j < var->vars; j++) { + next = positions[j]; + + if (next != plain) { + p = nxt_cpymem(p, &src[plain], next - plain); + plain = next; + } + + p = nxt_cpymem(p, part[j]->start, part[j]->length); + } + + if (plain != var->plain) { + nxt_memcpy(p, &src[plain], var->plain - plain); + } + + nxt_array_reset(&query->parts); + } + +done: + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + query->failed ? query->error : query->ready, + task, query->ctx, query->data); +} diff --git a/src/nxt_var.h b/src/nxt_var.h new file mode 100644 index 00000000..7e0a2a21 --- /dev/null +++ b/src/nxt_var.h @@ -0,0 +1,48 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_VAR_H_INCLUDED_ +#define _NXT_VAR_H_INCLUDED_ + + +typedef struct nxt_var_s nxt_var_t; +typedef struct nxt_var_query_s nxt_var_query_t; + + +typedef nxt_int_t (*nxt_var_handler_t)(nxt_task_t *task, + nxt_var_query_t *query, + nxt_str_t *str, + void *ctx); + +typedef struct { + nxt_str_t name; + nxt_var_handler_t handler; + uint32_t index; +} nxt_var_decl_t; + + +nxt_inline nxt_bool_t +nxt_is_var(nxt_str_t *str) +{ + return (nxt_memchr(str->start, '$', str->length) != NULL); +} + + +nxt_int_t nxt_var_register(nxt_var_decl_t *decl, size_t n); +nxt_int_t nxt_var_index_init(void); +nxt_var_t *nxt_var_compile(nxt_str_t *str, nxt_mp_t *mp); +nxt_int_t nxt_var_test(nxt_str_t *str, u_char *error); + +nxt_int_t nxt_var_query_init(nxt_var_query_t **query_p, void *ctx, + nxt_mp_t *mp); +void nxt_var_query(nxt_task_t *task, nxt_var_query_t *query, + nxt_var_t *var, nxt_str_t *str); +void nxt_var_query_resolve(nxt_task_t *task, nxt_var_query_t *query, void *data, + nxt_work_handler_t ready, nxt_work_handler_t error); +void nxt_var_query_handle(nxt_task_t *task, nxt_var_query_t *query, + nxt_bool_t failed); + + +#endif /* _NXT_VAR_H_INCLUDED_ */ diff --git a/src/perl/nxt_perl_psgi.c b/src/perl/nxt_perl_psgi.c index 14e107e4..16079a38 100644 --- a/src/perl/nxt_perl_psgi.c +++ b/src/perl/nxt_perl_psgi.c @@ -118,12 +118,8 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { nxt_perl_psgi_compat, nxt_string("perl"), PERL_VERSION_STRING, - -#if (NXT_HAVE_ISOLATION_ROOTFS) NULL, 0, -#endif - NULL, nxt_perl_psgi_start, }; diff --git a/src/ruby/nxt_ruby.c b/src/ruby/nxt_ruby.c index 489ddcf4..743bf646 100644 --- a/src/ruby/nxt_ruby.c +++ b/src/ruby/nxt_ruby.c @@ -7,7 +7,8 @@ #include <nxt_unit.h> #include <nxt_unit_request.h> -#include <nxt_ruby_mounts.h> + +#include NXT_RUBY_MOUNTS_H #define NXT_RUBY_RACK_API_VERSION_MAJOR 1 @@ -79,10 +80,8 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { compat, nxt_string("ruby"), ruby_version, -#if (NXT_HAVE_ISOLATION_ROOTFS) nxt_ruby_mounts, nxt_nitems(nxt_ruby_mounts), -#endif NULL, nxt_ruby_start, }; diff --git a/src/test/nxt_cq_test.c b/src/test/nxt_cq_test.c new file mode 100644 index 00000000..ae69505a --- /dev/null +++ b/src/test/nxt_cq_test.c @@ -0,0 +1,578 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <math.h> +#include <inttypes.h> + +#ifndef NXT_NCQ_TEST +#define NXT_NCQ_TEST 1 +#endif + +#define NXT_QTEST_USE_THREAD 0 + +#if NXT_NCQ_TEST +#include <nxt_nncq.h> +#else +#include <nxt_nvbcq.h> +#endif + + +#define MAX_ITER 20 +#define STAT_ITER 5 +#define MIN_COV 0.02 + +extern char **environ; +static uintptr_t nops = 10000000; + +static uintptr_t nprocs_enq = 0; +static uintptr_t nprocs_deq = 0; +static uintptr_t nprocs_wenq = 0; +static uintptr_t nprocs_wdeq = 0; +static uintptr_t nprocs_enq_deq = 0; +static uintptr_t nprocs_cas = 0; +static uintptr_t nprocs_faa = 0; + +static uintptr_t nprocs = 1; + + +static size_t +elapsed_time(size_t us) +{ + struct timeval t; + + gettimeofday(&t, NULL); + + return t.tv_sec * 1000000 + t.tv_usec - us; +} + + +static double +mean(const double *times, int n) +{ + int i; + double sum; + + sum = 0; + + for (i = 0; i < n; i++) { + sum += times[i]; + } + + return sum / n; +} + + +static double +cov(const double *times, double mean, int n) +{ + int i; + double variance; + + variance = 0; + + for (i = 0; i < n; i++) { + variance += (times[i] - mean) * (times[i] - mean); + } + + variance /= n; + + return sqrt(variance) / mean; +} + +typedef struct { +#if NXT_NCQ_TEST + nxt_nncq_t free_queue; + nxt_nncq_t active_queue; +#else + nxt_nvbcq_t free_queue; + nxt_nvbcq_t active_queue; +#endif + uint32_t counter; +} nxt_cq_t; + + +static nxt_cq_t *pgq; + + +#if NXT_NCQ_TEST +#define nxt_cq_enqueue nxt_nncq_enqueue +#define nxt_cq_dequeue nxt_nncq_dequeue +#define nxt_cq_empty nxt_nncq_empty +#define nxt_cq_init nxt_nncq_init +#define NXT_CQ_SIZE NXT_NNCQ_SIZE +#else +#define nxt_cq_enqueue nxt_nvbcq_enqueue +#define nxt_cq_dequeue nxt_nvbcq_dequeue +#define nxt_cq_empty nxt_nvbcq_empty +#define nxt_cq_init nxt_nvbcq_init +#define NXT_CQ_SIZE NXT_NVBCQ_SIZE +#endif + +typedef struct { + int id; + uint64_t enq; + uint64_t deq; + uint64_t wait_enq; + uint64_t wait_deq; + uint64_t own_res; + uint64_t cas; + uint64_t faa; + +#if NXT_QTEST_USE_THREAD + nxt_thread_handle_t handle; +#else + nxt_pid_t pid; + int status; +#endif +} nxt_worker_info_t; + + +static void +cas_worker(void *p) +{ + nxt_cq_t *q; + uint32_t c; + uintptr_t i; + nxt_worker_info_t *wi; + + q = pgq; + wi = p; + + for (i = 0; i < nops / nprocs_cas; i++) { + c = q->counter; + + if (nxt_atomic_cmp_set(&q->counter, c, c + 1)) { + ++wi->cas; + } + } +} + + +static void +faa_worker(void *p) +{ + nxt_cq_t *q; + uintptr_t i; + nxt_worker_info_t *wi; + + q = pgq; + wi = p; + + for (i = 0; i < nops / nprocs_faa; i++) { + nxt_atomic_fetch_add(&q->counter, 1); + wi->faa++; + } +} + + +static void +enq_deq_worker(void *p) +{ + nxt_cq_t *q; + uintptr_t i, v; + nxt_worker_info_t *wi; + + q = pgq; + wi = p; + + for (i = 0; i < nops / nprocs_enq_deq; i++) { + v = nxt_cq_dequeue(&q->free_queue); + + if (v != nxt_cq_empty(&q->free_queue)) { + nxt_cq_enqueue(&q->active_queue, wi->id); + wi->enq++; + } + + v = nxt_cq_dequeue(&q->active_queue); + + if (v != nxt_cq_empty(&q->active_queue)) { + nxt_cq_enqueue(&q->free_queue, v); + wi->deq++; + + if ((int) v == wi->id) { + wi->own_res++; + } + } + } +} + + +static void +enq_worker(void *p) +{ + nxt_cq_t *q; + uintptr_t i, v; + nxt_worker_info_t *wi; + + q = pgq; + wi = p; + + for (i = 0; i < nops / nprocs_enq; i++) { + v = nxt_cq_dequeue(&q->free_queue); + + if (v != nxt_cq_empty(&q->free_queue)) { + nxt_cq_enqueue(&q->active_queue, v); + wi->enq++; + } + } +} + + +static void +deq_worker(void *p) +{ + nxt_cq_t *q; + uintptr_t i, v; + nxt_worker_info_t *wi; + + q = pgq; + wi = p; + + for (i = 0; i < nops / nprocs_deq; i++) { + v = nxt_cq_dequeue(&q->active_queue); + + if (v != nxt_cq_empty(&q->active_queue)) { + nxt_cq_enqueue(&q->free_queue, v); + ++wi->deq; + } + } +} + + +static void +wenq_worker(void *p) +{ + nxt_cq_t *q; + uintptr_t i, v; + nxt_worker_info_t *wi; + + q = pgq; + wi = p; + + for (i = 0; i < nops / nprocs_wenq; i++) { + + do { + wi->wait_enq++; + v = nxt_cq_dequeue(&q->free_queue); + } while (v == nxt_cq_empty(&q->free_queue)); + + nxt_cq_enqueue(&q->active_queue, v); + + wi->enq++; + wi->wait_enq--; + } +} + + +static void +wdeq_worker(void *p) +{ + nxt_cq_t *q; + uintptr_t i, v; + nxt_worker_info_t *wi; + + q = pgq; + wi = p; + + for (i = 0; i < nops / nprocs_wdeq; i++) { + + do { + wi->wait_deq++; + v = nxt_cq_dequeue(&q->active_queue); + } while (v == nxt_cq_empty(&q->active_queue)); + + nxt_cq_enqueue(&q->free_queue, v); + + wi->deq++; + wi->wait_deq--; + } +} + + +static nxt_int_t +worker_create(nxt_worker_info_t *wi, int id, nxt_thread_start_t start) +{ + wi->id = id; + +#if NXT_QTEST_USE_THREAD + nxt_thread_link_t *link; + + link = nxt_zalloc(sizeof(nxt_thread_link_t)); + + link->start = start; + link->work.data = wi; + + return nxt_thread_create(&wi->handle, link); + +#else + pid_t pid = fork(); + + if (pid == 0) { + start(wi); + exit(0); + + } else { + wi->pid = pid; + } + + return NXT_OK; +#endif +} + + +static void +worker_wait(nxt_worker_info_t *wi) +{ +#if NXT_QTEST_USE_THREAD + pthread_join(wi->handle, NULL); + +#else + waitpid(wi->pid, &wi->status, 0); +#endif +} + + +int nxt_cdecl +main(int argc, char **argv) +{ + int i, k, id, verbose, objective, rk; + char *a; + size_t start, elapsed; + double *stats, m, c; + uint64_t total_ops; + uintptr_t j; + nxt_task_t task; + nxt_thread_t *thr; + nxt_worker_info_t *wi; + double times[MAX_ITER], mopsec[MAX_ITER]; + + verbose = 0; + objective = 0; + + for (i = 1; i < argc; i++) { + a = argv[i]; + + if (strcmp(a, "-v") == 0) { + verbose++; + continue; + } + + if (strcmp(a, "-n") == 0 && (i + 1) < argc) { + nops = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--enq") == 0 && (i + 1) < argc) { + nprocs_enq = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--deq") == 0 && (i + 1) < argc) { + nprocs_deq = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--wenq") == 0 && (i + 1) < argc) { + nprocs_wenq = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--wdeq") == 0 && (i + 1) < argc) { + nprocs_wdeq = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--ed") == 0 && (i + 1) < argc) { + nprocs_enq_deq = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--cas") == 0 && (i + 1) < argc) { + nprocs_cas = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--faa") == 0 && (i + 1) < argc) { + nprocs_faa = atoi(argv[++i]); + continue; + } + + if (strcmp(a, "--obj") == 0 && (i + 1) < argc) { + objective = atoi(argv[++i]); + continue; + } + + printf("unknown option %s", a); + + return 1; + } + + if (nxt_lib_start("ncq_test", argv, &environ) != NXT_OK) { + return 1; + } + + nprocs = nprocs_enq + nprocs_deq + nprocs_wenq + nprocs_wdeq + + nprocs_enq_deq + nprocs_cas + nprocs_faa; + + if (nprocs == 0) { + return 0; + } + + nxt_main_log.level = NXT_LOG_INFO; + task.log = &nxt_main_log; + + thr = nxt_thread(); + thr->task = &task; + + pgq = mmap(NULL, sizeof(nxt_cq_t), PROT_READ | PROT_WRITE, + MAP_ANON | MAP_SHARED, -1, 0); + if (pgq == MAP_FAILED) { + return 2; + } + + nxt_cq_init(&pgq->free_queue); + nxt_cq_init(&pgq->active_queue); + + for(i = 0; i < NXT_CQ_SIZE; i++) { + nxt_cq_enqueue(&pgq->free_queue, i); + } + + if (verbose >= 1) { + printf("number of workers: %d\n", (int) nprocs); + printf("number of ops: %d\n", (int) nops); + } + + wi = mmap(NULL, nprocs * sizeof(nxt_worker_info_t), PROT_READ | PROT_WRITE, + MAP_ANON | MAP_SHARED, -1, 0); + if (wi == MAP_FAILED) { + return 3; + } + + for (k = 0; k < MAX_ITER; k++) { + nxt_memzero(wi, nprocs * sizeof(nxt_worker_info_t)); + + nxt_cq_init(&pgq->free_queue); + nxt_cq_init(&pgq->active_queue); + + for(i = 0; i < NXT_CQ_SIZE; i++) { + nxt_cq_enqueue(&pgq->free_queue, i); + } + + start = elapsed_time(0); + + id = 0; + + for (j = 0; j < nprocs_enq; j++, id++) { + worker_create(wi + id, id, enq_worker); + } + + for (j = 0; j < nprocs_deq; j++, id++) { + worker_create(wi + id, id, deq_worker); + } + + for (j = 0; j < nprocs_wenq; j++, id++) { + worker_create(wi + id, id, wenq_worker); + } + + for (j = 0; j < nprocs_wdeq; j++, id++) { + worker_create(wi + id, id, wdeq_worker); + } + + for (j = 0; j < nprocs_enq_deq; j++, id++) { + worker_create(wi + id, id, enq_deq_worker); + } + + for (j = 0; j < nprocs_cas; j++, id++) { + worker_create(wi + id, id, cas_worker); + } + + for (j = 0; j < nprocs_faa; j++, id++) { + worker_create(wi + id, id, faa_worker); + } + + for (j = 0; j < nprocs; j++) { + worker_wait(wi + j); + } + + elapsed = elapsed_time(start); + + for (j = 1; j < nprocs; j++) { + wi[0].enq += wi[j].enq; + wi[0].deq += wi[j].deq; + wi[0].wait_enq += wi[j].wait_enq; + wi[0].wait_deq += wi[j].wait_deq; + wi[0].own_res += wi[j].own_res; + wi[0].cas += wi[j].cas; + wi[0].faa += wi[j].faa; + } + + total_ops = wi[0].enq + wi[0].deq + wi[0].cas + wi[0].faa; + + if (total_ops == 0) { + total_ops = nops; + } + + times[k] = elapsed / 1000.0; + mopsec[k] = (double) total_ops / elapsed; + + if (verbose >= 2) { + printf("enq %10"PRIu64"\n", wi[0].enq); + printf("deq %10"PRIu64"\n", wi[0].deq); + printf("wait_enq %10"PRIu64"\n", wi[0].wait_enq); + printf("wait_deq %10"PRIu64"\n", wi[0].wait_deq); + printf("own_res %10"PRIu64"\n", wi[0].own_res); + printf("cas %10"PRIu64"\n", wi[0].cas); + printf("faa %10"PRIu64"\n", wi[0].faa); + printf("total ops %10"PRIu64"\n", total_ops); + printf("Mops/sec %13.2f\n", mopsec[k]); + + printf("elapsed %10d us\n", (int) elapsed); + printf("per op %10d ns\n", (int) ((1000 * elapsed) / total_ops)); + } + + if (k >= STAT_ITER) { + stats = (objective == 0) ? times : mopsec; + + m = mean(stats + k - STAT_ITER, STAT_ITER); + c = cov(stats + k - STAT_ITER, m, STAT_ITER); + + if (verbose >= 1) { + if (objective == 0) { + printf(" #%02d elapsed time: %.2f ms; Mops/sec %.2f; " + "mean time %.2f ms; cov %.4f\n", + (int) k + 1, times[k], mopsec[k], m, c); + + } else { + printf(" #%02d elapsed time: %.2f ms; Mops/sec %.2f; " + "mean Mop/sec %.2f; cov %.4f\n", + (int) k + 1, times[k], mopsec[k], m, c); + } + } + + if (c < MIN_COV) { + rk = k - STAT_ITER; + + for (i = rk + 1; i <= k; i++) { + if (fabs(stats[i] - m) < fabs(stats[rk] - m)) { + rk = i; + } + } + + printf("#%d %.2f ms; %.2f\n", rk, times[rk], mopsec[rk]); + + return 0; + } + + } else { + if (verbose >= 1) { + printf(" #%02d elapsed time: %.2f ms; Mops/sec %.2f\n", + (int) k + 1, times[k], mopsec[k]); + } + } + } + + return 0; +} diff --git a/src/test/nxt_lvlhsh_test.c b/src/test/nxt_lvlhsh_test.c index 2e1e0b20..baa6d0e1 100644 --- a/src/test/nxt_lvlhsh_test.c +++ b/src/test/nxt_lvlhsh_test.c @@ -19,20 +19,6 @@ nxt_lvlhsh_test_key_test(nxt_lvlhsh_query_t *lhq, void *data) } -static void * -nxt_lvlhsh_test_pool_alloc(void *pool, size_t size) -{ - return nxt_mp_align(pool, size, size); -} - - -static void -nxt_lvlhsh_test_pool_free(void *pool, void *p) -{ - nxt_mp_free(pool, p); -} - - static const nxt_lvlhsh_proto_t malloc_proto nxt_aligned(64) = { //NXT_LVLHSH_LARGE_MEMALIGN, NXT_LVLHSH_DEFAULT, @@ -44,8 +30,8 @@ static const nxt_lvlhsh_proto_t malloc_proto nxt_aligned(64) = { static const nxt_lvlhsh_proto_t pool_proto nxt_aligned(64) = { NXT_LVLHSH_LARGE_SLAB, nxt_lvlhsh_test_key_test, - nxt_lvlhsh_test_pool_alloc, - nxt_lvlhsh_test_pool_free, + nxt_mp_lvlhsh_alloc, + nxt_mp_lvlhsh_free, }; |