diff options
Diffstat (limited to '')
188 files changed, 49907 insertions, 0 deletions
diff --git a/src/nxt_aix_send_file.c b/src/nxt_aix_send_file.c new file mode 100644 index 00000000..b7cb3b28 --- /dev/null +++ b/src/nxt_aix_send_file.c @@ -0,0 +1,128 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* send_file() has been introduced in AIX 4.3.2 */ + +ssize_t nxt_aix_event_conn_io_send_file(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + + +ssize_t +nxt_aix_event_conn_io_send_file(nxt_event_conn_t *c, nxt_buf_t *b, size_t limit) +{ + ssize_t n; + nxt_buf_t *fb; + nxt_err_t err; + nxt_off_t file_size, sent; + nxt_uint_t nhd, ntr; + struct iovec hd[NXT_IOBUF_MAX], tr; + struct sf_parms sfp; + nxt_sendbuf_coalesce_t sb; + + sb.buf = b; + sb.iobuf = hd; + sb.nmax = NXT_IOBUF_MAX; + sb.sync = 0; + sb.size = 0; + sb.limit = limit; + + nhd = nxt_sendbuf_mem_coalesce(&sb); + + if (nhd == 0 && sb.sync) { + return 0; + } + + if (nhd > 1 || sb.buf == NULL || !nxt_buf_is_file(sb.buf)) { + return nxt_event_conn_io_writev(c, hd, nhd); + } + + fb = sb.buf; + + file_size = nxt_sendbuf_file_coalesce(&sb); + + if (file_size == 0) { + return nxt_event_conn_io_writev(c, hd, nhd); + } + + sb.iobuf = &tr; + sb.nmax = 1; + + ntr = nxt_sendbuf_mem_coalesce(&sb); + + nxt_memzero(&sfp, sizeof(struct sf_parms)); + + if (nhd != 0) { + sfp.header_data = hd[0].iov_base; + sfp.header_length = hd[0].iov_len; + } + + sfp.file_descriptor = fb->file->fd; + sfp.file_offset = fb->file_pos; + sfp.file_bytes = file_size; + + if (ntr != 0) { + sfp.trailer_data = tr.iov_base; + sfp.trailer_length = tr.iov_len; + } + + nxt_log_debug(c->socket.log, "send_file(%d) fd:%FD @%O:%O hd:%ui tr:%ui", + c->socket.fd, fb->file->fd, fb->file_pos, file_size, + nhd, ntr); + + n = send_file(&c->socket.fd, &sfp, 0); + + err = (n == -1) ? nxt_errno : 0; + sent = sfp.bytes_sent; + + nxt_log_debug(c->socket.log, "send_file(%d): %d sent:%O", + c->socket.fd, n, sent); + + /* + * -1 an error has occurred, errno contains the error code; + * 0 the command has completed successfully; + * 1 the command was completed partially, some data has been + * transmitted but the command has to return for some reason, + * for example, the command was interrupted by signals. + */ + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "send_file(%d) failed %E \"%FN\" " + "fd:%FD @%O:%O hd:%ui tr:%ui", c->socket.fd, err, + fb->file->name, fb->file->fd, fb->file_pos, + file_size, nhd, ntr); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "sendfile() %E", err); + + return sent; + } + + if (n == 1) { + return sent; + } + + if (sent < (nxt_off_t) sb.size) { + c->socket.write_ready = 0; + } + + return sent; +} diff --git a/src/nxt_app_log.c b/src/nxt_app_log.c new file mode 100644 index 00000000..8aedfacb --- /dev/null +++ b/src/nxt_app_log.c @@ -0,0 +1,126 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_cycle.h> + + +static nxt_time_string_t nxt_log_error_time_cache; +static u_char *nxt_log_error_time(u_char *buf, nxt_realtime_t *now, + struct tm *tm, size_t size, const char *format); +static nxt_time_string_t nxt_log_debug_time_cache; +static u_char *nxt_log_debug_time(u_char *buf, nxt_realtime_t *now, + struct tm *tm, size_t size, const char *format); + + +void nxt_cdecl +nxt_log_time_handler(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...) +{ + u_char *p, *syslogmsg, *end; + va_list args; + nxt_fid_t fid; + const char *id; + nxt_fiber_t *fib; + nxt_thread_t *thr; + nxt_time_string_t *time_cache; + u_char msg[NXT_MAX_ERROR_STR]; + + thr = nxt_thread(); + + end = msg + NXT_MAX_ERROR_STR; + + time_cache = (log->level != NXT_LOG_DEBUG) ? &nxt_log_error_time_cache: + &nxt_log_debug_time_cache; + + p = nxt_thread_time_string(thr, time_cache, msg); + + syslogmsg = p; + + fib = nxt_fiber_self(thr); + + if (fib != NULL) { + id = "[%V] %PI#%PT#%PF "; + fid = nxt_fiber_id(fib); + + } else { + id = "[%V] %PI#%PT "; + fid = 0; + } + + p = nxt_sprintf(p, end, id, &nxt_log_levels[level], nxt_pid, + nxt_thread_tid(thr), fid); + + if (log->ident != 0) { + p = nxt_sprintf(p, end, "*%D ", log->ident); + } + + va_start(args, fmt); + p = nxt_vsprintf(p, end, fmt, args); + va_end(args); + + if (level != NXT_LOG_DEBUG && log->ctx_handler != NULL) { + p = log->ctx_handler(log->ctx, p, end); + } + + if (p > end - NXT_LINEFEED_SIZE) { + p = end - NXT_LINEFEED_SIZE; + } + + nxt_linefeed(p); + + (void) nxt_write_console(nxt_stderr, msg, p - msg); + + if (level <= NXT_LOG_ALERT) { + *(p - NXT_LINEFEED_SIZE) = '\0'; + + /* + * The syslog LOG_ALERT level is enough, because + * LOG_EMERG level broadcasts a message to all users. + */ + nxt_write_syslog(LOG_ALERT, syslogmsg); + } +} + + +static nxt_time_string_t nxt_log_error_time_cache = { + (nxt_atomic_uint_t) -1, + nxt_log_error_time, + "%4d/%02d/%02d %02d:%02d:%02d ", + sizeof("1970/09/28 12:00:00 ") - 1, + NXT_THREAD_TIME_LOCAL, + NXT_THREAD_TIME_MSEC, +}; + + +static u_char * +nxt_log_error_time(u_char *buf, nxt_realtime_t *now, struct tm *tm, size_t size, + const char *format) +{ + return nxt_sprintf(buf, buf + size, format, + tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, + tm->tm_hour, tm->tm_min, tm->tm_sec); +} + + +static nxt_time_string_t nxt_log_debug_time_cache = { + (nxt_atomic_uint_t) -1, + nxt_log_debug_time, + "%4d/%02d/%02d %02d:%02d:%02d.%03d ", + sizeof("1970/09/28 12:00:00.000 ") - 1, + NXT_THREAD_TIME_LOCAL, + NXT_THREAD_TIME_MSEC, +}; + + +static u_char * +nxt_log_debug_time(u_char *buf, nxt_realtime_t *now, struct tm *tm, size_t size, + const char *format) +{ + return nxt_sprintf(buf, buf + size, format, + tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, + tm->tm_hour, tm->tm_min, tm->tm_sec, + now->nsec / 1000000); +} diff --git a/src/nxt_application.c b/src/nxt_application.c new file mode 100644 index 00000000..08d32b37 --- /dev/null +++ b/src/nxt_application.c @@ -0,0 +1,903 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Valentin V. Bartenev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_cycle.h> +#include <nxt_application.h> + + +#define NXT_PARSE_AGAIN (u_char *) -1 + + +static void nxt_app_thread(void *ctx); +static nxt_app_request_t *nxt_app_request_create(nxt_socket_t s, + nxt_log_t *log); +static void nxt_app_conn_update(nxt_thread_t *thr, nxt_event_conn_t *c, + nxt_log_t *log); +static nxt_int_t nxt_app_write_finish(nxt_app_request_t *r); +static void nxt_app_buf_complettion(nxt_thread_t *thr, void *obj, void *data); +static void nxt_app_delivery_handler(nxt_thread_t *thr, void *obj, void *data); +static void nxt_app_delivery_ready(nxt_thread_t *thr, void *obj, void *data); +static void nxt_app_delivery_complettion(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_app_delivery_error(nxt_thread_t *thr, void *obj, void *data); +static void nxt_app_delivery_timeout(nxt_thread_t *thr, void *obj, void *data); +static nxt_msec_t nxt_app_delivery_timer_value(nxt_event_conn_t *c, + uintptr_t data); +static void nxt_app_delivery_done(nxt_thread_t *thr, nxt_event_conn_t *c); +static void nxt_app_close_request(nxt_thread_t *thr, nxt_app_request_t *r); + + +typedef struct nxt_app_http_parse_state_s nxt_app_http_parse_state_t; + +struct nxt_app_http_parse_state_s { + u_char *pos; + nxt_int_t (*handler)(nxt_app_request_header_t *h, u_char *start, + u_char *end, nxt_app_http_parse_state_t *state); +}; + +static nxt_int_t nxt_app_http_parse_request(nxt_app_request_t *r, u_char *buf, + size_t size); +static nxt_int_t nxt_app_http_parse_request_line(nxt_app_request_header_t *h, + u_char *start, u_char *end, nxt_app_http_parse_state_t *state); +static nxt_int_t nxt_app_http_parse_field_value(nxt_app_request_header_t *h, + u_char *start, u_char *end, nxt_app_http_parse_state_t *state); +static nxt_int_t nxt_app_http_parse_field_name(nxt_app_request_header_t *h, + u_char *start, u_char *end, nxt_app_http_parse_state_t *state); + +static nxt_int_t nxt_app_http_process_headers(nxt_app_request_t *r); + + +static const nxt_event_conn_state_t nxt_app_delivery_write_state; + +static nxt_application_module_t *nxt_app = &nxt_python_module; + +static nxt_thread_mutex_t nxt_app_mutex; +static nxt_thread_cond_t nxt_app_cond; + +static nxt_buf_t *nxt_app_buf_free; +static nxt_buf_t *nxt_app_buf_done; + +static nxt_event_engine_t *nxt_app_engine; +static nxt_mem_pool_t *nxt_app_mem_pool; + +static nxt_uint_t nxt_app_buf_current_number; +static nxt_uint_t nxt_app_buf_max_number = 16; + + +nxt_int_t +nxt_app_start(nxt_cycle_t *cycle) +{ + nxt_thread_link_t *link; + nxt_thread_handle_t handle; + + if (nxt_slow_path(nxt_thread_mutex_create(&nxt_app_mutex) != NXT_OK)) { + return NXT_ERROR; + } + + if (nxt_slow_path(nxt_thread_cond_create(&nxt_app_cond) != NXT_OK)) { + return NXT_ERROR; + } + + link = nxt_malloc(sizeof(nxt_thread_link_t)); + + if (nxt_fast_path(link != NULL)) { + link->start = nxt_app_thread; + link->data = cycle; + link->engine = NULL; + link->exit = NULL; + + return nxt_thread_create(&handle, link); + } + + return NXT_ERROR; +} + + +#define SIZE 4096 + +static void +nxt_app_thread(void *ctx) +{ + ssize_t n; + nxt_err_t err; + nxt_cycle_t *cycle; + nxt_socket_t s; + nxt_thread_t *thr; + nxt_app_request_t *r; + nxt_event_engine_t **engines; + nxt_listen_socket_t *ls; + u_char buf[SIZE]; + const size_t size = SIZE; + nxt_app_header_field_t fields[128]; + + thr = nxt_thread(); + + nxt_log_debug(thr->log, "app thread"); + + cycle = ctx; + engines = cycle->engines->elts; + + nxt_app_engine = engines[0]; + + nxt_app_mem_pool = nxt_mem_pool_create(512); + if (nxt_slow_path(nxt_app_mem_pool == NULL)) { + return; + } + + if (nxt_slow_path(nxt_app->init(thr) != NXT_OK)) { + nxt_log_debug(thr->log, "application init failed"); + } + + ls = cycle->listen_sockets->elts; + + for ( ;; ) { + s = accept(ls->socket, NULL, NULL); + + if (nxt_slow_path(s == -1)) { + err = nxt_socket_errno; + + nxt_log_error(NXT_LOG_ERR, thr->log, "accept(%d) failed %E", + ls->socket, err); + + if (err == EBADF) { + /* STUB: ls->socket has been closed on exit. */ + return; + } + + continue; + } + + nxt_log_debug(thr->log, "accept(%d): %d", ls->socket, s); + + n = recv(s, buf, size, 0); + + if (nxt_slow_path(n <= 0)) { + err = (n == 0) ? 0 : nxt_socket_errno; + + nxt_log_error(NXT_LOG_ERR, thr->log, "recv(%d, %uz) failed %E", + s, size, err); + close(s); + continue; + } + + nxt_log_debug(thr->log, "recv(%d, %uz): %z", s, size, n); + + r = nxt_app_request_create(s, thr->log); + if (nxt_slow_path(r == NULL)) { + goto fail; + } + + r->header.fields = fields; + + //nxt_app->start(r); + + if (nxt_app_http_parse_request(r, buf, n) != NXT_OK) { + nxt_log_debug(thr->log, "nxt_app_http_parse_request() failed"); + nxt_mem_pool_destroy(r->mem_pool); + goto fail; + } + + if (nxt_app_http_process_headers(r) != NXT_OK) { + nxt_log_debug(thr->log, "nxt_app_http_process_headers() failed"); + nxt_mem_pool_destroy(r->mem_pool); + goto fail; + } + + nxt_app->run(r); + + if (nxt_slow_path(nxt_app_write_finish(r) == NXT_ERROR)) { + goto fail; + } + + continue; + + fail: + + close(s); + nxt_nanosleep(1000000000); /* 1s */ + } +} + + +static nxt_app_request_t * +nxt_app_request_create(nxt_socket_t s, nxt_log_t *log) +{ + nxt_mem_pool_t *mp; + nxt_event_conn_t *c; + nxt_app_request_t *r; + + mp = nxt_mem_pool_create(1024); + if (nxt_slow_path(mp == NULL)) { + return NULL; + } + + r = nxt_mem_zalloc(mp, sizeof(nxt_app_request_t)); + if (nxt_slow_path(r == NULL)) { + return NULL; + } + + c = nxt_mem_zalloc(mp, sizeof(nxt_event_conn_t)); + if (nxt_slow_path(c == NULL)) { + return NULL; + } + + c->socket.fd = s; + c->socket.data = r; + + r->mem_pool = mp; + r->event_conn = c; + r->log = log; + + return r; +} + + +static nxt_int_t +nxt_app_http_parse_request(nxt_app_request_t *r, u_char *buf, size_t size) +{ + u_char *end; + ssize_t n; + nxt_err_t err; + nxt_socket_t s; + nxt_app_http_parse_state_t state; + + end = buf + size; + + state.pos = buf; + state.handler = nxt_app_http_parse_request_line; + + for ( ;; ) { + switch (state.handler(&r->header, state.pos, end, &state)) { + + case NXT_OK: + continue; + + case NXT_DONE: + r->body_preread.len = end - state.pos; + r->body_preread.data = state.pos; + + return NXT_OK; + + case NXT_AGAIN: + s = r->event_conn->socket.fd; + n = recv(s, end, SIZE - size, 0); + + if (nxt_slow_path(n <= 0)) { + err = (n == 0) ? 0 : nxt_socket_errno; + + nxt_log_error(NXT_LOG_ERR, r->log, "recv(%d, %uz) failed %E", + s, size, err); + + return NXT_ERROR; + } + + nxt_log_debug(r->log, "recv(%d, %uz): %z", s, SIZE - size, n); + + size += n; + end += n; + + continue; + } + + return NXT_ERROR; + } +} + + +static nxt_int_t +nxt_app_http_parse_request_line(nxt_app_request_header_t *h, u_char *start, + u_char *end, nxt_app_http_parse_state_t *state) +{ + u_char *p; + + for (p = start; /* void */; p++) { + + if (nxt_slow_path(p == end)) { + state->pos = p; + return NXT_AGAIN; + } + + if (*p == ' ') { + break; + } + } + + h->method.len = p - start; + h->method.data = start; + + start = p + 1; + + p = nxt_memchr(start, ' ', end - start); + + if (nxt_slow_path(p == NULL)) { + return NXT_AGAIN; + } + + h->path.len = p - start; + h->path.data = start; + + start = p + 1; + + if (nxt_slow_path((size_t) (end - start) < sizeof("HTTP/1.1\n") - 1)) { + return NXT_AGAIN; + } + + h->version.len = sizeof("HTTP/1.1") - 1; + h->version.data = start; + + p = start + sizeof("HTTP/1.1") - 1; + + if (nxt_slow_path(*p == '\n')) { + return nxt_app_http_parse_field_name(h, p + 1, end, state); + } + + if (nxt_slow_path(end - p < 2)) { + return NXT_AGAIN; + } + + return nxt_app_http_parse_field_name(h, p + 2, end, state); +} + + +static nxt_int_t +nxt_app_http_parse_field_name(nxt_app_request_header_t *h, u_char *start, + u_char *end, nxt_app_http_parse_state_t *state) +{ + u_char *p; + nxt_app_header_field_t *fld; + + if (nxt_slow_path(start == end)) { + goto again; + } + + if (nxt_slow_path(*start == '\n')) { + state->pos = start + 1; + return NXT_DONE; + } + + if (*start == '\r') { + if (nxt_slow_path(end - start < 2)) { + goto again; + } + + if (nxt_slow_path(start[1] != '\n')) { + return NXT_ERROR; + } + + state->pos = start + 2; + return NXT_DONE; + } + + p = nxt_memchr(start, ':', end - start); + + if (nxt_slow_path(p == NULL)) { + goto again; + } + + fld = &h->fields[h->fields_num]; + + fld->name.len = p - start; + fld->name.data = start; + + return nxt_app_http_parse_field_value(h, p + 1, end, state); + +again: + + state->pos = start; + state->handler = nxt_app_http_parse_field_name; + + return NXT_AGAIN; +} + + +static nxt_int_t +nxt_app_http_parse_field_value(nxt_app_request_header_t *h, u_char *start, + u_char *end, nxt_app_http_parse_state_t *state) +{ + u_char *p; + nxt_app_header_field_t *fld; + + for ( ;; ) { + if (nxt_slow_path(start == end)) { + goto again; + } + + if (*start != ' ') { + break; + } + + start++; + } + + p = nxt_memchr(start, '\n', end - start); + + if (nxt_slow_path(p == NULL)) { + goto again; + } + + fld = &h->fields[h->fields_num]; + + fld->value.len = p - start; + fld->value.data = start; + + fld->value.len -= (p[-1] == '\r'); + + h->fields_num++; + + state->pos = p + 1; + state->handler = nxt_app_http_parse_field_name; + + return NXT_OK; + +again: + + state->pos = start; + state->handler = nxt_app_http_parse_field_value; + + return NXT_AGAIN; +} + + +static nxt_int_t +nxt_app_http_process_headers(nxt_app_request_t *r) +{ + nxt_uint_t i; + nxt_app_header_field_t *fld; + + static const u_char content_length[14] = "Content-Length"; + static const u_char content_type[12] = "Content-Type"; + + for (i = 0; i < r->header.fields_num; i++) { + fld = &r->header.fields[i]; + + if (fld->name.len == sizeof(content_length) + && nxt_memcasecmp(fld->name.data, content_length, + sizeof(content_length)) == 0) + { + r->header.content_length = &fld->value; + r->body_rest = nxt_off_t_parse(fld->value.data, fld->value.len); + continue; + } + + if (fld->name.len == sizeof(content_type) + && nxt_memcasecmp(fld->name.data, content_type, + sizeof(content_type)) == 0) + { + r->header.content_type = &fld->value; + continue; + } + } + + return NXT_OK; +} + + +static void +nxt_app_conn_update(nxt_thread_t *thr, nxt_event_conn_t *c, nxt_log_t *log) +{ + static nxt_atomic_t ident = 1; + + c->socket.write_ready = 1; + + c->socket.log = &c->log; + c->log = *log; + + /* The while loop skips possible uint32_t overflow. */ + + while (c->log.ident == 0) { + c->log.ident = (uint32_t) nxt_atomic_fetch_add(&ident, 1); + } + + thr->engine->connections++; + + c->io = thr->engine->event->io; + c->max_chunk = NXT_INT32_T_MAX; + c->sendfile = NXT_CONN_SENDFILE_UNSET; + + c->socket.read_work_queue = &thr->engine->read_work_queue; + c->socket.write_work_queue = &thr->engine->write_work_queue; + c->read_work_queue = &thr->engine->read_work_queue; + c->write_work_queue = &thr->engine->write_work_queue; + + nxt_event_conn_timer_init(&c->read_timer, c, c->socket.read_work_queue); + nxt_event_conn_timer_init(&c->write_timer, c, c->socket.write_work_queue); + + nxt_log_debug(&c->log, "event connections: %uD", thr->engine->connections); +} + + +nxt_int_t +nxt_app_http_read_body(nxt_app_request_t *r, u_char *data, size_t len) +{ + size_t preread; + ssize_t n; + nxt_err_t err; + + if ((off_t) len > r->body_rest) { + len = (size_t) r->body_rest; + } + + preread = 0; + + if (r->body_preread.len != 0) { + preread = nxt_min(r->body_preread.len, len); + + nxt_memcpy(data, r->body_preread.data, preread); + + r->body_preread.len -= preread; + r->body_preread.data += preread; + + r->body_rest -= preread; + + len -= preread; + } + + if (len == 0) { + return NXT_OK; + } + + n = recv(r->event_conn->socket.fd, data + preread, len, 0); + + if (nxt_slow_path(n < (ssize_t) len)) { + if (n <= 0) { + err = (n == 0) ? 0 : nxt_socket_errno; + + nxt_log_error(NXT_LOG_ERR, r->log, "recv(%d, %uz) failed %E", + r->event_conn->socket.fd, len, err); + + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_ERR, r->log, + "client prematurely closed connection"); + + return NXT_ERROR; + } + + r->body_rest -= n; + + return NXT_OK; +} + + +nxt_int_t +nxt_app_write(nxt_app_request_t *r, const u_char *data, size_t len) +{ + size_t free; + nxt_err_t err; + nxt_buf_t *b, *out, **next; + nxt_uint_t bufs; + + out = NULL; + next = &out; + + b = r->output_buf; + + if (b == NULL) { + bufs = 0; + goto get_buf; + } + + bufs = 1; + + for ( ;; ) { + free = nxt_buf_mem_free_size(&b->mem); + + if (free > len) { + b->mem.free = nxt_cpymem(b->mem.free, data, len); + break; + } + + b->mem.free = nxt_cpymem(b->mem.free, data, free); + + data += free; + len -= free; + + *next = b; + next = &b->next; + + if (len == 0) { + b = NULL; + break; + } + + if (bufs == nxt_app_buf_max_number) { + bufs = 0; + *next = NULL; + + nxt_event_engine_post(nxt_app_engine, nxt_app_delivery_handler, + r->event_conn, out, &nxt_main_log); + + out = NULL; + next = &out; + } + + get_buf: + + if (nxt_slow_path(nxt_thread_mutex_lock(&nxt_app_mutex) != NXT_OK)) { + return NXT_ERROR; + } + + for ( ;; ) { + b = nxt_app_buf_free; + + if (b != NULL) { + nxt_app_buf_free = b->next; + break; + } + + if (nxt_app_buf_current_number < nxt_app_buf_max_number) { + break; + } + + err = nxt_thread_cond_wait(&nxt_app_cond, &nxt_app_mutex, + NXT_INFINITE_NSEC); + + if (nxt_slow_path(err != 0)) { + (void) nxt_thread_mutex_unlock(&nxt_app_mutex); + return NXT_ERROR; + } + } + + (void) nxt_thread_mutex_unlock(&nxt_app_mutex); + + if (b == NULL) { + b = nxt_buf_mem_alloc(nxt_app_mem_pool, 4096, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + b->completion_handler = nxt_app_buf_complettion; + + nxt_app_buf_current_number++; + } + + bufs++; + } + + r->output_buf = b; + + if (out != NULL) { + *next = NULL; + + nxt_event_engine_post(nxt_app_engine, nxt_app_delivery_handler, + r->event_conn, out, &nxt_main_log); + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_app_write_finish(nxt_app_request_t *r) +{ + nxt_buf_t *b, *out; + + b = nxt_buf_sync_alloc(r->mem_pool, NXT_BUF_SYNC_LAST); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + b->completion_handler = nxt_app_buf_complettion; + b->parent = (nxt_buf_t *) r; + + out = r->output_buf; + + if (out != NULL) { + r->output_buf = NULL; + out->next = b; + + } else { + out = b; + } + + nxt_event_engine_post(nxt_app_engine, nxt_app_delivery_handler, + r->event_conn, out, &nxt_main_log); + + return NXT_OK; +} + + +static void +nxt_app_buf_complettion(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + + b = obj; + + nxt_log_debug(thr->log, "app buf completion"); + + b->next = nxt_app_buf_done; + nxt_app_buf_done = b; +} + + +static void +nxt_app_delivery_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_mem_pool_t *mp; + nxt_event_conn_t *c; + + c = obj; + b = data; + + nxt_log_debug(thr->log, "app delivery handler"); + + if (c->write != NULL) { + nxt_buf_chain_add(&c->write, b); + return; + } + + if (c->mem_pool == NULL) { + mp = nxt_mem_pool_create(256); + if (nxt_slow_path(mp == NULL)) { + close(c->socket.fd); + return; + } + + c->mem_pool = mp; + nxt_app_conn_update(thr, c, &nxt_main_log); + } + + if (c->socket.timedout || c->socket.error != 0) { + nxt_buf_chain_add(&nxt_app_buf_done, b); + nxt_thread_work_queue_add(thr, c->write_work_queue, + nxt_app_delivery_complettion, c, NULL, + thr->log); + return; + } + + c->write = b; + c->write_state = &nxt_app_delivery_write_state; + + nxt_event_conn_write(thr, c); +} + + +static const nxt_event_conn_state_t nxt_app_delivery_write_state + nxt_aligned(64) = +{ + NXT_EVENT_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_app_delivery_ready, + NULL, + nxt_app_delivery_error, + + nxt_app_delivery_timeout, + nxt_app_delivery_timer_value, + 0, +}; + + +static void +nxt_app_delivery_ready(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + + c = obj; + + nxt_thread_work_queue_add(thr, c->write_work_queue, + nxt_app_delivery_complettion, c, NULL, thr->log); +} + + +static void +nxt_app_delivery_complettion(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b, *bn, *free; + nxt_app_request_t *r; + + nxt_log_debug(thr->log, "app delivery complettion"); + + free = NULL; + + for (b = nxt_app_buf_done; b; b = bn) { + bn = b->next; + + if (nxt_buf_is_mem(b)) { + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + + b->next = free; + free = b; + + continue; + } + + if (nxt_buf_is_last(b)) { + r = (nxt_app_request_t *) b->parent; + nxt_app_close_request(thr, r); + } + } + + nxt_app_buf_done = NULL; + + if (free == NULL) { + return; + } + + if (nxt_slow_path(nxt_thread_mutex_lock(&nxt_app_mutex) != NXT_OK)) { + return; + } + + nxt_buf_chain_add(&nxt_app_buf_free, free); + + (void) nxt_thread_mutex_unlock(&nxt_app_mutex); + + nxt_thread_time_update(thr); + + (void) nxt_thread_cond_signal(&nxt_app_cond); +} + + +static void +nxt_app_delivery_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "app delivery error"); + + nxt_app_delivery_done(thr, c); +} + + +static void +nxt_app_delivery_timeout(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "app delivery timeout"); + + nxt_app_delivery_done(thr, c); +} + + +static nxt_msec_t +nxt_app_delivery_timer_value(nxt_event_conn_t *c, uintptr_t data) +{ + /* 30000 ms */ + return 30000; +} + + +static void +nxt_app_delivery_done(nxt_thread_t *thr, nxt_event_conn_t *c) +{ + if (c->write == NULL) { + return; + } + + nxt_buf_chain_add(&nxt_app_buf_done, c->write); + + c->write = NULL; + + nxt_thread_work_queue_add(thr, c->write_work_queue, + nxt_app_delivery_complettion, c, NULL, thr->log); +} + + +static void +nxt_app_close_request(nxt_thread_t *thr, nxt_app_request_t *r) +{ + nxt_event_conn_t *c; + + nxt_log_debug(thr->log, "app close connection"); + + c = r->event_conn; + + nxt_event_conn_close(thr, c); + + nxt_mem_pool_destroy(c->mem_pool); + nxt_mem_pool_destroy(r->mem_pool); +} diff --git a/src/nxt_application.h b/src/nxt_application.h new file mode 100644 index 00000000..efbf9c51 --- /dev/null +++ b/src/nxt_application.h @@ -0,0 +1,60 @@ + +/* + * Copyright (C) Valentin V. Bartenev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_APPLICATION_H_INCLUDED_ +#define _NXT_APPLICATION_H_INCLUDED_ + + +typedef struct { + nxt_str_t name; + nxt_str_t value; +} nxt_app_header_field_t; + + +typedef struct { + nxt_str_t method; + nxt_str_t path; + nxt_str_t version; + nxt_uint_t fields_num; + nxt_app_header_field_t *fields; + + nxt_str_t *content_length; + nxt_str_t *content_type; +} nxt_app_request_header_t; + + +typedef struct { + nxt_event_engine_t *engine; + nxt_mem_pool_t *mem_pool; + nxt_event_conn_t *event_conn; + nxt_log_t *log; + + nxt_buf_t *output_buf; + + nxt_app_request_header_t header; + nxt_str_t body_preread; + off_t body_rest; + void *ctx; +} nxt_app_request_t; + + +typedef struct { + nxt_int_t (*init)(nxt_thread_t *thr); + nxt_int_t (*start)(nxt_app_request_t *r); + nxt_int_t (*header)(nxt_app_request_t *r, + nxt_app_header_field_t *field); + nxt_int_t (*run)(nxt_app_request_t *r); +} nxt_application_module_t; + + +extern nxt_application_module_t nxt_python_module; + + +nxt_int_t nxt_app_http_read_body(nxt_app_request_t *r, u_char *data, size_t len); +nxt_int_t nxt_app_write(nxt_app_request_t *r, const u_char *data, size_t len); + + +#endif /* _NXT_APPLICATION_H_INCLIDED_ */ diff --git a/src/nxt_array.c b/src/nxt_array.c new file mode 100644 index 00000000..69e59c67 --- /dev/null +++ b/src/nxt_array.c @@ -0,0 +1,96 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_array_t * +nxt_array_create(nxt_mem_pool_t *mp, nxt_uint_t n, size_t size) +{ + nxt_array_t *array; + + array = nxt_mem_alloc(mp, sizeof(nxt_array_t) + n * size); + + if (nxt_slow_path(array == NULL)) { + return NULL; + } + + array->elts = (char *) array + sizeof(nxt_array_t); + array->nelts = 0; + array->size = size; + array->nalloc = n; + array->mem_pool = mp; + + return array; +} + + +void * +nxt_array_add(nxt_array_t *array) +{ + void *p; + uint32_t nalloc, new_alloc; + + nalloc = array->nalloc; + + if (array->nelts == nalloc) { + + if (nalloc < 16) { + /* Allocate new array twice larger than current. */ + new_alloc = nalloc * 2; + + } else { + /* Allocate new array 1.5 times larger than current. */ + new_alloc = nalloc + nalloc / 2; + } + + p = nxt_mem_alloc(array->mem_pool, array->size * new_alloc); + + if (nxt_slow_path(p == NULL)) { + return NULL; + } + + nxt_memcpy(p, array->elts, array->size * nalloc); + + array->elts = p; + array->nalloc = new_alloc; + } + + p = (char *) array->elts + array->size * array->nelts; + array->nelts++; + + return p; +} + + +void * +nxt_array_zero_add(nxt_array_t *array) +{ + void *p; + + p = nxt_array_add(array); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, array->size); + } + + return p; +} + + +void +nxt_array_remove(nxt_array_t *array, void *elt) +{ + void *last; + + last = nxt_array_last(array); + + if (elt != last) { + nxt_memcpy(elt, last, array->size); + } + + array->nelts--; +} diff --git a/src/nxt_array.h b/src/nxt_array.h new file mode 100644 index 00000000..491572da --- /dev/null +++ b/src/nxt_array.h @@ -0,0 +1,51 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_ARRAY_H_INCLUDED_ +#define _NXT_ARRAY_H_INCLUDED_ + + +typedef struct { + void *elts; + /* nelts has uint32_t type because it is used most often. */ + uint32_t nelts; + uint16_t size; + uint16_t nalloc; + nxt_mem_pool_t *mem_pool; +} nxt_array_t; + + +NXT_EXPORT nxt_array_t *nxt_array_create(nxt_mem_pool_t *mp, nxt_uint_t n, + size_t size); +NXT_EXPORT void *nxt_array_add(nxt_array_t *array); +NXT_EXPORT void *nxt_array_zero_add(nxt_array_t *array); +NXT_EXPORT void nxt_array_remove(nxt_array_t *array, void *elt); + + +#define \ +nxt_array_last(array) \ + ((void *) ((char *) (array)->elts + (array)->size * ((array)->nelts - 1))) + + +#define \ +nxt_array_reset(array) \ + (array)->nelts = 0; + + +#define \ +nxt_array_is_empty(array) \ + ((array)->nelts == 0) + + +nxt_inline void * +nxt_array_remove_last(nxt_array_t *array) +{ + array->nelts--; + return (char *) array->elts + array->size * array->nelts; +} + + +#endif /* _NXT_ARRAY_H_INCLUDED_ */ diff --git a/src/nxt_atomic.h b/src/nxt_atomic.h new file mode 100644 index 00000000..b3a1e95a --- /dev/null +++ b/src/nxt_atomic.h @@ -0,0 +1,268 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_ATOMIC_H_INCLUDED_ +#define _NXT_ATOMIC_H_INCLUDED_ + + +/* + * nxt_atomic_try_lock() must set an acquire barrier on lock. + * nxt_atomic_xchg() must set an acquire barrier. + * nxt_atomic_release() must set a release barrier. + */ + +#if (NXT_HAVE_GCC_ATOMIC) /* GCC 4.1 builtin atomic operations */ + +typedef intptr_t nxt_atomic_int_t; +typedef uintptr_t nxt_atomic_uint_t; +typedef volatile nxt_atomic_uint_t nxt_atomic_t; + +/* + * __sync_bool_compare_and_swap() is a full barrier. + * __sync_lock_test_and_set() is an acquire barrier. + * __sync_lock_release() is a release barrier. + */ + +#define \ +nxt_atomic_cmp_set(lock, cmp, set) \ + __sync_bool_compare_and_swap(lock, cmp, set) + + +#define \ +nxt_atomic_xchg(lock, set) \ + __sync_lock_test_and_set(lock, set) + + +#define \ +nxt_atomic_fetch_add(value, add) \ + __sync_fetch_and_add(value, add) + + +#define \ +nxt_atomic_try_lock(lock) \ + nxt_atomic_cmp_set(lock, 0, 1) + + +#define \ +nxt_atomic_release(lock) \ + __sync_lock_release(lock) + + +#if (__i386__ || __i386 || __amd64__ || __amd64) +#define \ +nxt_cpu_pause() \ + __asm__ ("pause") + +#else +#define \ +nxt_cpu_pause() +#endif + + +#elif (NXT_HAVE_SOLARIS_ATOMIC) /* Solaris 10 */ + +#include <atomic.h> + +typedef long nxt_atomic_int_t; +typedef ulong_t nxt_atomic_uint_t; +typedef volatile nxt_atomic_uint_t nxt_atomic_t; + + +#define \ +nxt_atomic_cmp_set(lock, cmp, set) \ + (atomic_cas_ulong(lock, cmp, set) == (ulong_t) cmp) + + +#define \ +nxt_atomic_xchg(lock, set) \ + atomic_add_swap(lock, set) + + +#define \ +nxt_atomic_fetch_add(value, add) \ + (atomic_add_long_nv(value, add) - add) + +/* + * Solaris uses SPARC Total Store Order model. In this model: + * 1) Each atomic load-store instruction behaves as if it were followed by + * #LoadLoad, #LoadStore, and #StoreStore barriers. + * 2) Each load instruction behaves as if it were followed by + * #LoadLoad and #LoadStore barriers. + * 3) Each store instruction behaves as if it were followed by + * #StoreStore barrier. + * + * In X86_64 atomic instructions set a full barrier and usual instructions + * set implicit #LoadLoad, #LoadStore, and #StoreStore barriers. + * + * An acquire barrier requires at least #LoadLoad and #LoadStore barriers + * and they are provided by atomic load-store instruction. + * + * A release barrier requires at least #LoadStore and #StoreStore barriers, + * so a lock release does not require an explicit barrier: all load + * instructions in critical section is followed by implicit #LoadStore + * barrier and all store instructions are followed by implicit #StoreStore + * barrier. + */ + +#define \ +nxt_atomic_try_lock(lock) \ + nxt_atomic_cmp_set(lock, 0, 1) + + +#define \ +nxt_atomic_release(lock) \ + *lock = 0; + + +/* + * The "rep; nop" is used instead of "pause" to omit the "[ PAUSE ]" hardware + * capability added by linker since Solaris ld.so.1 does not know about it: + * + * ld.so.1: ...: fatal: hardware capability unsupported: 0x2000 [ PAUSE ] + */ + +#if (__i386__ || __i386 || __amd64__ || __amd64) +#define \ +nxt_cpu_pause() \ + __asm__ ("rep; nop") + +#else +#define \ +nxt_cpu_pause() +#endif + + +/* elif (NXT_HAVE_MACOSX_ATOMIC) */ + +/* + * The atomic(3) interface has been introduced in MacOS 10.4 (Tiger) and + * extended in 10.5 (Leopard). However its support is omitted because: + * + * 1) the interface is still incomplete: + * *) there are OSAtomicAdd32Barrier() and OSAtomicAdd64Barrier() + * but no OSAtomicAddLongBarrier(); + * *) there is no interface for XCHG operation. + * + * 2) the interface is tuned for non-SMP systems due to omission of the + * LOCK prefix on single CPU system but nowadays MacOSX systems are at + * least dual core. Thus these indirect calls just add overhead as + * compared with inlined atomic operations which are supported by GCC + * and Clang in modern MacOSX systems. + */ + + +#elif (NXT_HAVE_XLC_ATOMIC) /* XL C/C++ V8.0 for AIX */ + +#if (NXT_64BIT) + +typedef long nxt_atomic_int_t; +typedef unsigned long nxt_atomic_uint_t; +typedef volatile nxt_atomic_int_t nxt_atomic_t; + + +nxt_inline nxt_bool_t +nxt_atomic_cmp_set(nxt_atomic_t *lock, nxt_atomic_int_t cmp, + nxt_atomic_int_t set) +{ + nxt_atomic_int_t old; + + old = cmp; + + return __compare_and_swaplp(lock, &old, set); +} + + +#define \ +nxt_atomic_xchg(lock, set) \ + __fetch_and_swaplp(lock, set) + + +#define \ +nxt_atomic_fetch_add(value, add) \ + __fetch_and_addlp(value, add) + + +#else /* NXT_32BIT */ + +typedef int nxt_atomic_int_t; +typedef unsigned int nxt_atomic_uint_t; +typedef volatile nxt_atomic_int_t nxt_atomic_t; + + +nxt_inline nxt_bool_t +nxt_atomic_cmp_set(nxt_atomic_t *lock, nxt_atomic_int_t cmp, + nxt_atomic_int_t set) +{ + nxt_atomic_int_t old; + + old = cmp; + + return __compare_and_swap(lock, &old, set); +} + + +#define \ +nxt_atomic_xchg(lock, set) \ + __fetch_and_swap(lock, set) + + +#define \ +nxt_atomic_fetch_add(value, add) \ + __fetch_and_add(value, add) + + +#endif /* NXT_32BIT*/ + + +/* + * __lwsync() is a "lwsync" instruction that sets #LoadLoad, #LoadStore, + * and #StoreStore barrier. + * + * __compare_and_swap() is a pair of "ldarx" and "stdcx" instructions. + * A "lwsync" does not set #StoreLoad barrier so it can not be used after + * this pair since a next load inside critical section can be performed + * after the "ldarx" instruction but before the "stdcx" instruction. + * However, this next load instruction will load correct data because + * otherwise the "ldarx/stdcx" pair will fail and this data will be + * discarded. Nevertheless, the "isync" instruction is used for sure. + * + * A full barrier can be set with __sync(), a "sync" instruction, but there + * is also a faster __isync(), an "isync" instruction. This instruction is + * not a memory barrier but an instruction barrier. An "isync" instruction + * causes the processor to complete execution of all previous instructions + * and then to discard instructions (which may have begun execution) following + * the "isync". After the "isync" is executed, the following instructions + * then begin execution. The "isync" is used to ensure that the loads + * following entry into a critical section are not performed (because of + * aggressive out-of-order or speculative execution in the processor) until + * the lock is granted. + */ + +nxt_inline nxt_bool_t +nxt_atomic_try_lock(nxt_atomic_t *lock) +{ + if (nxt_atomic_cmp_set(lock, 0, 1)) { + __isync(); + return 1; + } + + return 0; +} + + +#define \ +nxt_atomic_release(lock) \ + do { __lwsync(); *lock = 0; } while (0) + + +#define \ +nxt_cpu_pause() + + +#endif /* NXT_HAVE_XLC_ATOMIC */ + + +#endif /* _NXT_ATOMIC_H_INCLUDED_ */ diff --git a/src/nxt_buf.c b/src/nxt_buf.c new file mode 100644 index 00000000..4789c0c7 --- /dev/null +++ b/src/nxt_buf.c @@ -0,0 +1,171 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_buf_completion(nxt_thread_t *thr, void *obj, void *data); + + +nxt_buf_t * +nxt_buf_mem_alloc(nxt_mem_pool_t *mp, size_t size, nxt_uint_t flags) +{ + nxt_buf_t *b; + + b = nxt_mem_cache_zalloc0(mp, NXT_BUF_MEM_SIZE); + if (nxt_slow_path(b == NULL)) { + return NULL; + } + + b->data = mp; + b->completion_handler = nxt_buf_completion; + b->size = NXT_BUF_MEM_SIZE; + + if (size != 0) { + b->mem.start = nxt_mem_buf(mp, &size, flags); + if (nxt_slow_path(b->mem.start == NULL)) { + return NULL; + } + + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + b->mem.end = b->mem.start + size; + } + + return b; +} + + +nxt_buf_t * +nxt_buf_file_alloc(nxt_mem_pool_t *mp, size_t size, nxt_uint_t flags) +{ + nxt_buf_t *b; + + b = nxt_mem_cache_zalloc0(mp, NXT_BUF_FILE_SIZE); + if (nxt_slow_path(b == NULL)) { + return NULL; + } + + b->data = mp; + b->completion_handler = nxt_buf_completion; + b->size = NXT_BUF_FILE_SIZE; + nxt_buf_set_file(b); + + if (size != 0) { + b->mem.start = nxt_mem_buf(mp, &size, flags); + if (nxt_slow_path(b->mem.start == NULL)) { + return NULL; + } + + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + b->mem.end = b->mem.start + size; + } + + return b; +} + + +nxt_buf_t * +nxt_buf_mmap_alloc(nxt_mem_pool_t *mp, size_t size) +{ + nxt_buf_t *b; + + b = nxt_mem_cache_zalloc0(mp, NXT_BUF_MMAP_SIZE); + + if (nxt_fast_path(b != NULL)) { + b->data = mp; + b->completion_handler = nxt_buf_completion; + b->size = NXT_BUF_MMAP_SIZE; + + nxt_buf_set_file(b); + nxt_buf_set_mmap(b); + nxt_buf_mem_set_size(&b->mem, size); + } + + return b; +} + + +nxt_buf_t * +nxt_buf_sync_alloc(nxt_mem_pool_t *mp, nxt_uint_t flags) +{ + nxt_buf_t *b; + + b = nxt_mem_cache_zalloc0(mp, NXT_BUF_SYNC_SIZE); + + if (nxt_fast_path(b != NULL)) { + b->data = mp; + b->completion_handler = nxt_buf_completion; + b->size = NXT_BUF_SYNC_SIZE; + + nxt_buf_set_sync(b); + b->is_nobuf = ((flags & NXT_BUF_SYNC_NOBUF) != 0); + b->is_flush = ((flags & NXT_BUF_SYNC_FLUSH) != 0); + b->is_last = ((flags & NXT_BUF_SYNC_LAST) != 0); + } + + return b; +} + + +void +nxt_buf_chain_add(nxt_buf_t **head, nxt_buf_t *in) +{ + nxt_buf_t *b, **prev; + + prev = head; + + for (b = *head; b != NULL; b = b->next) { + prev = &b->next; + } + + *prev = in; +} + + +size_t +nxt_buf_chain_length(nxt_buf_t *b) +{ + size_t length; + + length = 0; + + while (b != NULL) { + length += b->mem.free - b->mem.pos; + b = b->next; + } + + return length; +} + + +static void +nxt_buf_completion(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b, *parent; + nxt_mem_pool_t *mp; + + b = obj; + parent = data; + + nxt_log_debug(thr->log, "buf completion: %p %p", b, b->mem.start); + + mp = b->data; + nxt_buf_free(mp, b); + + if (parent != NULL) { + nxt_log_debug(thr->log, "parent retain:%uD", parent->retain); + + parent->retain--; + + if (parent->retain == 0) { + parent->mem.pos = parent->mem.free; + + parent->completion_handler(thr, parent, parent->parent); + } + } +} diff --git a/src/nxt_buf.h b/src/nxt_buf.h new file mode 100644 index 00000000..240a1d22 --- /dev/null +++ b/src/nxt_buf.h @@ -0,0 +1,246 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_BUF_H_INCLUDED_ +#define _NXT_BUF_H_INCLUDED_ + + +/* + * There are four types of buffers. They are different sizes, so they + * should be allocated by appropriate nxt_buf_XXX_alloc() function. + * + * 1) Memory-only buffers, their size is less than nxt_buf_t size, it + * is equal to offsetof(nxt_buf_t, file_pos), that is it is nxt_buf_t + * without file and mmap part. The buffers are frequently used, so + * the reduction allows to save 20-32 bytes depending on platform. + * + * 2) Memory/file buffers, on Unix their size is exactly nxt_buf_t size, + * since nxt_mem_map_file_ctx_t() is empty macro. On Windows the size + * equals offsetof(nxt_buf_t, mmap), that is it is nxt_buf_t without + * memory map context part. The buffers can contain both memory and + * file pointers at once, or only memory or file pointers. + * + * 3) Memory mapped buffers are similar to the memory/file buffers. Their + * size is exactly nxt_buf_t size. The buffers can contain both memory + * and file pointers at once, or only memory or file pointers. If a + * buffer is not currently mapped in memory, its mapping size is stored + * in the mem.end field and available via nxt_buf_mem_size() macro. + * + * 4) Sync buffers, their size is the same size as memory-only buffers + * size. A sync buffer can be smaller but for memory pool cache + * purpose it is better to allocate it as frequently used memory-only + * buffer. The buffers are used to synchronize pipeline processing + * completion, because data buffers in the pipeline can be completed + * and freed before their final output will even be passed to a peer. + * For this purpose a sync buffer is allocated with the stop flag which + * stops buffer chain completion processing on the sync buffer in + * nxt_sendbuf_update() and nxt_sendbuf_completion(). + * Clearing the stop flag allows to continue completion processing. + * + * The last flag means the end of the output and must be set only + * in a sync buffer. The last flag is not permitted in memory and + * file buffers since it requires special handling while conversion + * one buffer to another. + * + * The nxt_buf_used_size() macro treats a sync buffer as a memory-only + * buffer which has NULL pointers, thus the buffer content size is zero. + * If allocated size of sync buffer would be lesser than memory-only + * buffer, then the special memory flag would be required because + * currently presence of memory part is indicated by non-NULL pointer + * to a content in memory. + * + * All types of buffers can have the flush flag that means the buffer + * should be sent as much as possible. + */ + +typedef struct { + u_char *pos; + u_char *free; + u_char *start; + u_char *end; +} nxt_buf_mem_t; + + +struct nxt_buf_s { + void *data; + nxt_work_handler_t completion_handler; + nxt_buf_t *parent; + + /* + * The next link, flags, and nxt_buf_mem_t should + * reside together to improve cache locality. + */ + nxt_buf_t *next; + + uint32_t retain; + /* + * Used by nxt_mem_cache_free() to return buffer + * in appropriate memory pool cache. + */ + uint8_t size; + + uint8_t is_file; /* 1 bit */ + + uint16_t is_mmap:1; + + uint16_t is_sync:1; + uint16_t is_nobuf:1; + uint16_t is_flush:1; + uint16_t is_last:1; + + nxt_buf_mem_t mem; + + /* The file and mmap parts are not allocated by nxt_buf_mem_alloc(). */ + nxt_file_t *file; + nxt_off_t file_pos; + nxt_off_t file_end; + + /* The mmap part is not allocated by nxt_buf_file_alloc(). */ + nxt_mem_map_file_ctx_t (mmap) +}; + + +#define NXT_BUF_MEM_SIZE offsetof(nxt_buf_t, file) +#define NXT_BUF_SYNC_SIZE NXT_BUF_MEM_SIZE +#define NXT_BUF_MMAP_SIZE sizeof(nxt_buf_t) +#define NXT_BUF_FILE_SIZE sizeof(nxt_buf_t) + + +#define NXT_BUF_SYNC_NOBUF 1 +#define NXT_BUF_SYNC_FLUSH 2 +#define NXT_BUF_SYNC_LAST 4 + + +#define \ +nxt_buf_is_mem(b) \ + ((b)->mem.pos != NULL) + + +#define \ +nxt_buf_is_file(b) \ + ((b)->is_file) + +#define \ +nxt_buf_set_file(b) \ + (b)->is_file = 1 + +#define \ +nxt_buf_clear_file(b) \ + (b)->is_file = 0 + + +#define \ +nxt_buf_is_mmap(b) \ + ((b)->is_mmap) + +#define \ +nxt_buf_set_mmap(b) \ + (b)->is_mmap = 1 + +#define \ +nxt_buf_clear_mmap(b) \ + (b)->is_mmap = 0 + + +#define \ +nxt_buf_is_sync(b) \ + ((b)->is_sync) + +#define \ +nxt_buf_set_sync(b) \ + (b)->is_sync = 1 + +#define \ +nxt_buf_clear_sync(b) \ + (b)->is_sync = 0 + + +#define \ +nxt_buf_is_nobuf(b) \ + ((b)->is_nobuf) + +#define \ +nxt_buf_set_nobuf(b) \ + (b)->is_nobuf = 1 + +#define \ +nxt_buf_clear_nobuf(b) \ + (b)->is_nobuf = 0 + + +#define \ +nxt_buf_is_flush(b) \ + ((b)->is_flush) + +#define \ +nxt_buf_set_flush(b) \ + (b)->is_flush = 1 + +#define \ +nxt_buf_clear_flush(b) \ + (b)->is_flush = 0 + + +#define \ +nxt_buf_is_last(b) \ + ((b)->is_last) + +#define \ +nxt_buf_set_last(b) \ + (b)->is_last = 1 + +#define \ +nxt_buf_clear_last(b) \ + (b)->is_last = 0 + + +#define \ +nxt_buf_mem_set_size(bm, size) \ + do { \ + (bm)->start = 0; \ + (bm)->end = (void *) size; \ + } while (0) + + +#define \ +nxt_buf_mem_size(bm) \ + ((bm)->end - (bm)->start) + + +#define \ +nxt_buf_mem_used_size(bm) \ + ((bm)->free - (bm)->pos) + + +#define \ +nxt_buf_mem_free_size(bm) \ + ((bm)->end - (bm)->free) + + +#define \ +nxt_buf_used_size(b) \ + (nxt_buf_is_file(b) ? (b)->file_end - (b)->file_pos: \ + nxt_buf_mem_used_size(&(b)->mem)) + + +NXT_EXPORT nxt_buf_t *nxt_buf_mem_alloc(nxt_mem_pool_t *mp, size_t size, + nxt_uint_t flags); +NXT_EXPORT nxt_buf_t *nxt_buf_file_alloc(nxt_mem_pool_t *mp, size_t size, + nxt_uint_t flags); +NXT_EXPORT nxt_buf_t *nxt_buf_mmap_alloc(nxt_mem_pool_t *mp, size_t size); +NXT_EXPORT nxt_buf_t *nxt_buf_sync_alloc(nxt_mem_pool_t *mp, nxt_uint_t flags); + + +#define \ +nxt_buf_free(mp, b) \ + nxt_mem_cache_free0((mp), (b), (b)->size) + + +NXT_EXPORT void nxt_buf_chain_add(nxt_buf_t **head, nxt_buf_t *in); +NXT_EXPORT size_t nxt_buf_chain_length(nxt_buf_t *b); + + +#endif /* _NXT_BUF_H_INCLIDED_ */ diff --git a/src/nxt_buf_filter.c b/src/nxt_buf_filter.c new file mode 100644 index 00000000..0f040fc9 --- /dev/null +++ b/src/nxt_buf_filter.c @@ -0,0 +1,448 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_buf_filter_nobuf(nxt_buf_filter_t *f); +nxt_inline void nxt_buf_filter_next(nxt_buf_filter_t *f); +static void nxt_buf_filter_file_read_start(nxt_thread_t *thr, + nxt_buf_filter_t *f); +static void nxt_buf_filter_file_read(nxt_thread_t *thr, nxt_buf_filter_t *f); +static void nxt_buf_filter_file_job_completion(nxt_thread_t *thr, + void *obj, void *data); +static void nxt_buf_filter_buf_completion(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_buf_filter_file_read_error(nxt_thread_t *thr, void *obj, + void *data); + + +void +nxt_buf_filter_add(nxt_thread_t *thr, nxt_buf_filter_t *f, nxt_buf_t *b) +{ + nxt_buf_chain_add(&f->input, b); + + nxt_buf_filter(thr, f, NULL); +} + + +void +nxt_buf_filter(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_int_t ret; + nxt_buf_t *b; + nxt_buf_filter_t *f; + + f = obj; + + nxt_log_debug(thr->log, "buf filter"); + + if (f->done) { + return; + } + + f->queued = 0; + + for ( ;; ) { + /* + * f->input is a chain of original incoming buffers: memory, + * mapped, file, and sync buffers; + * f->current is a currently processed memory buffer or a chain + * of memory/file or mapped/file buffers which are read of + * or populated from file; + * f->output is a chain of output buffers; + * f->last is the last output buffer in the chain. + */ + + b = f->current; + + nxt_log_debug(thr->log, "buf filter current: %p", b); + + if (b == NULL) { + + if (f->reading) { + return; + } + + b = f->input; + + nxt_log_debug(thr->log, "buf filter input: %p", b); + + if (b == NULL) { + /* + * The end of the input chain, pass + * the output chain to the next filter. + */ + nxt_buf_filter_next(f); + + return; + } + + if (nxt_buf_is_mem(b)) { + + f->current = b; + f->input = b->next; + b->next = NULL; + + } else if (nxt_buf_is_file(b)) { + + if (f->run->filter_ready(f) != NXT_OK) { + nxt_buf_filter_next(f); + } + + nxt_buf_filter_file_read_start(thr, f); + return; + } + } + + if (nxt_buf_is_sync(b)) { + + ret = NXT_OK; + f->current = b; + f->input = b->next; + b->next = NULL; + + if (nxt_buf_is_nobuf(b)) { + ret = f->run->filter_sync_nobuf(f); + + } else if (nxt_buf_is_flush(b)) { + ret = f->run->filter_sync_flush(f); + + } else if (nxt_buf_is_last(b)) { + ret = f->run->filter_sync_last(f); + + f->done = (ret == NXT_OK); + } + + if (nxt_fast_path(ret == NXT_OK)) { + continue; + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + goto fail; + } + + /* ret == NXT_AGAIN: No filter internal buffers available. */ + goto nobuf; + } + + ret = f->run->filter_process(f); + + if (nxt_fast_path(ret == NXT_OK)) { + b = f->current; + /* + * A filter may just move f->current to f->output + * and then set f->current to NULL. + */ + if (b != NULL && b->mem.pos == b->mem.free) { + f->current = b->next; + nxt_thread_work_queue_add(thr, f->work_queue, + b->completion_handler, + b, b->parent, thr->log); + } + + continue; + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + goto fail; + } + + /* ret == NXT_AGAIN: No filter internal buffers available. */ + goto nobuf; + } + +nobuf: + + /* ret == NXT_AGAIN: No filter internal buffers available. */ + + if (nxt_buf_filter_nobuf(f) == NXT_OK) { + return; + } + +fail: + + nxt_thread_work_queue_add(thr, f->work_queue, f->run->filter_error, + f, f->data, thr->log); +} + + +static nxt_int_t +nxt_buf_filter_nobuf(nxt_buf_filter_t *f) +{ + nxt_buf_t *b; + + nxt_thread_log_debug("buf filter nobuf"); + + b = nxt_buf_sync_alloc(f->mem_pool, NXT_BUF_SYNC_NOBUF); + + if (nxt_fast_path(b != NULL)) { + + nxt_buf_chain_add(&f->output, b); + f->last = NULL; + + f->run->filter_next(f); + + f->output = NULL; + + return NXT_OK; + } + + return NXT_ERROR; +} + + +nxt_inline void +nxt_buf_filter_next(nxt_buf_filter_t *f) +{ + if (f->output != NULL) { + f->last = NULL; + + f->run->filter_next(f); + f->output = NULL; + } +} + + +void +nxt_buf_filter_enqueue(nxt_thread_t *thr, nxt_buf_filter_t *f) +{ + nxt_log_debug(thr->log, "buf filter enqueue: %d", f->queued); + + if (!f->queued && !f->done) { + f->queued = 1; + nxt_thread_work_queue_add(thr, f->work_queue, nxt_buf_filter, + f, NULL, thr->log); + } +} + + +static void +nxt_buf_filter_file_read_start(nxt_thread_t *thr, nxt_buf_filter_t *f) +{ + nxt_job_file_t *jbf; + nxt_buf_filter_file_t *ff; + + ff = f->run->job_file_create(f); + + if (nxt_slow_path(ff == NULL)) { + nxt_thread_work_queue_add(thr, f->work_queue, f->run->filter_error, + f, f->data, thr->log); + return; + } + + f->filter_file = ff; + + jbf = &ff->job_file; + jbf->file = *f->input->file; + + jbf->ready_handler = nxt_buf_filter_file_job_completion; + jbf->error_handler = nxt_buf_filter_file_read_error; + + nxt_job_set_name(&jbf->job, "buf filter job file"); + + f->reading = 1; + + nxt_buf_filter_file_read(thr, f); +} + + +static void +nxt_buf_filter_file_read(nxt_thread_t *thr, nxt_buf_filter_t *f) +{ + nxt_int_t ret; + nxt_off_t size; + nxt_buf_t *b; + nxt_buf_filter_file_t *ff; + + ff = f->filter_file; + + if (ff->job_file.buffer != NULL) { + /* File is now being read. */ + return; + } + + size = f->input->file_end - f->input->file_pos; + + if (size > (nxt_off_t) NXT_SIZE_T_MAX) { + /* + * Small size value is a hint for buffer pool allocation + * size, but if size of the size_t type is lesser than size + * of the nxt_off_t type, the large size value may be truncated, + * so use a default buffer pool allocation size. + */ + size = 0; + } + + if (f->mmap) { + ret = nxt_buf_pool_mmap_alloc(&ff->buffers, (size_t) size); + + } else { + ret = nxt_buf_pool_file_alloc(&ff->buffers, (size_t) size); + } + + if (nxt_fast_path(ret == NXT_OK)) { + b = ff->buffers.current; + + b->file_pos = f->input->file_pos; + b->file_end = f->input->file_pos; + b->file = f->input->file; + + ff->job_file.buffer = b; + ff->job_file.offset = f->input->file_pos; + + f->run->job_file_retain(f); + + nxt_job_file_read(thr, &ff->job_file.job); + return; + } + + if (nxt_fast_path(ret != NXT_ERROR)) { + + /* ret == NXT_AGAIN: No buffers available. */ + + if (f->buffering) { + f->buffering = 0; + + if (nxt_fast_path(f->run->filter_flush(f) != NXT_ERROR)) { + return; + } + + } else if (nxt_fast_path(nxt_buf_filter_nobuf(f) == NXT_OK)) { + return; + } + } + + nxt_thread_work_queue_add(thr, f->work_queue, f->run->filter_error, + f, f->data, thr->log); +} + + +typedef struct { + nxt_buf_filter_t *filter; + nxt_buf_t *buf; +} nxt_buf_filter_ctx_t; + + +static void +nxt_buf_filter_file_job_completion(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_bool_t done; + nxt_job_file_t *jbf; + nxt_buf_filter_t *f; + nxt_buf_filter_ctx_t *ctx; + + jbf = obj; + f = data; + b = jbf->buffer; + jbf->buffer = NULL; + + nxt_log_debug(thr->log, "buf filter file completion: \"%FN\" %O-%O", + jbf->file.name, b->file_pos, b->file_end); + + f->run->job_file_release(f); + + ctx = nxt_mem_cache_alloc0(f->mem_pool, sizeof(nxt_buf_filter_ctx_t)); + if (nxt_slow_path(ctx == NULL)) { + goto fail; + } + + ctx->filter = f; + ctx->buf = f->input; + + f->input->file_pos = b->file_end; + + done = (f->input->file_pos == f->input->file_end); + + if (done) { + f->input = f->input->next; + f->reading = 0; + } + + b->data = f->data; + b->completion_handler = nxt_buf_filter_buf_completion; + b->parent = (nxt_buf_t *) ctx; + b->next = NULL; + + nxt_buf_chain_add(&f->current, b); + + nxt_buf_filter(thr, f, NULL); + + if (b->mem.pos == b->mem.free) { + /* + * The buffer has been completely processed by nxt_buf_filter(), + * its completion handler has been placed in workqueue and + * nxt_buf_filter_buf_completion() should be eventually called. + */ + return; + } + + if (!done) { + /* Try to allocate another buffer and read the next file part. */ + nxt_buf_filter_file_read(thr, f); + } + + return; + +fail: + + nxt_thread_work_queue_add(thr, f->work_queue, f->run->filter_error, + f, f->data, thr->log); +} + + +static void +nxt_buf_filter_buf_completion(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *fb, *b; + nxt_buf_filter_t *f; + nxt_buf_filter_ctx_t *ctx; + + b = obj; + ctx = data; + f = ctx->filter; + + nxt_log_debug(thr->log, "buf filter completion: %p \"%FN\" %O-%O", + b, f->filter_file->job_file.file.name, + b->file_pos, b->file_end); + + /* nxt_http_send_filter() might clear a buffer's file status. */ + b->is_file = 1; + + fb = ctx->buf; + + nxt_mem_cache_free0(f->mem_pool, ctx, sizeof(nxt_buf_filter_ctx_t)); + nxt_buf_pool_free(&f->filter_file->buffers, b); + + if (fb->file_pos < fb->file_end) { + nxt_buf_filter_file_read(thr, f); + return; + } + + if (b->file_end == fb->file_end) { + nxt_buf_pool_destroy(&f->filter_file->buffers); + + nxt_job_destroy(&f->filter_file->job_file.job); + + nxt_thread_work_queue_add(thr, f->work_queue, fb->completion_handler, + fb, fb->parent, thr->log); + } + + nxt_buf_filter(thr, f, NULL); +} + + +static void +nxt_buf_filter_file_read_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_filter_t *f; + + f = data; + + nxt_thread_work_queue_add(thr, f->work_queue, f->run->filter_error, + f, f->data, thr->log); +} diff --git a/src/nxt_buf_filter.h b/src/nxt_buf_filter.h new file mode 100644 index 00000000..8252ca3d --- /dev/null +++ b/src/nxt_buf_filter.h @@ -0,0 +1,116 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_BUF_FILTER_H_INCLUDED_ +#define _NXT_BUF_FILTER_H_INCLUDED_ + + +/* + * nxt_buf_filter is a framework intended to simplify processing file + * buffers content by a filter. The filter should set callbacks and + * call nxt_buf_filter_add() to start processing. + * + * At first buf_filter calls filter_ready() and the filter ensures + * it may allocate or reuse its internal buffer. No real allocation + * is performed at this step. + * + * TODO prevent unneeded allocaiton if no input data. + * + * + * TODO: The filter can flush data buffered + * previously, if all internal buffers are full. + * + * Then buf_filter looks buffer chains. There are two buffer chains: + * the input chain is a chain of original incoming memory, file, and sync + * buffers; and the current chain is a chain of memory/file buffers read + * from a file-only buffer. The current chain is processed first. Since + * buffers in this chain always contains a memory part, they can be passed + * one by one to the filter using filter_process(). If there is an output + * buffer after the buffer processing, it is added to output chain. The + * output buffers are not filter internal buffers. They just point to these + * internal buffers and one internal buffer can correspond to several output + * buffers which point to adjoining parts of the internal buffer. Further + * processing depends on filter_process() result code: if it returns NXT_OK, + * then the filter internal buffer is not full and buf_filter looks the next + * current or input buffer. If result code is NXT_AGAIN, then the filter + * internal buffer is full and buf_filter calls filter_flush() and then + * schedules to run nxt_buf_filter_repeat(). nxt_buf_filter_repeat() will + * run after all ready output buffer completion handlers and will call + * buf_filter again if no one completion handler will do it already using + * nxt_buf_filter_enqueue(). So in any case buf_filter will run again only + * once. + * + * TODO: + * in ideal just one the filter internal buffer. + * This allows to minimize number of the filter internal buffers if they + * flush fast. + * + * If the current chain is empty, the buf_filter processes the input chain. + * Memory buffers are passed to the filter using filter_process(). If an + * input buffer is a file buffer, then buf_filter calls filter_flush() + * and starts a file job to read the buffer in memory. The file job reads + * file parts into memory/file buffers and adds them to the current chain. + * + * Sync buffers are passed to the filter using filter_sync(). Its + * post-processing is similar to the filter_process() post-processing, + * except sync buffers are always added unmodified to the output chain. + */ + +typedef struct { + nxt_job_file_t job_file; + nxt_buf_pool_t buffers; +} nxt_buf_filter_file_t; + + +typedef struct nxt_buf_filter_s nxt_buf_filter_t; + +typedef struct { + nxt_int_t (*filter_ready)(nxt_buf_filter_t *f); + nxt_int_t (*filter_process)(nxt_buf_filter_t *f); + nxt_int_t (*filter_flush)(nxt_buf_filter_t *f); + + nxt_int_t (*filter_sync_nobuf)(nxt_buf_filter_t *f); + nxt_int_t (*filter_sync_flush)(nxt_buf_filter_t *f); + nxt_int_t (*filter_sync_last)(nxt_buf_filter_t *f); + + void (*filter_next)(nxt_buf_filter_t *f); + void (*filter_error)(nxt_thread_t *thr, void *obj, + void *data); + + nxt_buf_filter_file_t *(*job_file_create)(nxt_buf_filter_t *f); + void (*job_file_retain)(nxt_buf_filter_t *f); + void (*job_file_release)(nxt_buf_filter_t *f); +} nxt_buf_filter_ops_t; + + +struct nxt_buf_filter_s { + nxt_buf_t *current; + nxt_buf_t *input; + nxt_buf_t *output; + nxt_buf_t *last; + + nxt_work_queue_t *work_queue; + nxt_buf_filter_file_t *filter_file; + void *data; + nxt_mem_pool_t *mem_pool; + + const nxt_buf_filter_ops_t *run; + + uint8_t mmap; /* 1 bit */ + uint8_t done; /* 1 bit */ + uint8_t queued; /* 1 bit */ + uint8_t reading; /* 1 bit */ + uint8_t buffering; /* 1 bit */ +}; + + +NXT_EXPORT void nxt_buf_filter_add(nxt_thread_t *thr, nxt_buf_filter_t *f, + nxt_buf_t *b); +NXT_EXPORT void nxt_buf_filter(nxt_thread_t *thr, void *obj, void *data); +NXT_EXPORT void nxt_buf_filter_enqueue(nxt_thread_t *thr, nxt_buf_filter_t *f); + + +#endif /* _NXT_BUF_FILTER_H_INCLUDED_ */ diff --git a/src/nxt_buf_pool.c b/src/nxt_buf_pool.c new file mode 100644 index 00000000..092cf58d --- /dev/null +++ b/src/nxt_buf_pool.c @@ -0,0 +1,191 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_int_t +nxt_buf_pool_mem_alloc(nxt_buf_pool_t *bp, size_t size) +{ + nxt_buf_t *b; + + b = bp->current; + + if (b != NULL && b->mem.free < b->mem.end) { + return NXT_OK; + } + + b = bp->free; + + if (b != NULL) { + bp->current = b; + bp->free = b->next; + b->next = NULL; + return NXT_OK; + } + + if (bp->num >= bp->max) { + return NXT_AGAIN; + } + + if (size == 0 || size >= bp->size + bp->size / 4) { + size = bp->size; + } + + b = nxt_buf_mem_alloc(bp->mem_pool, size, bp->flags); + + if (nxt_fast_path(b != NULL)) { + bp->current = b; + bp->num++; + return NXT_OK; + } + + return NXT_ERROR; +} + + +nxt_int_t +nxt_buf_pool_file_alloc(nxt_buf_pool_t *bp, size_t size) +{ + nxt_buf_t *b; + + b = bp->current; + + if (b != NULL && b->mem.free < b->mem.end) { + return NXT_OK; + } + + b = bp->free; + + if (b != NULL) { + bp->current = b; + bp->free = b->next; + b->next = NULL; + return NXT_OK; + } + + if (bp->num >= bp->max) { + return NXT_AGAIN; + } + + if (size == 0 || size >= bp->size + bp->size / 4) { + size = bp->size; + } + + b = nxt_buf_file_alloc(bp->mem_pool, size, bp->flags); + + if (nxt_fast_path(b != NULL)) { + bp->current = b; + bp->num++; + return NXT_OK; + } + + return NXT_ERROR; +} + + +nxt_int_t +nxt_buf_pool_mmap_alloc(nxt_buf_pool_t *bp, size_t size) +{ + nxt_buf_t *b; + + b = bp->current; + + if (b != NULL) { + return NXT_OK; + } + + b = bp->free; + + if (b != NULL) { + bp->current = b; + bp->free = b->next; + b->next = NULL; + return NXT_OK; + } + + if (bp->num >= bp->max) { + return NXT_AGAIN; + } + + if (size == 0 || size >= bp->size + bp->size / 4) { + size = bp->size; + } + + b = nxt_buf_mmap_alloc(bp->mem_pool, size); + + if (nxt_fast_path(b != NULL)) { + bp->mmap = 1; + bp->current = b; + bp->num++; + return NXT_OK; + } + + return NXT_ERROR; +} + + +void +nxt_buf_pool_free(nxt_buf_pool_t *bp, nxt_buf_t *b) +{ + size_t size; + + nxt_thread_log_debug("buf pool free: %p %p", b, b->mem.start); + + size = nxt_buf_mem_size(&b->mem); + + if (bp->mmap) { + nxt_mem_unmap(b->mem.start, &b->mmap, size); + } + + if (bp->destroy) { + + if (b == bp->current) { + bp->current = NULL; + } + + if (!bp->mmap) { + nxt_mem_free(bp->mem_pool, b->mem.start); + } + + nxt_buf_free(bp->mem_pool, b); + + return; + } + + if (bp->mmap) { + b->mem.pos = NULL; + b->mem.free = NULL; + nxt_buf_mem_set_size(&b->mem, size); + + } else { + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + } + + if (b != bp->current) { + b->next = bp->free; + bp->free = b; + } +} + + +void +nxt_buf_pool_destroy(nxt_buf_pool_t *bp) +{ + u_char *p; + nxt_buf_t *b; + + bp->destroy = 1; + + for (b = bp->free; b != NULL; b = b->next) { + p = b->mem.start; + nxt_buf_free(bp->mem_pool, b); + nxt_mem_free(bp->mem_pool, p); + } + + bp->free = b; /* NULL */ +} diff --git a/src/nxt_buf_pool.h b/src/nxt_buf_pool.h new file mode 100644 index 00000000..59609b26 --- /dev/null +++ b/src/nxt_buf_pool.h @@ -0,0 +1,80 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_BUF_POOL_H_INCLUDED_ +#define _NXT_BUF_POOL_H_INCLUDED_ + + +/* + * nxt_buf_pool_t is intended to allocate up to the "max" number + * memory, memory/file, or mmap/file buffers. A size of the buffers + * is set in the "size" field. The size however can be overridden in + * nxt_buf_pool_XXX_alloc() by the "size" argument if the argument is + * not zero and lesser than or equal to the "size" field multiplied + * by 1.25. The "flags" field is passed as the nxt_mem_buf() flags. + */ + +typedef struct { + nxt_buf_t *current; + nxt_buf_t *free; + nxt_mem_pool_t *mem_pool; + + uint16_t num; + uint16_t max; + + uint32_t size; + + uint8_t flags; /* 2 bits */ + uint8_t destroy; /* 1 bit */ + uint8_t mmap; /* 1 bit */ +} nxt_buf_pool_t; + + +NXT_EXPORT nxt_int_t nxt_buf_pool_mem_alloc(nxt_buf_pool_t *bp, size_t size); +NXT_EXPORT nxt_int_t nxt_buf_pool_file_alloc(nxt_buf_pool_t *bp, size_t size); +NXT_EXPORT nxt_int_t nxt_buf_pool_mmap_alloc(nxt_buf_pool_t *bp, size_t size); +NXT_EXPORT void nxt_buf_pool_free(nxt_buf_pool_t *bp, nxt_buf_t *b); +NXT_EXPORT void nxt_buf_pool_destroy(nxt_buf_pool_t *bp); + + +/* There is ready free buffer. */ + +#define \ +nxt_buf_pool_ready(bp) \ + ((bp)->free != NULL \ + || ((bp)->current != NULL \ + && (bp)->current->mem.free < (bp)->current->mem.end)) + + +/* A free buffer is allowed to be allocated. */ + +#define \ +nxt_buf_pool_obtainable(bp) \ + ((bp)->num < (bp)->max) + + +/* There is ready free buffer or it is allowed to be allocated. */ + +#define \ +nxt_buf_pool_available(bp) \ + (nxt_buf_pool_obtainable(bp) || nxt_buf_pool_ready(bp)) + + +/* Reserve allocation of "n" free buffers as they were allocated. */ + +#define \ +nxt_buf_pool_reserve(bp, n) \ + (bp)->num += (n) + + +/* Release a reservation. */ + +#define \ +nxt_buf_pool_release(bp, n) \ + (bp)->num -= (n) + + +#endif /* _NXT_BUF_POOL_H_INCLUDED_ */ diff --git a/src/nxt_cache.c b/src/nxt_cache.c new file mode 100644 index 00000000..409ba301 --- /dev/null +++ b/src/nxt_cache.c @@ -0,0 +1,643 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* A cache time resolution is 10ms. */ +#define \ +nxt_cache_time(thr) \ + (uint64_t) (nxt_thread_time(thr) * 100) + + +static nxt_int_t nxt_cache_lvlhsh_test(nxt_lvlhsh_query_t *lhq, void *data); +static nxt_work_handler_t nxt_cache_query_locked(nxt_cache_t *cache, + nxt_cache_query_t *q, nxt_lvlhsh_query_t *lhq); +static nxt_work_handler_t nxt_cache_node_hold(nxt_cache_t *cache, + nxt_cache_query_t *q, nxt_lvlhsh_query_t *lhq); +static nxt_work_handler_t nxt_cache_node_test(nxt_cache_t *cache, + nxt_cache_query_t *q); + +static void nxt_cache_wait_handler(nxt_thread_t *thr, void *obj, void *data); +static void nxt_cache_timeout_handler(nxt_thread_t *thr, void *obj, void *data); +static void nxt_cache_wake_handler(nxt_thread_t *thr, void *obj, void *data); +static ssize_t nxt_cache_release_locked(nxt_cache_t *cache, + nxt_cache_query_t *q, u_char *buf, size_t size); + +static nxt_cache_node_t *nxt_cache_node_alloc(nxt_cache_t *cache); +static void nxt_cache_node_free(nxt_cache_t *cache, nxt_cache_node_t *node, + nxt_bool_t fast); +static nxt_cache_query_wait_t *nxt_cache_query_wait_alloc(nxt_cache_t *cache, + nxt_bool_t *slow); +static void nxt_cache_query_wait_free(nxt_cache_t *cache, + nxt_cache_query_wait_t *qw); + + +/* STUB */ +nxt_int_t nxt_cache_shm_create(nxt_mem_zone_t *pool); +static void *nxt_cache_shm_alloc(void *data, size_t size, nxt_uint_t nalloc); +/**/ + + +nxt_int_t +nxt_cache_shm_create(nxt_mem_zone_t *mz) +{ + nxt_cache_t *cache; + + static const nxt_lvlhsh_proto_t proto nxt_aligned(64) = { + NXT_LVLHSH_LARGE_SLAB, + 0, + nxt_cache_lvlhsh_test, + (nxt_lvlhsh_alloc_t) nxt_cache_shm_alloc, + (nxt_lvlhsh_free_t) nxt_mem_zone_free, + }; + + cache = nxt_mem_zone_zalloc(mz, sizeof(nxt_cache_t)); + + if (cache == NULL) { + return NXT_ERROR; + } + + cache->proto = &proto; + cache->pool = mz; + + cache->start_time = nxt_cache_time(nxt_thread()); + + return NXT_OK; +} + + +static void * +nxt_cache_shm_alloc(void *data, size_t size, nxt_uint_t nalloc) +{ + return nxt_mem_zone_align(data, size, size); +} + + +void +nxt_cache_init(nxt_cache_t *cache) +{ + static const nxt_lvlhsh_proto_t proto nxt_aligned(64) = { + NXT_LVLHSH_LARGE_MEMALIGN, + 0, + nxt_cache_lvlhsh_test, + nxt_lvlhsh_alloc, + nxt_lvlhsh_free, + }; + + cache->proto = &proto; + + cache->start_time = nxt_cache_time(nxt_thread()); +} + + +static nxt_int_t +nxt_cache_lvlhsh_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + nxt_cache_node_t *node; + + node = data; + + if (nxt_str_eq(&lhq->key, node->key_data, node->key_len)) { + return NXT_OK; + } + + return NXT_DECLINED; +} + + +nxt_inline void +nxt_cache_lock(nxt_cache_t *cache) +{ + if (cache->shared) { + nxt_thread_spin_lock(&cache->lock); + } +} + + +nxt_inline void +nxt_cache_unlock(nxt_cache_t *cache) +{ + if (cache->shared) { + nxt_thread_spin_unlock(&cache->lock); + } +} + + +void +nxt_cache_query(nxt_cache_t *cache, nxt_cache_query_t *q) +{ + nxt_thread_t *thr; + nxt_lvlhsh_query_t lhq; + nxt_work_handler_t handler; + + thr = nxt_thread(); + + if (cache != NULL) { + lhq.key_hash = nxt_murmur_hash2(q->key_data, q->key_len); + lhq.replace = 0; + lhq.key.len = q->key_len; + lhq.key.data = q->key_data; + lhq.proto = cache->proto; + lhq.pool = cache->pool; + + q->now = nxt_cache_time(thr); + + nxt_cache_lock(cache); + + handler = nxt_cache_query_locked(cache, q, &lhq); + + nxt_cache_unlock(cache); + + } else { + handler = q->state->nocache_handler; + } + + handler(thr, q, NULL); +} + + +static nxt_work_handler_t +nxt_cache_query_locked(nxt_cache_t *cache, nxt_cache_query_t *q, + nxt_lvlhsh_query_t *lhq) +{ + nxt_int_t ret; + nxt_time_t expiry; + nxt_cache_node_t *node; + nxt_cache_query_state_t *state; + + if (q->hold) { + return nxt_cache_node_hold(cache, q, lhq); + } + + ret = nxt_lvlhsh_find(&cache->lvlhsh, lhq); + + state = q->state; + + if (ret != NXT_OK) { + /* NXT_DECLINED */ + return state->nocache_handler; + } + + node = lhq->value; + node->count++; + q->node = node; + + expiry = cache->start_time + node->expiry; + + if (q->now < expiry) { + return state->ready_handler; + } + + q->stale = 1; + + return state->stale_handler; +} + + +static nxt_work_handler_t +nxt_cache_node_hold(nxt_cache_t *cache, nxt_cache_query_t *q, + nxt_lvlhsh_query_t *lhq) +{ + nxt_int_t ret; + nxt_bool_t slow; + nxt_cache_node_t *node, *sentinel; + nxt_work_handler_t handler; + nxt_cache_query_wait_t *qw; + nxt_cache_query_state_t *state; + + state = q->state; + sentinel = nxt_cache_node_alloc(cache); + + if (nxt_slow_path(sentinel == NULL)) { + return state->error_handler; + } + + sentinel->key_data = q->key_data; + sentinel->key_len = q->key_len; + lhq->value = sentinel; + + /* + * Try to insert an empty sentinel node to hold updating + * process if there is no existent cache node in cache. + */ + ret = nxt_lvlhsh_insert(&cache->lvlhsh, lhq); + + if (ret == NXT_OK) { + /* The sentinel node was successully added. */ + + q->node = sentinel; + sentinel->updating = 1; + return state->update_handler; + } + + nxt_cache_node_free(cache, sentinel, 1); + + if (ret == NXT_ERROR) { + return state->error_handler; + } + + /* NXT_DECLINED: a cache node exists. */ + + node = lhq->value; + node->count++; + q->node = node; + + handler = nxt_cache_node_test(cache, q); + if (handler != NULL) { + return handler; + } + + /* Add the node to a wait queue. */ + + qw = nxt_cache_query_wait_alloc(cache, &slow); + if (nxt_slow_path(qw == NULL)) { + return state->error_handler; + } + + if (slow) { + /* The node state may have been changed during slow allocation. */ + + handler = nxt_cache_node_test(cache, q); + if (handler != NULL) { + nxt_cache_query_wait_free(cache, qw); + return handler; + } + } + + qw->query = q; + qw->next = node->waiting; + qw->busy = 0; + qw->deleted = 0; + qw->pid = nxt_pid; + qw->engine = nxt_thread_event_engine(); + qw->handler = nxt_cache_wake_handler; + qw->cache = cache; + + node->waiting = qw; + + return nxt_cache_wait_handler; +} + + +static nxt_work_handler_t +nxt_cache_node_test(nxt_cache_t *cache, nxt_cache_query_t *q) +{ + nxt_time_t expiry; + nxt_cache_node_t *node; + nxt_cache_query_state_t *state; + + q->stale = 0; + state = q->state; + node = q->node; + + expiry = cache->start_time + node->expiry; + + if (q->now < expiry) { + return state->ready_handler; + } + + /* + * A valid stale or empty sentinel cache node. + * The sentinel node can be only in updating state. + */ + + if (node->updating) { + + if (node->expiry != 0) { + /* A valid stale cache node. */ + + q->stale = 1; + + if (q->use_stale) { + return state->stale_handler; + } + } + + /* A sentinel node. */ + return NULL; + } + + /* A valid stale cache node is not being updated now. */ + + q->stale = 1; + + if (q->use_stale) { + + if (q->update_stale) { + node->updating = 1; + return state->update_stale_handler; + } + + return state->stale_handler; + } + + node->updating = 1; + return state->update_handler; +} + + +static void +nxt_cache_wait_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_timer_t *ev; + nxt_cache_query_t *cq; + + cq = obj; + + if (cq->timeout != 0) { + + ev = &cq->timer; + + if (ev->state == NXT_EVENT_TIMER_DISABLED) { + ev->handler = nxt_cache_timeout_handler; + nxt_event_timer_ident(ev, -1); + + nxt_event_timer_add(thr->engine, ev, cq->timeout); + } + } +} + + +static void +nxt_cache_timeout_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_cache_query_t *cq; + nxt_event_timer_t *ev; + + ev = obj; + + cq = nxt_event_timer_data(ev, nxt_cache_query_t, timer); + + cq->state->timeout_handler(thr, cq, NULL); +} + + +static void +nxt_cache_wake_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_cache_t *cache; + nxt_work_handler_t handler; + nxt_cache_query_t *q; + nxt_cache_query_wait_t *qw; + + qw = obj; + q = qw->query; + cache = qw->cache; + + nxt_cache_lock(cache); + + handler = nxt_cache_node_test(cache, q); + + if (handler != NULL) { + nxt_cache_query_wait_free(cache, qw); + + } else { + /* Wait again. */ + qw->next = q->node->waiting; + q->node->waiting = qw; + } + + nxt_cache_unlock(cache); + + handler(thr, q, NULL); +} + + +nxt_int_t +nxt_cache_update(nxt_cache_t *cache, nxt_cache_query_t *q) +{ + nxt_int_t ret; + nxt_cache_node_t *node; + nxt_lvlhsh_query_t lhq; + + node = q->node; + + node->accessed = nxt_cache_time(nxt_thread()) - cache->start_time; + + node->updating = 0; + node->count = 1; + + lhq.key_hash = nxt_murmur_hash2(node->key_data, node->key_len); + lhq.replace = 1; + lhq.key.len = node->key_len; + lhq.key.data = node->key_data; + lhq.value = node; + lhq.proto = cache->proto; + lhq.pool = cache->pool; + + nxt_cache_lock(cache); + + ret = nxt_lvlhsh_insert(&cache->lvlhsh, &lhq); + + if (nxt_fast_path(ret != NXT_OK)) { + + nxt_queue_insert_head(&cache->expiry_queue, &node->link); + + node = lhq.value; + + if (node != NULL) { + /* A replaced node. */ + + nxt_queue_remove(&node->link); + + if (node->count != 0) { + node->deleted = 1; + + } else { + // delete cache node + } + } + } + + nxt_cache_unlock(cache); + + return ret; +} + + +void +nxt_cache_release(nxt_cache_t *cache, nxt_cache_query_t *q) +{ + u_char *p, *data; + size_t size; + ssize_t ret; + nxt_thread_t *thr; + u_char buf[1024]; + + thr = nxt_thread(); + q->now = nxt_cache_time(thr); + + p = buf; + size = sizeof(buf); + + for ( ;; ) { + nxt_cache_lock(cache); + + ret = nxt_cache_release_locked(cache, q, p, size); + + nxt_cache_unlock(cache); + + if (ret == 0) { + return; + } + + size = nxt_abs(ret); + + data = nxt_malloc(size); + + if (data == NULL) { + /* TODO: retry */ + return; + } + + if (ret < 0) { + p = data; + continue; + } + + if (p != data) { + nxt_memcpy(data, p, size); + } + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + cache->delete_handler, data, NULL, thr->log); + } +} + + +static ssize_t +nxt_cache_release_locked(nxt_cache_t *cache, nxt_cache_query_t *q, + u_char *buf, size_t size) +{ + ssize_t ret; + nxt_cache_node_t *node; + + node = q->node; + node->count--; + + if (node->count != 0) { + return 0; + } + + if (!node->deleted) { + /* + * A cache node is locked whilst its count is non zero. + * To minimize number of operations the node's place in expiry + * queue can be updated only if the node is not currently used. + */ + node->accessed = q->now - cache->start_time; + + nxt_queue_remove(&node->link); + nxt_queue_insert_head(&cache->expiry_queue, &node->link); + + return 0; + } + + ret = 0; +#if 0 + + ret = cache->delete_copy(cache, node, buf, size); + + if (ret < 0) { + return ret; + } + +#endif + + nxt_cache_node_free(cache, node, 0); + + return ret; +} + + +static nxt_cache_node_t * +nxt_cache_node_alloc(nxt_cache_t *cache) +{ + nxt_queue_link_t *link; + nxt_cache_node_t *node; + + link = nxt_queue_first(&cache->free_nodes); + + if (nxt_fast_path(link != nxt_queue_tail(&cache->free_nodes))) { + cache->nfree_nodes--; + nxt_queue_remove(link); + + node = nxt_queue_link_data(link, nxt_cache_node_t, link); + nxt_memzero(node, sizeof(nxt_cache_node_t)); + + return node; + } + + nxt_cache_unlock(cache); + + node = cache->alloc(cache->data, sizeof(nxt_cache_node_t)); + + nxt_cache_lock(cache); + + return node; +} + + +static void +nxt_cache_node_free(nxt_cache_t *cache, nxt_cache_node_t *node, nxt_bool_t fast) +{ + if (fast || cache->nfree_nodes < 32) { + nxt_queue_insert_head(&cache->free_nodes, &node->link); + cache->nfree_nodes++; + return; + } + + nxt_cache_unlock(cache); + + cache->free(cache->data, node); + + nxt_cache_lock(cache); +} + + +static nxt_cache_query_wait_t * +nxt_cache_query_wait_alloc(nxt_cache_t *cache, nxt_bool_t *slow) +{ + nxt_cache_query_wait_t *qw; + + qw = cache->free_query_wait; + + if (nxt_fast_path(qw != NULL)) { + cache->free_query_wait = qw->next; + cache->nfree_query_wait--; + + *slow = 0; + return qw; + } + + nxt_cache_unlock(cache); + + qw = cache->alloc(cache->data, sizeof(nxt_cache_query_wait_t)); + *slow = 1; + + nxt_cache_lock(cache); + + return qw; +} + + +static void +nxt_cache_query_wait_free(nxt_cache_t *cache, nxt_cache_query_wait_t *qw) +{ + if (cache->nfree_query_wait < 32) { + qw->next = cache->free_query_wait; + cache->free_query_wait = qw; + cache->nfree_query_wait++; + return; + } + + nxt_cache_unlock(cache); + + cache->free(cache->data, qw); + + nxt_cache_lock(cache); +} diff --git a/src/nxt_cache.h b/src/nxt_cache.h new file mode 100644 index 00000000..74cdffb2 --- /dev/null +++ b/src/nxt_cache.h @@ -0,0 +1,122 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_CACHE_INCLUDED_ +#define _NXT_CACHE_INCLUDED_ + + +typedef struct nxt_cache_query_s nxt_cache_query_t; +typedef struct nxt_cache_query_wait_s nxt_cache_query_wait_t; + + +typedef struct { + uint32_t shared; /* 1 bit */ + nxt_thread_spinlock_t lock; + + nxt_lvlhsh_t lvlhsh; + const nxt_lvlhsh_proto_t *proto; + void *pool; + + nxt_queue_t expiry_queue; + + nxt_queue_t free_nodes; + uint32_t nfree_nodes; + + uint32_t nfree_query_wait; + nxt_cache_query_wait_t *free_query_wait; + + uint64_t start_time; + + /* STUB: use nxt_lvlhsh_proto_t */ + void *(*alloc)(void *data, size_t size); + void (*free)(void *data, void *p); + void *data; + + nxt_work_handler_t delete_handler; +} nxt_cache_t; + + +typedef struct { + u_char *key_data; + + uint16_t key_len; /* 16 bits */ + uint8_t uses; /* 8 bits */ + uint8_t updating:1; + uint8_t deleted:1; + + uint32_t count; + + /* Times relative to the cache->start_time. */ + uint32_t expiry; + uint32_t accessed; + + nxt_off_t size; + + nxt_queue_link_t link; + + nxt_cache_query_wait_t *waiting; +} nxt_cache_node_t; + + +struct nxt_cache_query_wait_s { + nxt_cache_query_t *query; + nxt_cache_query_wait_t *next; + + uint8_t busy; /* 1 bit */ + uint8_t deleted; /* 1 bit */ + + nxt_pid_t pid; + nxt_event_engine_t *engine; + nxt_work_handler_t handler; + nxt_cache_t *cache; +}; + + +typedef struct { + nxt_work_handler_t nocache_handler; + nxt_work_handler_t ready_handler; + nxt_work_handler_t stale_handler; + nxt_work_handler_t update_stale_handler; + nxt_work_handler_t update_handler; + nxt_work_handler_t timeout_handler; + nxt_work_handler_t error_handler; +} nxt_cache_query_state_t; + + +struct nxt_cache_query_s { + u_char *key_data; + + uint16_t key_len; /* 16 bits */ +#if (NXT_64_BIT) + uint8_t hold; /* 1 bit */ + uint8_t use_stale; /* 1 bit */ + uint8_t update_stale; /* 1 bit */ + uint8_t stale; /* 1 bit */ +#else + uint8_t hold:1; + uint8_t use_stale:1; + uint8_t update_stale:1; + uint8_t stale:1; +#endif + + nxt_cache_node_t *node; + nxt_cache_query_t *next; + nxt_cache_query_state_t *state; + + nxt_time_t now; + + nxt_msec_t timeout; + nxt_event_timer_t timer; +}; + + +NXT_EXPORT void nxt_cache_init(nxt_cache_t *cache); +NXT_EXPORT void nxt_cache_query(nxt_cache_t *cache, nxt_cache_query_t *q); +NXT_EXPORT void nxt_cache_release(nxt_cache_t *cache, nxt_cache_query_t *q); +NXT_EXPORT nxt_int_t nxt_cache_update(nxt_cache_t *cache, nxt_cache_query_t *q); + + +#endif /* _NXT_CACHE_INCLUDED_ */ diff --git a/src/nxt_chan.c b/src/nxt_chan.c new file mode 100644 index 00000000..cc6ba786 --- /dev/null +++ b/src/nxt_chan.c @@ -0,0 +1,456 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_chan_write_handler(nxt_thread_t *thr, void *obj, void *data); +static void nxt_chan_read_handler(nxt_thread_t *thr, void *obj, void *data); +static void nxt_chan_read_msg_process(nxt_thread_t *thr, nxt_chan_t *chan, + nxt_chan_msg_t *msg, nxt_fd_t fd, nxt_buf_t *b, size_t size); +static nxt_buf_t *nxt_chan_buf_alloc(nxt_chan_t *chan); +static void nxt_chan_buf_free(nxt_chan_t *chan, nxt_buf_t *b); +static void nxt_chan_error_handler(nxt_thread_t *thr, void *obj, void *data); + + +nxt_chan_t * +nxt_chan_alloc(void) +{ + nxt_chan_t *chan; + nxt_mem_pool_t *mp; + + mp = nxt_mem_pool_create(1024); + + if (nxt_fast_path(mp != NULL)) { + /* This allocation cannot fail. */ + chan = nxt_mem_zalloc(mp, sizeof(nxt_chan_t)); + chan->mem_pool = mp; + + chan->pair[0] = -1; + chan->pair[1] = -1; + + nxt_queue_init(&chan->messages); + + return chan; + } + + return NULL; +} + + +nxt_chan_t * +nxt_chan_create(size_t max_size) +{ + nxt_int_t sndbuf, rcvbuf, size; + nxt_chan_t *chan; + nxt_socket_t snd, rcv; + + chan = nxt_chan_alloc(); + if (nxt_slow_path(chan == NULL)) { + return NULL; + } + + if (nxt_slow_path(nxt_socketpair_create(chan->pair) != NXT_OK)) { + goto socketpair_fail; + } + + snd = chan->pair[1]; + + sndbuf = nxt_socket_getsockopt(snd, SOL_SOCKET, SO_SNDBUF); + if (nxt_slow_path(sndbuf < 0)) { + goto getsockopt_fail; + } + + rcv = chan->pair[0]; + + rcvbuf = nxt_socket_getsockopt(rcv, SOL_SOCKET, SO_RCVBUF); + if (nxt_slow_path(rcvbuf < 0)) { + goto getsockopt_fail; + } + + if (max_size == 0) { + max_size = 16 * 1024; + } + + if ((size_t) sndbuf < max_size) { + /* + * On Unix domain sockets + * Linux uses 224K on both send and receive directions; + * FreeBSD, MacOSX, NetBSD, and OpenBSD use 2K buffer size + * on send direction and 4K buffer size on receive direction; + * Solaris uses 16K on send direction and 5K on receive direction. + */ + (void) nxt_socket_setsockopt(snd, SOL_SOCKET, SO_SNDBUF, max_size); + + sndbuf = nxt_socket_getsockopt(snd, SOL_SOCKET, SO_SNDBUF); + if (nxt_slow_path(sndbuf < 0)) { + goto getsockopt_fail; + } + + size = sndbuf * 4; + + if (rcvbuf < size) { + (void) nxt_socket_setsockopt(rcv, SOL_SOCKET, SO_RCVBUF, size); + + rcvbuf = nxt_socket_getsockopt(rcv, SOL_SOCKET, SO_RCVBUF); + if (nxt_slow_path(rcvbuf < 0)) { + goto getsockopt_fail; + } + } + } + + chan->max_size = nxt_min(max_size, (size_t) sndbuf); + chan->max_share = (64 * 1024); + + return chan; + +getsockopt_fail: + + nxt_socket_close(chan->pair[0]); + nxt_socket_close(chan->pair[1]); + +socketpair_fail: + + nxt_mem_pool_destroy(chan->mem_pool); + + return NULL; +} + + +void +nxt_chan_destroy(nxt_chan_t *chan) +{ + nxt_socket_close(chan->socket.fd); + nxt_mem_pool_destroy(chan->mem_pool); +} + + +void +nxt_chan_write_enable(nxt_thread_t *thr, nxt_chan_t *chan) +{ + chan->socket.fd = chan->pair[1]; + chan->socket.log = &nxt_main_log; + chan->socket.write_ready = 1; + + chan->socket.write_work_queue = &thr->work_queue.main; + chan->socket.write_handler = nxt_chan_write_handler; + chan->socket.error_handler = nxt_chan_error_handler; +} + + +void +nxt_chan_write_close(nxt_chan_t *chan) +{ + nxt_socket_close(chan->pair[1]); + chan->pair[1] = -1; +} + + +nxt_int_t +nxt_chan_write(nxt_chan_t *chan, nxt_uint_t type, nxt_fd_t fd, uint32_t stream, + nxt_buf_t *b) +{ + nxt_thread_t *thr; + nxt_queue_link_t *link; + nxt_chan_send_msg_t *msg; + + for (link = nxt_queue_first(&chan->messages); + link != nxt_queue_tail(&chan->messages); + link = nxt_queue_next(link)) + { + msg = (nxt_chan_send_msg_t *) link; + + if (msg->chan_msg.stream == stream) { + /* + * An fd is ignored since a file descriptor + * must be sent only in the first message of a stream. + */ + nxt_buf_chain_add(&msg->buf, b); + + return NXT_OK; + } + } + + msg = nxt_mem_cache_zalloc0(chan->mem_pool, sizeof(nxt_chan_send_msg_t)); + if (nxt_slow_path(msg == NULL)) { + return NXT_ERROR; + } + + msg->buf = b; + msg->fd = fd; + msg->share = 0; + + msg->chan_msg.stream = stream; + msg->chan_msg.type = type; + msg->chan_msg.last = 0; + + nxt_queue_insert_tail(&chan->messages, &msg->link); + + if (chan->socket.write_ready) { + thr = nxt_thread(); + nxt_chan_write_handler(thr, chan, NULL); + } + + return NXT_OK; +} + + +static void +nxt_chan_write_handler(nxt_thread_t *thr, void *obj, void *data) +{ + ssize_t n; + nxt_uint_t niob; + nxt_chan_t *chan; + struct iovec iob[NXT_IOBUF_MAX]; + nxt_queue_link_t *link; + nxt_chan_send_msg_t *msg; + nxt_sendbuf_coalesce_t sb; + + chan = obj; + + do { + link = nxt_queue_first(&chan->messages); + + if (link == nxt_queue_tail(&chan->messages)) { + nxt_event_fd_block_write(thr->engine, &chan->socket); + return; + } + + msg = (nxt_chan_send_msg_t *) link; + + nxt_iobuf_set(&iob[0], &msg->chan_msg, sizeof(nxt_chan_msg_t)); + + sb.buf = msg->buf; + sb.iobuf = &iob[1]; + sb.nmax = NXT_IOBUF_MAX - 1; + sb.sync = 0; + sb.last = 0; + sb.size = sizeof(nxt_chan_msg_t); + sb.limit = chan->max_size; + + niob = nxt_sendbuf_mem_coalesce(&sb); + + msg->chan_msg.last = sb.last; + + n = nxt_socketpair_send(&chan->socket, msg->fd, iob, niob + 1); + + if (n > 0) { + if (nxt_slow_path((size_t) n != sb.size)) { + nxt_log_alert(thr->log, + "chan %d: short write: %z instead of %uz", + chan->socket.fd, n, sb.size); + goto fail; + } + + msg->buf = nxt_sendbuf_completion(thr, + chan->socket.write_work_queue, + msg->buf, + n - sizeof(nxt_chan_msg_t)); + + if (msg->buf != NULL) { + /* + * A file descriptor is sent only + * in the first message of a stream. + */ + msg->fd = -1; + msg->share += n; + + if (msg->share >= chan->max_share) { + msg->share = 0; + nxt_queue_remove(link); + nxt_queue_insert_tail(&chan->messages, link); + } + + } else { + nxt_queue_remove(link); + nxt_mem_cache_free0(chan->mem_pool, msg, + sizeof(nxt_chan_send_msg_t)); + } + + } else if (nxt_slow_path(n == NXT_ERROR)) { + goto fail; + } + + /* n == NXT_AGAIN */ + + } while (chan->socket.write_ready); + + if (nxt_event_fd_is_disabled(chan->socket.write)) { + nxt_event_fd_enable_write(thr->engine, &chan->socket); + } + + return; + +fail: + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_chan_error_handler, + &chan->socket, NULL, chan->socket.log); +} + + +void +nxt_chan_read_enable(nxt_thread_t *thr, nxt_chan_t *chan) +{ + chan->socket.fd = chan->pair[0]; + chan->socket.log = &nxt_main_log; + + chan->socket.read_work_queue = &thr->work_queue.main; + chan->socket.read_handler = nxt_chan_read_handler; + chan->socket.error_handler = nxt_chan_error_handler; + + nxt_event_fd_enable_read(thr->engine, &chan->socket); +} + + +void +nxt_chan_read_close(nxt_chan_t *chan) +{ + nxt_socket_close(chan->pair[0]); + chan->pair[0] = -1; +} + + +static void +nxt_chan_read_handler(nxt_thread_t *thr, void *obj, void *data) +{ + ssize_t n; + nxt_fd_t fd; + nxt_buf_t *b; + nxt_chan_t *chan; + nxt_iobuf_t iob[2]; + nxt_chan_msg_t msg; + + chan = obj; + + for ( ;; ) { + + b = nxt_chan_buf_alloc(chan); + + if (nxt_slow_path(b == NULL)) { + /* TODO: disable event for some time */ + } + + nxt_iobuf_set(&iob[0], &msg, sizeof(nxt_chan_msg_t)); + nxt_iobuf_set(&iob[1], b->mem.pos, chan->max_size); + + n = nxt_socketpair_recv(&chan->socket, &fd, iob, 2); + + if (n > 0) { + nxt_chan_read_msg_process(thr, chan, &msg, fd, b, n); + + if (b->mem.pos == b->mem.free) { + + if (b->next != NULL) { + /* A sync buffer */ + nxt_buf_free(chan->mem_pool, b->next); + } + + nxt_chan_buf_free(chan, b); + } + + if (chan->socket.read_ready) { + continue; + } + + return; + } + + if (n == NXT_AGAIN) { + nxt_chan_buf_free(chan, b); + + nxt_event_fd_enable_read(thr->engine, &chan->socket); + return; + } + + /* n == 0 || n == NXT_ERROR */ + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_chan_error_handler, + &chan->socket, NULL, chan->socket.log); + return; + } +} + + +static void +nxt_chan_read_msg_process(nxt_thread_t *thr, nxt_chan_t *chan, + nxt_chan_msg_t *msg, nxt_fd_t fd, nxt_buf_t *b, size_t size) +{ + nxt_buf_t *sync; + nxt_chan_recv_msg_t recv_msg; + + if (nxt_slow_path(size < sizeof(nxt_chan_msg_t))) { + nxt_log_alert(chan->socket.log, "chan %d: too small message:%uz", + chan->socket.fd, size); + goto fail; + } + + recv_msg.stream = msg->stream; + recv_msg.type = msg->type; + recv_msg.fd = fd; + recv_msg.buf = b; + recv_msg.chan = chan; + + b->mem.free += size - sizeof(nxt_chan_msg_t); + + if (msg->last) { + sync = nxt_buf_sync_alloc(chan->mem_pool, NXT_BUF_SYNC_LAST); + if (nxt_slow_path(sync == NULL)) { + goto fail; + } + + b->next = sync; + } + + chan->handler(thr, &recv_msg); + + return; + +fail: + + if (fd != -1) { + nxt_fd_close(fd); + } +} + + +static nxt_buf_t * +nxt_chan_buf_alloc(nxt_chan_t *chan) +{ + nxt_buf_t *b; + + if (chan->free_bufs != NULL) { + b = chan->free_bufs; + chan->free_bufs = b->next; + + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + + } else { + b = nxt_buf_mem_alloc(chan->mem_pool, chan->max_size, 0); + if (nxt_slow_path(b == NULL)) { + return NULL; + } + } + + return b; +} + + +static void +nxt_chan_buf_free(nxt_chan_t *chan, nxt_buf_t *b) +{ + b->next = chan->free_bufs; + chan->free_bufs = b; +} + + +static void +nxt_chan_error_handler(nxt_thread_t *thr, void *obj, void *data) +{ + /* TODO */ +} diff --git a/src/nxt_chan.h b/src/nxt_chan.h new file mode 100644 index 00000000..f9550f17 --- /dev/null +++ b/src/nxt_chan.h @@ -0,0 +1,73 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_CHAN_H_INCLUDED_ +#define _NXT_UNIX_CHAN_H_INCLUDED_ + + +typedef struct { + uint32_t stream; + + uint16_t type; + uint8_t last; /* 1 bit */ +} nxt_chan_msg_t; + + +typedef struct { + nxt_queue_link_t link; + nxt_buf_t *buf; + size_t share; + nxt_fd_t fd; + nxt_chan_msg_t chan_msg; +} nxt_chan_send_msg_t; + + +typedef struct nxt_chan_recv_msg_s nxt_chan_recv_msg_t; +typedef void (*nxt_chan_handler_t)(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg); + + +typedef struct { + /* Must be the first field. */ + nxt_event_fd_t socket; + + nxt_queue_t messages; /* of nxt_chan_send_msg_t */ + + /* Maximum size of message part. */ + uint32_t max_size; + /* Maximum interleave of message parts. */ + uint32_t max_share; + + nxt_chan_handler_t handler; + void *data; + + nxt_mem_pool_t *mem_pool; + nxt_buf_t *free_bufs; + nxt_socket_t pair[2]; +} nxt_chan_t; + + +struct nxt_chan_recv_msg_s { + uint32_t stream; + uint16_t type; + + nxt_fd_t fd; + nxt_buf_t *buf; + nxt_chan_t *chan; +}; + + +NXT_EXPORT nxt_chan_t *nxt_chan_alloc(void); +NXT_EXPORT nxt_chan_t *nxt_chan_create(size_t bufsize); +NXT_EXPORT void nxt_chan_destroy(nxt_chan_t *chan); +NXT_EXPORT void nxt_chan_write_enable(nxt_thread_t *thr, nxt_chan_t *chan); +NXT_EXPORT void nxt_chan_write_close(nxt_chan_t *chan); +NXT_EXPORT void nxt_chan_read_enable(nxt_thread_t *thr, nxt_chan_t *chan); +NXT_EXPORT void nxt_chan_read_close(nxt_chan_t *chan); +NXT_EXPORT nxt_int_t nxt_chan_write(nxt_chan_t *chan, nxt_uint_t type, + nxt_fd_t fd, uint32_t stream, nxt_buf_t *b); + + +#endif /* _NXT_UNIX_CHAN_H_INCLUDED_ */ diff --git a/src/nxt_clang.h b/src/nxt_clang.h new file mode 100644 index 00000000..0ed55b64 --- /dev/null +++ b/src/nxt_clang.h @@ -0,0 +1,214 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_CLANG_H_INCLUDED_ +#define _NXT_CLANG_H_INCLUDED_ + + +#define nxt_inline static inline __attribute__((always_inline)) +#define nxt_noinline __attribute__((noinline)) +#define nxt_cdecl + + +#if (NXT_CLANG) + +/* Any __asm__ directive disables loop vectorization in GCC and Clang. */ +#define \ +nxt_pragma_loop_disable_vectorization \ + __asm__("") + +#else + +#define \ +nxt_pragma_loop_disable_vectorization + +#endif + + +#if (NXT_HAVE_BUILTIN_EXPECT) + +#define \ +nxt_fast_path(x) \ + __builtin_expect((long) (x), 1) + +#define \ +nxt_slow_path(x) \ + __builtin_expect((long) (x), 0) + + +#else + +#define \ +nxt_fast_path(x) \ + (x) + +#define \ +nxt_slow_path(x) \ + (x) + +#endif + + +#if (NXT_HAVE_BUILTIN_UNREACHABLE) + +#define \ +nxt_unreachable() \ + __builtin_unreachable() + +#else + +#define \ +nxt_unreachable() + +#endif + + +#if (NXT_HAVE_BUILTIN_PREFETCH) + +#define \ +nxt_prefetch(a) \ + __builtin_prefetch(a) + +#else + +#define \ +nxt_prefetch(a) + +#endif + + +#if (NXT_HAVE_GCC_ATTRIBUTE_VISIBILITY) + +#define NXT_EXPORT __attribute__((visibility("default"))) + +#else + +#define NXT_EXPORT + +#endif + + +#if (NXT_HAVE_GCC_ATTRIBUTE_MALLOC) + +#define NXT_MALLOC_LIKE __attribute__((__malloc__)) + +#else + +#define NXT_MALLOC_LIKE + +#endif + + +#if (NXT_HAVE_GCC_ATTRIBUTE_ALIGNED) + +#define nxt_aligned(x) __attribute__((aligned(x))) + +#else + +#define nxt_aligned(x) + +#endif + + +#ifndef NXT_ALIGNMENT + +#if (NXT_SOLARIS) +#define NXT_ALIGNMENT _POINTER_ALIGNMENT /* x86_64: 8, i386: 4 */ + /* sparcv9: 8, sparcv8: 4 */ +#elif (__i386__ || __i386) +#define NXT_ALIGNMENT 4 + +#elif (__arm__) +#define NXT_ALIGNMENT 8 /* 32-bit ARM may use 64-bit load/store */ + +#elif (__ia64__) +#define NXT_ALIGNMENT 8 /* long long */ + +#else +#define NXT_ALIGNMENT NXT_PTR_SIZE +#endif + +#endif + + +#ifndef NXT_MAX_ALIGNMENT + +#if (NXT_SOLARIS) +#define NXT_MAX_ALIGNMENT _MAX_ALIGNMENT /* x86_64: 16, i386: 4 */ + /* sparcv9: 16, sparcv8: 8 */ +#elif (__i386__ || __i386) +#define NXT_MAX_ALIGNMENT 4 + +#elif (__arm__) +#define NXT_MAX_ALIGNMENT 16 + +#elif (__ia64__) +#define NXT_MAX_ALIGNMENT 16 + +#else +#define NXT_MAX_ALIGNMENT 16 +#endif + +#endif + + +#define \ +nxt_alloca(size) \ + alloca(size) + + +#define \ +nxt_container_of(p, type, field) \ + (type *) ((u_char *) (p) - offsetof(type, field)) + + +#define \ +nxt_nitems(x) \ + (sizeof(x) / sizeof((x)[0])) + + +/* GCC and Clang use __builtin_abs() instead of libc abs(). */ + +#define \ +nxt_abs(val) \ + abs(val) + + +#define \ +nxt_max(val1, val2) \ + ((val1 < val2) ? (val2) : (val1)) + + +#define \ +nxt_min(val1, val2) \ + ((val1 > val2) ? (val2) : (val1)) + + +#define \ +nxt_bswap32(val) \ + ( ((val) >> 24) \ + | (((val) & 0x00ff0000) >> 8) \ + | (((val) & 0x0000ff00) << 8) \ + | ((val) << 24)) + + + +#define \ +nxt_align_size(d, a) \ + (((d) + ((size_t) (a) - 1)) & ~((size_t) (a) - 1)) + + +#define \ +nxt_align_ptr(p, a) \ + (u_char *) (((uintptr_t) (p) + ((uintptr_t) (a) - 1)) \ + & ~((uintptr_t) (a) - 1)) + +#define \ +nxt_trunc_ptr(p, a) \ + (u_char *) ((uintptr_t) (p) & ~((uintptr_t) (a) - 1)) + + +#endif /* _NXT_CLANG_H_INCLUDED_ */ diff --git a/src/nxt_cyassl.c b/src/nxt_cyassl.c new file mode 100644 index 00000000..404b89fe --- /dev/null +++ b/src/nxt_cyassl.c @@ -0,0 +1,621 @@ + +/* + * Copyright (C) NGINX, Inc. + * Copyright (C) Igor Sysoev + */ + +#include <nxt_main.h> +#include <cyassl/ssl.h> +#include <cyassl/error-ssl.h> + + +typedef struct { + CYASSL *session; + + int ssl_error; + uint8_t times; /* 2 bits */ + + nxt_buf_mem_t buffer; +} nxt_cyassl_conn_t; + + +static nxt_int_t nxt_cyassl_server_init(nxt_ssltls_conf_t *conf); +static void nxt_cyassl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c); +static void nxt_cyassl_session_cleanup(void *data); +static int nxt_cyassl_io_recv(CYASSL *ssl, char *buf, int size, void *data); +static int nxt_cyassl_io_send(CYASSL *ssl, char *buf, int size, void *data); +static void nxt_cyassl_conn_handshake(nxt_thread_t *thr, void *obj, void *data); +static void nxt_cyassl_conn_io_read(nxt_thread_t *thr, void *obj, void *data); +static void nxt_cyassl_conn_io_shutdown(nxt_thread_t *thr, void *obj, + void *data); +static ssize_t nxt_cyassl_conn_io_write_chunk(nxt_thread_t *thr, + nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); +static ssize_t nxt_cyassl_conn_io_send(nxt_event_conn_t *c, void *buf, + size_t size); +static nxt_int_t nxt_cyassl_conn_test_error(nxt_thread_t *thr, + nxt_event_conn_t *c, int err, nxt_work_handler_t handler); +static void nxt_cdecl nxt_cyassl_conn_error(nxt_event_conn_t *c, nxt_err_t err, + const char *fmt, ...); +static nxt_uint_t nxt_cyassl_log_error_level(nxt_event_conn_t *c, nxt_err_t err, + int ssl_error); +static void nxt_cdecl nxt_cyassl_log_error(nxt_uint_t level, nxt_log_t *log, + int ret, const char *fmt, ...); +static u_char *nxt_cyassl_copy_error(int err, u_char *p, u_char *end); + + +const nxt_ssltls_lib_t nxt_cyassl_lib = { + nxt_cyassl_server_init, + NULL, +}; + + +static nxt_event_conn_io_t nxt_cyassl_event_conn_io = { + NULL, + NULL, + + nxt_cyassl_conn_io_read, + NULL, + NULL, + + nxt_event_conn_io_write, + nxt_cyassl_conn_io_write_chunk, + NULL, + NULL, + nxt_cyassl_conn_io_send, + + nxt_cyassl_conn_io_shutdown, +}; + + +static nxt_int_t +nxt_cyassl_start(void) +{ + int err; + nxt_thread_t *thr; + static nxt_bool_t started; + + if (nxt_fast_path(started)) { + return NXT_OK; + } + + started = 1; + + thr = nxt_thread(); + + /* TODO: CyaSSL_Cleanup() */ + + err = CyaSSL_Init(); + if (err != SSL_SUCCESS) { + nxt_cyassl_log_error(NXT_LOG_CRIT, thr->log, err, + "CyaSSL_Init() failed"); + return NXT_ERROR; + } + + nxt_thread_log_error(NXT_LOG_INFO, "CyaSSL version: %s", + LIBCYASSL_VERSION_STRING); + + /* CyaSSL_SetLoggingCb */ + /* CyaSSL_SetAllocators */ + + return NXT_OK; +} + + +static nxt_int_t +nxt_cyassl_server_init(nxt_ssltls_conf_t *conf) +{ + int err; + char *certificate, *key; + CYASSL_CTX *ctx; + nxt_thread_t *thr; + + thr = nxt_thread(); + + if (nxt_slow_path(nxt_cyassl_start() != NXT_OK)) { + return NXT_ERROR; + } + + ctx = CyaSSL_CTX_new(CyaSSLv23_server_method()); + if (ctx == NULL) { + nxt_cyassl_log_error(NXT_LOG_CRIT, thr->log, 0, + "CyaSSL_CTX_new() failed"); + return NXT_ERROR; + } + + conf->ctx = ctx; + conf->conn_init = nxt_cyassl_conn_init; + + certificate = conf->certificate; + + err = CyaSSL_CTX_use_certificate_file(ctx, certificate, SSL_FILETYPE_PEM); + if (err != SSL_SUCCESS) { + nxt_cyassl_log_error(NXT_LOG_CRIT, thr->log, err, + "CyaSSL_CTX_use_certificate_file(\"%s\") failed", + certificate); + goto fail; + } + + key = conf->certificate_key; + + err = CyaSSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM); + if (err != SSL_SUCCESS) { + nxt_cyassl_log_error(NXT_LOG_CRIT, thr->log, err, + "CyaSSL_CTX_use_PrivateKey_file(\"%s\") failed", + key); + goto fail; + } + + if (conf->ciphers != NULL) { + err = CyaSSL_CTX_set_cipher_list(ctx, conf->ciphers); + if (err != SSL_SUCCESS) { + nxt_cyassl_log_error(NXT_LOG_CRIT, thr->log, err, + "CyaSSL_CTX_set_cipher_list(\"%s\") failed", + conf->ciphers); + goto fail; + } + } + + /* TODO: ca_certificate */ + + CyaSSL_SetIORecv(ctx, nxt_cyassl_io_recv); + CyaSSL_SetIOSend(ctx, nxt_cyassl_io_send); + + return NXT_OK; + +fail: + + CyaSSL_CTX_free(ctx); + + return NXT_ERROR; +} + + +static void +nxt_cyassl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c) +{ + CYASSL *s; + CYASSL_CTX *ctx; + nxt_cyassl_conn_t *ssltls; + nxt_mem_pool_cleanup_t *mpcl; + + nxt_log_debug(c->socket.log, "cyassl conn init"); + + ssltls = nxt_mem_zalloc(c->mem_pool, sizeof(nxt_cyassl_conn_t)); + if (ssltls == NULL) { + goto fail; + } + + c->u.ssltls = ssltls; + nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size); + + mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0); + if (mpcl == NULL) { + goto fail; + } + + ctx = conf->ctx; + + s = CyaSSL_new(ctx); + if (s == NULL) { + nxt_cyassl_log_error(NXT_LOG_CRIT, c->socket.log, 0, + "CyaSSL_new() failed"); + goto fail; + } + + ssltls->session = s; + mpcl->handler = nxt_cyassl_session_cleanup; + mpcl->data = ssltls; + + CyaSSL_SetIOReadCtx(s, c); + CyaSSL_SetIOWriteCtx(s, c); + + c->io = &nxt_cyassl_event_conn_io; + c->sendfile = NXT_CONN_SENDFILE_OFF; + + nxt_cyassl_conn_handshake(thr, c, c->socket.data); + return; + +fail: + + nxt_event_conn_io_handle(thr, c->read_work_queue, + c->read_state->error_handler, c, c->socket.data); +} + + +static void +nxt_cyassl_session_cleanup(void *data) +{ + nxt_cyassl_conn_t *ssltls; + + ssltls = data; + + nxt_thread_log_debug("cyassl session cleanup"); + + nxt_free(ssltls->buffer.start); + + CyaSSL_free(ssltls->session); +} + + +static int +nxt_cyassl_io_recv(CYASSL *ssl, char *buf, int size, void *data) +{ + ssize_t n; + nxt_thread_t *thr; + nxt_event_conn_t *c; + + c = data; + thr = nxt_thread(); + + n = thr->engine->event->io->recv(c, (u_char *) buf, size, 0); + + if (n > 0) { + return n; + } + + if (n == 0) { + return CYASSL_CBIO_ERR_CONN_CLOSE; + } + + if (n == NXT_AGAIN) { + return CYASSL_CBIO_ERR_WANT_READ; + } + + return CYASSL_CBIO_ERR_GENERAL; +} + + +static int +nxt_cyassl_io_send(CYASSL *ssl, char *buf, int size, void *data) +{ + ssize_t n; + nxt_thread_t *thr; + nxt_event_conn_t *c; + + c = data; + thr = nxt_thread(); + + n = thr->engine->event->io->send(c, (u_char *) buf, size); + + if (n > 0) { + return n; + } + + if (n == NXT_AGAIN) { + return CYASSL_CBIO_ERR_WANT_WRITE; + } + + return CYASSL_CBIO_ERR_GENERAL; +} + + +static void +nxt_cyassl_conn_handshake(nxt_thread_t *thr, void *obj, void *data) +{ + int ret; + nxt_int_t n; + nxt_err_t err; + nxt_event_conn_t *c; + nxt_cyassl_conn_t *ssltls; + + c = obj; + ssltls = c->u.ssltls; + + nxt_log_debug(thr->log, "cyassl conn handshake: %d", ssltls->times); + + /* "ssltls->times == 1" is suitable to run CyaSSL_negotiate() in job. */ + + ret = CyaSSL_negotiate(ssltls->session); + + err = (ret != 0) ? nxt_socket_errno : 0; + + nxt_thread_time_debug_update(thr); + + nxt_log_debug(thr->log, "CyaSSL_negotiate(%d): %d", c->socket.fd, ret); + + if (ret == 0) { + nxt_cyassl_conn_io_read(thr, c, data); + return; + } + + n = nxt_cyassl_conn_test_error(thr, c, ret, nxt_cyassl_conn_handshake); + + if (n == NXT_ERROR) { + nxt_cyassl_conn_error(c, err, "CyaSSL_negotiate(%d) failed", + c->socket.fd); + + nxt_event_conn_io_handle(thr, c->read_work_queue, + c->read_state->error_handler, c, data); + + } else if (ssltls->ssl_error == SSL_ERROR_WANT_READ && ssltls->times < 2) { + ssltls->times++; + } +} + + +static void +nxt_cyassl_conn_io_read(nxt_thread_t *thr, void *obj, void *data) +{ + int ret; + nxt_buf_t *b; + nxt_err_t err; + nxt_int_t n; + nxt_event_conn_t *c; + nxt_cyassl_conn_t *ssltls; + nxt_work_handler_t handler; + + c = obj; + + nxt_log_debug(thr->log, "cyassl conn read"); + + handler = c->read_state->ready_handler; + b = c->read; + + /* b == NULL is used to test descriptor readiness. */ + + if (b != NULL) { + ssltls = c->u.ssltls; + + ret = CyaSSL_read(ssltls->session, b->mem.free, + b->mem.end - b->mem.free); + + err = (ret <= 0) ? nxt_socket_errno : 0; + + nxt_log_debug(thr->log, "CyaSSL_read(%d, %p, %uz): %d", + c->socket.fd, b->mem.free, b->mem.end - b->mem.free, ret); + + if (ret > 0) { + /* c->socket.read_ready is kept. */ + b->mem.free += ret; + handler = c->read_state->ready_handler; + + } else { + n = nxt_cyassl_conn_test_error(thr, c, ret, + nxt_cyassl_conn_io_read); + + if (nxt_fast_path(n != NXT_ERROR)) { + return; + } + + nxt_cyassl_conn_error(c, err, "CyaSSL_read(%d, %p, %uz) failed", + c->socket.fd, b->mem.free, + b->mem.end - b->mem.free); + + handler = c->read_state->error_handler; + } + } + + nxt_event_conn_io_handle(thr, c->read_work_queue, handler, c, data); +} + + +static ssize_t +nxt_cyassl_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, + nxt_buf_t *b, size_t limit) +{ + nxt_cyassl_conn_t *ssltls; + + nxt_log_debug(thr->log, "cyassl conn write chunk"); + + ssltls = c->u.ssltls; + + return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit); +} + + +static ssize_t +nxt_cyassl_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) +{ + int ret; + nxt_err_t err; + nxt_int_t n; + nxt_cyassl_conn_t *ssltls; + + nxt_log_debug(c->socket.log, "cyassl send"); + + ssltls = c->u.ssltls; + + ret = CyaSSL_write(ssltls->session, buf, size); + + if (ret <= 0) { + err = nxt_socket_errno; + c->socket.error = err; + + } else { + err = 0; + } + + nxt_log_debug(c->socket.log, "CyaSSL_write(%d, %p, %uz): %d", + c->socket.fd, buf, size, ret); + + if (ret > 0) { + return ret; + } + + n = nxt_cyassl_conn_test_error(nxt_thread(), c, ret, + nxt_event_conn_io_write); + + if (nxt_slow_path(n == NXT_ERROR)) { + nxt_cyassl_conn_error(c, err, "CyaSSL_write(%d, %p, %uz) failed", + c->socket.fd, buf, size); + } + + return n; +} + + +static void +nxt_cyassl_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) +{ + int ret; + nxt_event_conn_t *c; + nxt_cyassl_conn_t *ssltls; + + c = obj; + + nxt_log_debug(thr->log, "cyassl conn shutdown"); + + ssltls = c->u.ssltls; + + ret = CyaSSL_shutdown(ssltls->session); + + nxt_log_debug(thr->log, "CyaSSL_shutdown(%d): %d", c->socket.fd, ret); + + if (nxt_slow_path(ret != SSL_SUCCESS)) { + nxt_cyassl_conn_error(c, 0, "CyaSSL_shutdown(%d) failed", c->socket.fd); + } + + nxt_event_conn_io_handle(thr, c->write_work_queue, + c->write_state->close_handler, c, data); +} + + +static nxt_int_t +nxt_cyassl_conn_test_error(nxt_thread_t *thr, nxt_event_conn_t *c, int ret, + nxt_work_handler_t handler) +{ + nxt_work_queue_t *wq; + nxt_cyassl_conn_t *ssltls; + + ssltls = c->u.ssltls; + ssltls->ssl_error = CyaSSL_get_error(ssltls->session, ret); + + nxt_log_debug(thr->log, "CyaSSL_get_error(): %d", ssltls->ssl_error); + + switch (ssltls->ssl_error) { + + case SSL_ERROR_WANT_READ: + nxt_event_fd_block_write(thr->engine, &c->socket); + + c->socket.read_ready = 0; + c->socket.read_handler = handler; + + if (nxt_event_fd_is_disabled(c->socket.read)) { + nxt_event_fd_enable_read(thr->engine, &c->socket); + } + + return NXT_AGAIN; + + case SSL_ERROR_WANT_WRITE: + nxt_event_fd_block_read(thr->engine, &c->socket); + + c->socket.write_ready = 0; + c->socket.write_handler = handler; + + if (nxt_event_fd_is_disabled(c->socket.write)) { + nxt_event_fd_enable_write(thr->engine, &c->socket); + } + + return NXT_AGAIN; + + case SSL_ERROR_ZERO_RETURN: + /* A "close notify" alert */ + + if (c->read_state != NULL) { + wq = c->read_work_queue; + handler = c->read_state->close_handler; + + } else { + wq = c->write_work_queue; + handler = c->write_state->close_handler; + } + + nxt_event_conn_io_handle(thr, wq, handler, c, c->socket.data); + + return 0; + + default: + return NXT_ERROR; + } +} + + +static void nxt_cdecl +nxt_cyassl_conn_error(nxt_event_conn_t *c, nxt_err_t err, const char *fmt, ...) +{ + u_char *p, *end; + va_list args; + nxt_uint_t level; + nxt_cyassl_conn_t *ssltls; + u_char msg[NXT_MAX_ERROR_STR]; + + ssltls = c->u.ssltls; + + level = nxt_cyassl_log_error_level(c, err, ssltls->ssl_error); + + if (nxt_log_level_enough(c->socket.log, level)) { + + end = msg + sizeof(msg); + + va_start(args, fmt); + p = nxt_vsprintf(msg, end, fmt, args); + va_end(args); + + if (err != 0) { + p = nxt_sprintf(p, end, " %E", err); + } + + p = nxt_cyassl_copy_error(ssltls->ssl_error, p, end); + + nxt_log_error(level, c->socket.log, "%*s", p - msg, msg); + } +} + + +static nxt_uint_t +nxt_cyassl_log_error_level(nxt_event_conn_t *c, nxt_err_t err, int ssl_error) +{ + switch (ssl_error) { + + case SOCKET_ERROR_E: /* -208 */ + case MATCH_SUITE_ERROR: /* -261 */ + break; + + default: + return NXT_LOG_CRIT; + } + + return NXT_LOG_INFO; +} + + +static void nxt_cdecl +nxt_cyassl_log_error(nxt_uint_t level, nxt_log_t *log, int err, + const char *fmt, ...) +{ + u_char *p, *end; + va_list args; + u_char msg[NXT_MAX_ERROR_STR]; + + if (nxt_log_level_enough(log, level)) { + + end = msg + sizeof(msg); + + va_start(args, fmt); + p = nxt_vsprintf(msg, end, fmt, args); + va_end(args); + + p = nxt_cyassl_copy_error(err, p, end); + + nxt_log_error(level, log, "%*s", p - msg, msg); + } +} + + +static u_char * +nxt_cyassl_copy_error(int err, u_char *p, u_char *end) +{ + p = nxt_sprintf(p, end, " (SSL:%d ", err); + + CyaSSL_ERR_error_string_n(err, (char *) p, end - p); + + p += nxt_strlen(p); + + if (p < end) { + *p++ = ')'; + } + + return p; +} diff --git a/src/nxt_cycle.c b/src/nxt_cycle.c new file mode 100644 index 00000000..8539bc05 --- /dev/null +++ b/src/nxt_cycle.c @@ -0,0 +1,1743 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Valentin V. Bartenev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_cycle.h> +#include <nxt_process_chan.h> +#include <nxt_master_process.h> + + +static nxt_int_t nxt_cycle_inherited_listen_sockets(nxt_thread_t *thr, + nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_systemd_listen_sockets(nxt_thread_t *thr, + nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_event_engines(nxt_thread_t *thr, nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_processes(nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_thread_pools(nxt_thread_t *thr, nxt_cycle_t *cycle); +static void nxt_cycle_start(nxt_thread_t *thr, void *obj, void *data); +static void nxt_cycle_initial_start(nxt_thread_t *thr, nxt_cycle_t *cycle); +static void nxt_cycle_conf_test(nxt_thread_t *thr, nxt_cycle_t *cycle); +static void nxt_single_process_start(nxt_thread_t *thr, nxt_cycle_t *cycle); +static void nxt_cycle_close_idle_connections(nxt_thread_t *thr); +static void nxt_cycle_exit(nxt_thread_t *thr, void *obj, void *data); +static nxt_int_t nxt_cycle_event_engine_change(nxt_thread_t *thr, + nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_conf_init(nxt_thread_t *thr, nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_conf_read_cmd(nxt_thread_t *thr, nxt_cycle_t *cycle); +static nxt_sockaddr_t *nxt_cycle_sockaddr_parse(nxt_str_t *addr, + nxt_mem_pool_t *mp, nxt_log_t *log); +static nxt_sockaddr_t *nxt_cycle_sockaddr_unix_parse(nxt_str_t *addr, + nxt_mem_pool_t *mp, nxt_log_t *log); +static nxt_sockaddr_t *nxt_cycle_sockaddr_inet6_parse(nxt_str_t *addr, + nxt_mem_pool_t *mp, nxt_log_t *log); +static nxt_sockaddr_t *nxt_cycle_sockaddr_inet_parse(nxt_str_t *addr, + nxt_mem_pool_t *mp, nxt_log_t *log); +static nxt_int_t nxt_cycle_conf_apply(nxt_thread_t *thr, nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_listen_socket(nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_hostname(nxt_thread_t *thr, nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_log_files_init(nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_log_files_create(nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_listen_sockets_create(nxt_cycle_t *cycle); +static void nxt_cycle_listen_sockets_close(nxt_cycle_t *cycle); +static void nxt_cycle_pid_file_delete(nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_shm_zones_enable(nxt_cycle_t *cycle); +static nxt_int_t nxt_cycle_shm_zone_create(nxt_cycle_shm_zone_t *shm_zone); + +#if (NXT_THREADS) +static void nxt_cycle_thread_pool_destroy(nxt_thread_t *thr, + nxt_cycle_t *cycle, nxt_cycle_cont_t cont); +#endif + + +nxt_thread_declare_data(nxt_cycle_t *, nxt_thread_cycle_data); + + +nxt_int_t +nxt_cycle_create(nxt_thread_t *thr, nxt_cycle_t *previous, + nxt_cycle_cont_t start, nxt_str_t *config_name, nxt_bool_t test_config) +{ + nxt_int_t ret; + nxt_cycle_t *cycle; + nxt_array_t *listen_sockets; + nxt_mem_pool_t *mp; + static nxt_str_t upstream_zone = nxt_string("upstream_zone"); + + mp = nxt_mem_pool_create(1024); + + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + /* This alloction cannot fail. */ + cycle = nxt_mem_zalloc(mp, sizeof(nxt_cycle_t)); + + cycle->mem_pool = mp; + cycle->previous = previous; + cycle->config_name = config_name; + cycle->test_config = test_config; + + if (previous == NULL) { + cycle->prefix = nxt_current_directory(mp); + + } else { + cycle->type = previous->type; + cycle->prefix = nxt_str_dup(mp, NULL, previous->prefix); + } + + if (nxt_slow_path(cycle->prefix == NULL)) { + goto fail; + } + + cycle->conf_prefix = cycle->prefix; + + cycle->services = nxt_services_init(mp); + if (nxt_slow_path(cycle->services == NULL)) { + goto fail; + } + + listen_sockets = nxt_array_create(mp, 1, sizeof(nxt_listen_socket_t)); + if (nxt_slow_path(listen_sockets == NULL)) { + goto fail; + } + + cycle->listen_sockets = listen_sockets; + + if (previous == NULL) { + ret = nxt_cycle_inherited_listen_sockets(thr, cycle); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + } + + if (nxt_slow_path(nxt_cycle_hostname(thr, cycle) != NXT_OK)) { + goto fail; + } + + if (nxt_slow_path(nxt_cycle_log_files_init(cycle) != NXT_OK)) { + goto fail; + } + + if (nxt_slow_path(nxt_cycle_event_engines(thr, cycle) != NXT_OK)) { + goto fail; + } + + if (nxt_slow_path(nxt_cycle_processes(cycle) != NXT_OK)) { + goto fail; + } + + if (nxt_slow_path(nxt_cycle_thread_pools(thr, cycle) != NXT_OK)) { + goto fail; + } + + ret = nxt_cycle_shm_zone_add(cycle, &upstream_zone, 1024 * 1024, 8192); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + + /* Cycle shm zones array is created on demand. */ + + if (previous != NULL) { + previous->reconfiguring = 1; + cycle->start = start; + + } else { + nxt_thread_init_data(nxt_thread_cycle_data); + nxt_thread_cycle_set(cycle); + + cycle->start = test_config ? nxt_cycle_conf_test: + nxt_cycle_initial_start; + } + + nxt_log_debug(thr->log, "new cycle: %p", cycle); + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, nxt_cycle_start, + cycle, NULL, &nxt_main_log); + + return NXT_OK; + +fail: + + nxt_mem_pool_destroy(mp); + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_cycle_inherited_listen_sockets(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + u_char *v, *p; + nxt_int_t type; + nxt_array_t *inherited_sockets; + nxt_socket_t s; + nxt_listen_socket_t *ls; + + v = (u_char *) getenv("NGINX"); + + if (v == NULL) { + return nxt_cycle_systemd_listen_sockets(thr, cycle); + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "using inherited listen sockets: %s", v); + + inherited_sockets = nxt_array_create(cycle->mem_pool, + 1, sizeof(nxt_listen_socket_t)); + if (inherited_sockets == NULL) { + return NXT_ERROR; + } + + cycle->inherited_sockets = inherited_sockets; + + for (p = v; *p != '\0'; p++) { + + if (*p == ';') { + s = nxt_int_parse(v, p - v); + + if (nxt_slow_path(s < 0)) { + nxt_log_emerg(thr->log, "invalid socket number " + "\"%s\" in NGINX environment variable, " + "ignoring the rest of the variable", v); + return NXT_ERROR; + } + + v = p + 1; + + ls = nxt_array_zero_add(inherited_sockets); + if (nxt_slow_path(ls == NULL)) { + return NXT_ERROR; + } + + ls->socket = s; + + ls->sockaddr = nxt_getsockname(cycle->mem_pool, s); + if (nxt_slow_path(ls->sockaddr == NULL)) { + return NXT_ERROR; + } + + type = nxt_socket_getsockopt(s, SOL_SOCKET, SO_TYPE); + if (nxt_slow_path(type == -1)) { + return NXT_ERROR; + } + + ls->sockaddr->type = (uint16_t) type; + } + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_cycle_systemd_listen_sockets(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + u_char *nfd, *pid; + nxt_int_t n; + nxt_array_t *inherited_sockets; + nxt_socket_t s; + nxt_listen_socket_t *ls; + + /* + * Number of listening sockets passed. The socket + * descriptors start from number 3 and are sequential. + */ + nfd = (u_char *) getenv("LISTEN_FDS"); + if (nfd == NULL) { + return NXT_OK; + } + + /* The pid of the service process. */ + pid = (u_char *) getenv("LISTEN_PID"); + if (pid == NULL) { + return NXT_OK; + } + + n = nxt_int_parse(nfd, nxt_strlen(nfd)); + if (n < 0) { + return NXT_OK; + } + + if (nxt_pid != nxt_int_parse(pid, nxt_strlen(pid))) { + return NXT_OK; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "using %s systemd listen sockets", n); + + inherited_sockets = nxt_array_create(cycle->mem_pool, + n, sizeof(nxt_listen_socket_t)); + if (inherited_sockets == NULL) { + return NXT_ERROR; + } + + cycle->inherited_sockets = inherited_sockets; + + for (s = 3; s < n; s++) { + ls = nxt_array_zero_add(inherited_sockets); + if (nxt_slow_path(ls == NULL)) { + return NXT_ERROR; + } + + ls->socket = s; + + ls->sockaddr = nxt_getsockname(cycle->mem_pool, s); + if (nxt_slow_path(ls->sockaddr == NULL)) { + return NXT_ERROR; + } + + ls->sockaddr->type = SOCK_STREAM; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_cycle_event_engines(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + nxt_event_engine_t *engine, **e, **engines; + const nxt_event_set_ops_t *event_set; + + cycle->engines = nxt_array_create(cycle->mem_pool, 1, + sizeof(nxt_event_engine_t *)); + + if (nxt_slow_path(cycle->engines == NULL)) { + return NXT_ERROR; + } + + e = nxt_array_add(cycle->engines); + if (nxt_slow_path(e == NULL)) { + return NXT_ERROR; + } + + if (cycle->previous != NULL) { + /* Event engines are not allocated in memory pool. */ + engines = cycle->previous->engines->elts; + *e = engines[0]; + + } else { + event_set = nxt_service_get(cycle->services, "engine", NULL); + + if (nxt_slow_path(event_set == NULL)) { + /* TODO: log */ + return NXT_ERROR; + } + + engine = nxt_event_engine_create(thr, event_set, + nxt_master_process_signals, 0, 0); + + if (nxt_slow_path(engine == NULL)) { + return NXT_ERROR; + } + + engine->id = cycle->last_engine_id++; + *e = engine; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_cycle_processes(nxt_cycle_t *cycle) +{ + nxt_uint_t n; + nxt_process_chan_t *proc, *prev; + + /* + * Preallocate double number of previous cycle + * process slots or 2 process slots for initial cycle. + */ + n = (cycle->previous != NULL) ? cycle->previous->processes->nelts : 1; + + cycle->processes = nxt_array_create(cycle->mem_pool, 2 * n, + sizeof(nxt_process_chan_t)); + + if (nxt_slow_path(cycle->processes == NULL)) { + return NXT_ERROR; + } + + if (cycle->previous != NULL) { + cycle->process_generation = cycle->previous->process_generation; + + prev = cycle->previous->processes->elts; + + while (n != 0) { + proc = nxt_array_add(cycle->processes); + if (nxt_slow_path(proc == NULL)) { + return NXT_ERROR; + } + + *proc = *prev++; + n--; + } + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_cycle_thread_pools(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ +#if (NXT_THREADS) + nxt_int_t ret; + nxt_array_t *thread_pools; + + thread_pools = nxt_array_create(cycle->mem_pool, 1, + sizeof(nxt_thread_pool_t *)); + + if (nxt_slow_path(thread_pools == NULL)) { + return NXT_ERROR; + } + + cycle->thread_pools = thread_pools; + + ret = nxt_cycle_thread_pool_create(thr, cycle, 2, 60000 * 1000000LL); + + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + +#endif + + return NXT_OK; +} + + +static void +nxt_cycle_start(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_uint_t i; + nxt_cycle_t *cycle; + + cycle = obj; + + nxt_log_debug(thr->log, "cycle conf done"); + + nxt_mem_pool_debug_lock(cycle->mem_pool, nxt_thread_tid(thr)); + + thr->log->ctx_handler = NULL; + thr->log->ctx = NULL; + + if (nxt_cycle_conf_init(thr, cycle) != NXT_OK) { + goto fail; + } + + for (i = 0; i < nxt_init_modules_n; i++) { + if (nxt_init_modules[i](thr, cycle) != NXT_OK) { + goto fail; + } + } + + if (nxt_cycle_conf_apply(thr, cycle) != NXT_OK) { + goto fail; + } + + nxt_thread_cycle_set(cycle); + +#if (NXT_THREADS) + + /* + * Thread pools should be destroyed before starting worker + * processes, because thread pool semaphores will stick in + * locked state in new processes after fork(). + */ + nxt_cycle_thread_pool_destroy(thr, cycle, cycle->start); + +#else + + cycle->start(thr, cycle); + +#endif + + return; + +fail: + + nxt_cycle_quit(thr, cycle); +} + + +static void +nxt_cycle_initial_start(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + nxt_int_t ret; + const nxt_event_set_ops_t *event_set; + + if (cycle->inherited_sockets == NULL && cycle->daemon) { + + if (nxt_process_daemon() != NXT_OK) { + goto fail; + } + + /* + * An event engine should be updated after fork() + * even if an event facility was not changed because: + * 1) inherited kqueue descriptor is invalid, + * 2) the signal thread is not inherited. + */ + event_set = nxt_service_get(cycle->services, "engine", cycle->engine); + if (event_set == NULL) { + goto fail; + } + + ret = nxt_event_engine_change(thr, event_set, cycle->batch); + if (ret != NXT_OK) { + goto fail; + } + } + + ret = nxt_cycle_pid_file_create(cycle->pid_file, cycle->test_config); + if (ret != NXT_OK) { + goto fail; + } + + if (nxt_cycle_event_engine_change(thr, cycle) != NXT_OK) { + goto fail; + } + + thr->engine->max_connections = cycle->engine_connections; + + if (cycle->master_process) { + if (nxt_master_process_start(thr, cycle) != NXT_ERROR) { + return; + } + + } else { + nxt_single_process_start(thr, cycle); + return; + } + +fail: + + nxt_cycle_quit(thr, cycle); +} + + +static void +nxt_cycle_conf_test(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + (void) nxt_cycle_pid_file_create(cycle->pid_file, cycle->test_config); + + nxt_cycle_quit(thr, cycle); +} + + +static void +nxt_single_process_start(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ +#if (NXT_THREADS) + nxt_int_t ret; + + ret = nxt_cycle_thread_pool_create(thr, cycle, cycle->auxiliary_threads, + 60000 * 1000000LL); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_cycle_quit(thr, cycle); + return; + } + +#endif + + cycle->type = NXT_PROCESS_SINGLE; + + nxt_cycle_listen_sockets_enable(thr, cycle); + + return; +} + + +void +nxt_cycle_quit(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + nxt_bool_t done; + + nxt_log_debug(thr->log, "exiting"); + + if (cycle == NULL) { + cycle = nxt_thread_cycle(); + } + + done = 1; + + if (!thr->engine->shutdown) { + thr->engine->shutdown = 1; + +#if (NXT_THREADS) + + if (!nxt_array_is_empty(cycle->thread_pools)) { + nxt_cycle_thread_pool_destroy(thr, cycle, nxt_cycle_quit); + done = 0; + } + +#endif + + if (!cycle->test_config && cycle->type == NXT_PROCESS_MASTER) { + nxt_master_stop_worker_processes(cycle); + done = 0; + } + } + + nxt_cycle_close_idle_connections(thr); + + if (done) { + nxt_thread_work_queue_add(thr, &thr->work_queue.main, nxt_cycle_exit, + cycle, NULL, &nxt_main_log); + } +} + + +static void +nxt_cycle_close_idle_connections(nxt_thread_t *thr) +{ + nxt_queue_t *idle; + nxt_queue_link_t *link, *next; + nxt_event_conn_t *c; + + nxt_log_debug(thr->log, "close idle connections"); + + idle = &thr->engine->idle_connections; + + for (link = nxt_queue_head(idle); + link != nxt_queue_tail(idle); + link = next) + { + next = nxt_queue_next(link); + c = nxt_queue_link_data(link, nxt_event_conn_t, link); + + if (!c->socket.read_ready) { + nxt_queue_remove(link); + nxt_event_conn_close(thr, c); + } + } +} + + +static void +nxt_cycle_exit(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_cycle_t *cycle; + + cycle = obj; + +#if (NXT_THREADS) + + nxt_log_debug(thr->log, "thread pools: %d", cycle->thread_pools->nelts); + + if (!nxt_array_is_empty(cycle->thread_pools)) { + return; + } + +#endif + + if (cycle->type <= NXT_PROCESS_MASTER) { + nxt_cycle_pid_file_delete(cycle); + } + + if (!thr->engine->event->signal_support) { + nxt_event_engine_signals_stop(thr->engine); + } + + nxt_log_debug(thr->log, "exit"); + + exit(0); + nxt_unreachable(); +} + + +static nxt_int_t +nxt_cycle_event_engine_change(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + const nxt_event_set_ops_t *event_set; + + if (thr->engine->batch == cycle->batch + && nxt_strcmp(thr->engine->event->name, cycle->engine) == 0) + { + return NXT_OK; + } + + event_set = nxt_service_get(cycle->services, "engine", cycle->engine); + if (event_set != NULL) { + return nxt_event_engine_change(thr, event_set, cycle->batch); + } + + return NXT_ERROR; +} + + +void +nxt_cycle_event_engine_free(nxt_cycle_t *cycle) +{ + nxt_event_engine_t *engine, **engines; + + engines = cycle->engines->elts; + engine = engines[0]; + nxt_array_remove(cycle->engines, &engines[0]); + + nxt_event_engine_free(engine); +} + + +#if (NXT_THREADS) + +static void nxt_cycle_thread_pool_init(void); +static void nxt_cycle_thread_pool_exit(nxt_thread_t *thr, void *obj, + void *data); + + +nxt_int_t +nxt_cycle_thread_pool_create(nxt_thread_t *thr, nxt_cycle_t *cycle, + nxt_uint_t max_threads, nxt_nsec_t timeout) +{ + nxt_thread_pool_t *thread_pool, **tp; + + tp = nxt_array_add(cycle->thread_pools); + if (tp == NULL) { + return NXT_ERROR; + } + + thread_pool = nxt_thread_pool_create(max_threads, timeout, + nxt_cycle_thread_pool_init, + thr->engine, + nxt_cycle_thread_pool_exit); + + if (nxt_fast_path(thread_pool != NULL)) { + *tp = thread_pool; + } + + return NXT_OK; +} + + +static void +nxt_cycle_thread_pool_destroy(nxt_thread_t *thr, nxt_cycle_t *cycle, + nxt_cycle_cont_t cont) +{ + nxt_uint_t n; + nxt_thread_pool_t **tp; + + cycle->continuation = cont; + + n = cycle->thread_pools->nelts; + + if (n == 0) { + cont(thr, cycle); + return; + } + + tp = cycle->thread_pools->elts; + + do { + nxt_thread_pool_destroy(*tp); + + tp++; + n--; + } while (n != 0); +} + + +static void +nxt_cycle_thread_pool_init(void) +{ +#if (NXT_REGEX) + nxt_regex_init(0); +#endif +} + + +static void +nxt_cycle_thread_pool_exit(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_uint_t i, n; + nxt_cycle_t *cycle; + nxt_thread_pool_t *tp, **thread_pools; + nxt_thread_handle_t handle; + + tp = obj; + + if (data != NULL) { + handle = (nxt_thread_handle_t) (uintptr_t) data; + nxt_thread_wait(handle); + } + + cycle = nxt_thread_cycle(); + + thread_pools = cycle->thread_pools->elts; + n = cycle->thread_pools->nelts; + + nxt_log_debug(thr->log, "thread pools: %ui, cycle %p", n, cycle); + + for (i = 0; i < n; i++) { + + if (tp == thread_pools[i]) { + nxt_array_remove(cycle->thread_pools, &thread_pools[i]); + + if (n == 1) { + /* The last thread pool. */ + cycle->continuation(thr, cycle); + } + + return; + } + } +} + +#endif + + +static nxt_int_t +nxt_cycle_conf_init(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + nxt_int_t ret; + nxt_str_t *prefix; + nxt_file_t *file; + nxt_file_name_str_t file_name; + const nxt_event_set_ops_t *event_set; + + cycle->daemon = 1; + cycle->master_process = 1; + cycle->engine_connections = 256; + cycle->worker_processes = 1; + cycle->auxiliary_threads = 2; + cycle->user_cred.user = "nobody"; + cycle->group = NULL; + cycle->pid = "nginman.pid"; + cycle->error_log = "error.log"; + + if (nxt_cycle_conf_read_cmd(thr, cycle) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_user_cred_get(&cycle->user_cred, cycle->group) != NXT_OK) { + return NXT_ERROR; + } + + /* An engine's parameters. */ + + event_set = nxt_service_get(cycle->services, "engine", cycle->engine); + if (event_set == NULL) { + return NXT_ERROR; + } + + cycle->engine = event_set->name; + + prefix = nxt_file_name_is_absolute(cycle->pid) ? NULL : cycle->prefix; + + ret = nxt_file_name_create(cycle->mem_pool, &file_name, "%V%s%Z", + prefix, cycle->pid); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + cycle->pid_file = file_name.start; + + prefix = nxt_file_name_is_absolute(cycle->error_log) ? NULL : cycle->prefix; + + ret = nxt_file_name_create(cycle->mem_pool, &file_name, "%V%s%Z", + prefix, cycle->error_log); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + file = nxt_list_first(cycle->log_files); + file->name = file_name.start; + + return NXT_OK; +} + + +static nxt_int_t +nxt_cycle_conf_read_cmd(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + char *p, **argv; + nxt_int_t n; + nxt_str_t addr; + nxt_sockaddr_t *sa; + + argv = nxt_process_argv; + + while (*argv != NULL) { + p = *argv++; + + if (nxt_strcmp(p, "--listen") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, "no argument for option \"--listen\""); + return NXT_ERROR; + } + + p = *argv++; + + addr.len = nxt_strlen(p); + addr.data = (u_char *) p; + + sa = nxt_cycle_sockaddr_parse(&addr, cycle->mem_pool, thr->log); + + if (sa == NULL) { + return NXT_ERROR; + } + + cycle->listen = sa; + + continue; + } + + if (nxt_strcmp(p, "--workers") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, "no argument for option \"--workers\""); + return NXT_ERROR; + } + + p = *argv++; + n = nxt_int_parse((u_char *) p, nxt_strlen(p)); + + if (n < 1) { + nxt_log_emerg(thr->log, "invalid number of workers: \"%s\"", p); + return NXT_ERROR; + } + + cycle->worker_processes = n; + + continue; + } + + if (nxt_strcmp(p, "--user") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, "no argument for option \"--user\""); + return NXT_ERROR; + } + + p = *argv++; + + cycle->user_cred.user = p; + + continue; + } + + if (nxt_strcmp(p, "--group") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, "no argument for option \"--group\""); + return NXT_ERROR; + } + + p = *argv++; + + cycle->group = p; + + continue; + } + + if (nxt_strcmp(p, "--pid") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, "no argument for option \"--pid\""); + return NXT_ERROR; + } + + p = *argv++; + + cycle->pid = p; + + continue; + } + + if (nxt_strcmp(p, "--log") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, "no argument for option \"--log\""); + return NXT_ERROR; + } + + p = *argv++; + + cycle->error_log = p; + + continue; + } + + if (nxt_strcmp(p, "--no-daemonize") == 0) { + cycle->daemon = 0; + continue; + } + } + + return NXT_OK; +} + + +static nxt_sockaddr_t * +nxt_cycle_sockaddr_parse(nxt_str_t *addr, nxt_mem_pool_t *mp, nxt_log_t *log) +{ + u_char *p; + size_t len; + + len = addr->len; + p = addr->data; + + if (len >= 5 && nxt_memcmp(p, (u_char *) "unix:", 5) == 0) { + return nxt_cycle_sockaddr_unix_parse(addr, mp, log); + } + + if (len != 0 && *p == '[') { + return nxt_cycle_sockaddr_inet6_parse(addr, mp, log); + } + + return nxt_cycle_sockaddr_inet_parse(addr, mp, log); +} + + +static nxt_sockaddr_t * +nxt_cycle_sockaddr_unix_parse(nxt_str_t *addr, nxt_mem_pool_t *mp, + nxt_log_t *log) +{ +#if (NXT_HAVE_UNIX_DOMAIN) + u_char *p; + size_t len, socklen; + nxt_sockaddr_t *sa; + + /* + * Actual sockaddr_un length can be lesser or even larger than defined + * struct sockaddr_un length (see comment in unix/nxt_socket.h). So + * limit maximum Unix domain socket address length by defined sun_path[] + * length because some OSes accept addresses twice larger than defined + * struct sockaddr_un. Also reserve space for a trailing zero to avoid + * ambiguity, since many OSes accept Unix domain socket addresses + * without a trailing zero. + */ + const size_t max_len = sizeof(struct sockaddr_un) + - offsetof(struct sockaddr_un, sun_path) - 1; + + /* cutting "unix:" */ + len = addr->len - 5; + p = addr->data + 5; + + if (len == 0) { + nxt_log_emerg(log, "unix domain socket \"%V\" name is invalid", addr); + return NULL; + } + + if (len > max_len) { + nxt_log_emerg(log, "unix domain socket \"%V\" name is too long", addr); + return NULL; + } + + socklen = offsetof(struct sockaddr_un, sun_path) + len + 1; + +#if (NXT_LINUX) + + /* + * Linux unix(7): + * + * abstract: an abstract socket address is distinguished by the fact + * that sun_path[0] is a null byte ('\0'). The socket's address in + * this namespace is given by the additional bytes in sun_path that + * are covered by the specified length of the address structure. + * (Null bytes in the name have no special significance.) + */ + if (p[0] == '@') { + p[0] = '\0'; + socklen--; + } + +#endif + + sa = nxt_sockaddr_alloc(mp, socklen); + + if (nxt_slow_path(sa == NULL)) { + return NULL; + } + + sa->type = SOCK_STREAM; + + sa->u.sockaddr_un.sun_family = AF_UNIX; + nxt_memcpy(sa->u.sockaddr_un.sun_path, p, len); + + return sa; + +#else /* !(NXT_HAVE_UNIX_DOMAIN) */ + + nxt_log_emerg(log, "unix domain socket \"%V\" is not supported", addr); + + return NULL; + +#endif +} + + +static nxt_sockaddr_t * +nxt_cycle_sockaddr_inet6_parse(nxt_str_t *addr, nxt_mem_pool_t *mp, + nxt_log_t *log) +{ +#if (NXT_INET6) + u_char *p, *addr, *addr_end; + size_t len; + nxt_int_t port; + nxt_mem_pool_t *mp; + nxt_sockaddr_t *sa; + struct in6_addr *in6_addr; + + len = addr->len - 1; + p = addr->data + 1; + + addr_end = nxt_memchr(p, ']', len); + + if (addr_end == NULL) { + goto invalid_address; + } + + sa = nxt_sockaddr_alloc(mp, sizeof(struct sockaddr_in6)); + + if (nxt_slow_path(sa == NULL)) { + return NULL; + } + + in6_addr = &sa->u.sockaddr_in6.sin6_addr; + + if (nxt_inet6_addr(in6_addr, p, addr_end - p) != NXT_OK) { + goto invalid_address; + } + + p = addr_end + 1; + len = (p + len) - p; + + if (len == 0) { + goto found; + } + + if (*p == ':') { + port = nxt_int_parse(p + 1, len - 1); + + if (port >= 1 && port <= 65535) { + goto found; + } + } + + nxt_log_emerg(log, "invalid port in \"%V\"", addr); + + return NULL; + +found: + + sa->type = SOCK_STREAM; + + sa->u.sockaddr_in6.sin6_family = AF_INET6; + sa->u.sockaddr_in6.sin6_port = htons((in_port_t) port); + + return sa; + +invalid_address: + + nxt_log_emerg(log, "invalid IPv6 address in \"%V\"", addr); + + return NULL; + +#else + + nxt_log_emerg(log, "IPv6 socket \"%V\" is not supported", addr); + + return NULL; + +#endif +} + + +static nxt_sockaddr_t * +nxt_cycle_sockaddr_inet_parse(nxt_str_t *addr, nxt_mem_pool_t *mp, + nxt_log_t *log) +{ + u_char *p, *ip; + size_t len; + in_addr_t s_addr; + nxt_int_t port; + nxt_sockaddr_t *sa; + + s_addr = INADDR_ANY; + + len = addr->len; + ip = addr->data; + + p = nxt_memchr(ip, ':', len); + + if (p == NULL) { + + /* single value port, or address */ + + port = nxt_int_parse(ip, len); + + if (port > 0) { + /* "*:XX" */ + + if (port < 1 || port > 65535) { + goto invalid_port; + } + + } else { + /* "x.x.x.x" */ + + s_addr = nxt_inet_addr(ip, len); + + if (s_addr == INADDR_NONE) { + goto invalid_port; + } + + port = 8080; + } + + } else { + + /* x.x.x.x:XX */ + + p++; + len = (ip + len) - p; + port = nxt_int_parse(p, len); + + if (port < 1 || port > 65535) { + goto invalid_port; + } + + len = (p - 1) - ip; + + if (len != 1 || ip[0] != '*') { + s_addr = nxt_inet_addr(ip, len); + + if (s_addr == INADDR_NONE) { + goto invalid_addr; + } + + /* "x.x.x.x:XX" */ + } + } + + sa = nxt_sockaddr_alloc(mp, sizeof(struct sockaddr_in)); + + if (nxt_slow_path(sa == NULL)) { + return NULL; + } + + sa->type = SOCK_STREAM; + + sa->u.sockaddr_in.sin_family = AF_INET; + sa->u.sockaddr_in.sin_port = htons((in_port_t) port); + sa->u.sockaddr_in.sin_addr.s_addr = s_addr; + + return sa; + +invalid_port: + + nxt_log_emerg(log, "invalid port in \"%V\"", addr); + + return NULL; + +invalid_addr: + + nxt_log_emerg(log, "invalid address in \"%V\"", addr); + + return NULL; +} + + +static nxt_int_t +nxt_cycle_conf_apply(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + if (nxt_cycle_log_files_create(cycle) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_cycle_listen_socket(cycle) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_cycle_event_engine_change(thr, cycle) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_cycle_listen_sockets_create(cycle) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_cycle_shm_zones_enable(cycle) != NXT_OK) { + return NXT_ERROR; + } + + nxt_cycle_listen_sockets_close(cycle); + + return NXT_OK; +} + + +static nxt_int_t +nxt_cycle_listen_socket(nxt_cycle_t *cycle) +{ + nxt_sockaddr_t *sa; +// nxt_work_queue_t *wq; + nxt_listen_socket_t *ls; + + if (cycle->listen == NULL) { + sa = nxt_sockaddr_alloc(cycle->mem_pool, sizeof(struct sockaddr_in)); + if (sa == NULL) { + return NXT_ERROR; + } + + sa->type = SOCK_STREAM; + sa->u.sockaddr_in.sin_family = AF_INET; + sa->u.sockaddr_in.sin_port = htons(8080); + + cycle->listen = sa; + } + + if (nxt_sockaddr_text(cycle->mem_pool, cycle->listen, 1) != NXT_OK) { + return NXT_ERROR; + } + + ls = nxt_cycle_listen_socket_add(cycle, cycle->listen); + if (ls == NULL) { + return NXT_ERROR; + } + + ls->read_after_accept = 1; + +#if 0 + ls->flags = NXT_NONBLOCK; + + /* STUB */ + wq = nxt_mem_zalloc(cf->mem_pool, sizeof(nxt_work_queue_t)); + if (wq == NULL) { + return NXT_ERROR; + } + nxt_work_queue_name(wq, "listen"); + /**/ + + ls->work_queue = wq; + ls->handler = nxt_stream_connection_init; + + /* + * Connection memory pool chunk size is tunned to + * allocate the most data in one mem_pool chunk. + */ + ls->mem_pool_size = nxt_listen_socket_pool_min_size(ls) + + sizeof(nxt_event_conn_proxy_t) + + sizeof(nxt_event_conn_t) + + 4 * sizeof(nxt_buf_t); +#endif + + return NXT_OK; +} + + +nxt_listen_socket_t * +nxt_cycle_listen_socket_add(nxt_cycle_t *cycle, nxt_sockaddr_t *sa) +{ + nxt_mem_pool_t *mp; + nxt_listen_socket_t *ls; + + ls = nxt_array_zero_add(cycle->listen_sockets); + if (ls == NULL) { + return NULL; + } + + mp = cycle->mem_pool; + + ls->sockaddr = nxt_sockaddr_create(mp, &sa->u.sockaddr, nxt_socklen(sa)); + if (ls->sockaddr == NULL) { + return NULL; + } + + ls->sockaddr->type = sa->type; + + if (nxt_sockaddr_text(mp, ls->sockaddr, 1) != NXT_OK) { + return NULL; + } + + ls->socket = -1; + ls->backlog = NXT_LISTEN_BACKLOG; + + return ls; +} + + +static nxt_int_t +nxt_cycle_hostname(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + size_t len; + char hostname[NXT_MAXHOSTNAMELEN + 1]; + + if (gethostname(hostname, NXT_MAXHOSTNAMELEN) != 0) { + nxt_log_emerg(thr->log, "gethostname() failed %E", nxt_errno); + return NXT_ERROR; + } + + /* + * Linux gethostname(2): + * + * If the null-terminated hostname is too large to fit, + * then the name is truncated, and no error is returned. + * + * For this reason an additional byte is reserved in the buffer. + */ + hostname[NXT_MAXHOSTNAMELEN] = '\0'; + + len = nxt_strlen(hostname); + cycle->hostname.len = len; + + cycle->hostname.data = nxt_mem_nalloc(cycle->mem_pool, len); + + if (cycle->hostname.data != NULL) { + nxt_memcpy_lowcase(cycle->hostname.data, (u_char *) hostname, len); + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_cycle_log_files_init(nxt_cycle_t *cycle) +{ + nxt_uint_t n; + nxt_file_t *file; + nxt_list_t *log_files; + + n = (cycle->previous != NULL) ? nxt_list_nelts(cycle->previous->log_files): + 1; + + log_files = nxt_list_create(cycle->mem_pool, n, sizeof(nxt_file_t)); + + if (nxt_fast_path(log_files != NULL)) { + cycle->log_files = log_files; + + /* Preallocate the main error_log. This allocation cannot fail. */ + file = nxt_list_zero_add(log_files); + + file->fd = NXT_FILE_INVALID; + file->log_level = NXT_LOG_CRIT; + + return NXT_OK; + } + + return NXT_ERROR; +} + + +nxt_file_t * +nxt_cycle_log_file_add(nxt_cycle_t *cycle, nxt_str_t *name) +{ + nxt_int_t ret; + nxt_str_t *prefix; + nxt_file_t *file; + nxt_file_name_str_t file_name; + + prefix = nxt_file_name_is_absolute(name->data) ? NULL : cycle->prefix; + + ret = nxt_file_name_create(cycle->mem_pool, &file_name, "%V%V%Z", + prefix, name); + + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + + nxt_list_each(file, cycle->log_files) { + + /* STUB: hardecoded case sensitive/insensitive. */ + + if (file->name != NULL + && nxt_file_name_eq(file->name, file_name.start)) + { + return file; + } + + } nxt_list_loop; + + file = nxt_list_zero_add(cycle->log_files); + + if (nxt_slow_path(file == NULL)) { + return NULL; + } + + file->fd = NXT_FILE_INVALID; + file->log_level = NXT_LOG_CRIT; + file->name = file_name.start; + + return file; +} + + +static nxt_int_t +nxt_cycle_log_files_create(nxt_cycle_t *cycle) +{ + nxt_int_t ret; + nxt_file_t *file; + + nxt_list_each(file, cycle->log_files) { + + ret = nxt_file_open(file, NXT_FILE_APPEND, NXT_FILE_CREATE_OR_OPEN, + NXT_FILE_OWNER_ACCESS); + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + } nxt_list_loop; + + file = nxt_list_first(cycle->log_files); + + return nxt_file_stderr(file); +} + + +static nxt_int_t +nxt_cycle_listen_sockets_create(nxt_cycle_t *cycle) +{ + nxt_uint_t c, p, ncurr, nprev; + nxt_listen_socket_t *curr, *prev; + + curr = cycle->listen_sockets->elts; + ncurr = cycle->listen_sockets->nelts; + + if (cycle->previous != NULL) { + prev = cycle->previous->listen_sockets->elts; + nprev = cycle->previous->listen_sockets->nelts; + + } else if (cycle->inherited_sockets != NULL) { + prev = cycle->inherited_sockets->elts; + nprev = cycle->inherited_sockets->nelts; + + } else { + prev = NULL; + nprev = 0; + } + + for (c = 0; c < ncurr; c++) { + + for (p = 0; p < nprev; p++) { + + if (nxt_sockaddr_cmp(curr[c].sockaddr, prev[p].sockaddr)) { + + if (nxt_listen_socket_update(&curr[c], &prev[p]) != NXT_OK) { + return NXT_ERROR; + } + + goto next; + } + } + + if (nxt_listen_socket_create(&curr[c], cycle->test_config) != NXT_OK) { + return NXT_ERROR; + } + + next: + + continue; + } + + return NXT_OK; +} + + +static void +nxt_cycle_listen_sockets_close(nxt_cycle_t *cycle) +{ + nxt_uint_t p, c, nprev, ncurr; + nxt_listen_socket_t *curr, *prev; + + if (cycle->previous == NULL) { + return; + } + + prev = cycle->previous->listen_sockets->elts; + nprev = cycle->previous->listen_sockets->nelts; + + curr = cycle->listen_sockets->elts; + ncurr = cycle->listen_sockets->nelts; + + for (p = 0; p < nprev; p++) { + + for (c = 0; c < ncurr; c++) { + if (nxt_sockaddr_cmp(prev[p].sockaddr, curr[c].sockaddr)) { + goto next; + } + } + + nxt_socket_close(prev[p].socket); + + next: + + continue; + } + + return; +} + + +nxt_int_t +nxt_cycle_listen_sockets_enable(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + nxt_uint_t i, n; + nxt_listen_socket_t *ls; + + ls = cycle->listen_sockets->elts; + n = cycle->listen_sockets->nelts; + + for (i = 0; i < n; i++) { + if (nxt_event_conn_listen(thr, &ls[i]) != NXT_OK) { + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +nxt_str_t * +nxt_current_directory(nxt_mem_pool_t *mp) +{ + size_t len; + u_char *p; + nxt_str_t *name; + char buf[NXT_MAX_PATH_LEN]; + + len = nxt_dir_current(buf, NXT_MAX_PATH_LEN); + + if (nxt_fast_path(len != 0)) { + name = nxt_str_alloc(mp, len + 1); + + if (nxt_fast_path(name != NULL)) { + p = nxt_cpymem(name->data, buf, len); + *p = '/'; + + return name; + } + } + + return NULL; +} + + +nxt_int_t +nxt_cycle_pid_file_create(nxt_file_name_t *pid_file, nxt_bool_t test) +{ + ssize_t len; + nxt_int_t n; + nxt_uint_t create; + nxt_file_t file; + u_char pid[NXT_INT64_T_LEN + NXT_LINEFEED_SIZE]; + + nxt_memzero(&file, sizeof(nxt_file_t)); + + file.name = pid_file; + + create = test ? NXT_FILE_CREATE_OR_OPEN : NXT_FILE_TRUNCATE; + + n = nxt_file_open(&file, NXT_FILE_WRONLY, create, NXT_FILE_DEFAULT_ACCESS); + + if (n != NXT_OK) { + return NXT_ERROR; + } + + if (!test) { + len = nxt_sprintf(pid, pid + sizeof(pid), "%PI%n", nxt_pid) - pid; + + if (nxt_file_write(&file, pid, len, 0) != len) { + return NXT_ERROR; + } + } + + nxt_file_close(&file); + + return NXT_OK; +} + + +static void +nxt_cycle_pid_file_delete(nxt_cycle_t *cycle) +{ + nxt_file_name_t *pid_file; + + if (!cycle->test_config) { + pid_file = (cycle->new_binary != 0) ? cycle->oldbin_file: + cycle->pid_file; + if (pid_file != NULL) { + nxt_file_delete(pid_file); + } + } +} + + +nxt_int_t +nxt_cycle_shm_zone_add(nxt_cycle_t *cycle, nxt_str_t *name, size_t size, + nxt_uint_t page_size) +{ + nxt_cycle_shm_zone_t *shm_zone; + + if (cycle->shm_zones == NULL) { + cycle->shm_zones = nxt_array_create(cycle->mem_pool, 1, + sizeof(nxt_cycle_shm_zone_t)); + if (cycle->shm_zones == NULL) { + return NXT_ERROR; + } + } + + shm_zone = nxt_array_add(cycle->shm_zones); + + if (shm_zone != NULL) { + shm_zone->size = size; + shm_zone->page_size = page_size; + shm_zone->name = *name; + + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_cycle_shm_zones_enable(nxt_cycle_t *cycle) +{ + nxt_uint_t i, n; + nxt_cycle_shm_zone_t *shm_zone; + + if (cycle->shm_zones != NULL) { + shm_zone = cycle->shm_zones->elts; + n = cycle->shm_zones->nelts; + + for (i = 0; i < n; i++) { + if (nxt_cycle_shm_zone_create(&shm_zone[i]) != NXT_OK) { + return NXT_ERROR; + } + } + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_cycle_shm_zone_create(nxt_cycle_shm_zone_t *shm_zone) +{ + nxt_mem_zone_t *zone; + + /* + * Unix-only code because Windows ASLR maps shared memory segments at + * different addresses in different processes. Unix ASLR does not affect + * this because all shared memory segments are inherited during fork(). + */ + + shm_zone->addr = nxt_mem_mmap(NULL, shm_zone->size, + NXT_MEM_MAP_READ | NXT_MEM_MAP_WRITE, + NXT_MEM_MAP_SHARED, NXT_FILE_INVALID, 0); + + if (shm_zone->addr != NXT_MEM_MAP_FAILED) { + + zone = nxt_mem_zone_init(shm_zone->addr, shm_zone->size, + shm_zone->page_size); + if (zone != NULL) { + return NXT_OK; + } + + nxt_mem_munmap(shm_zone->addr, shm_zone->size); + } + + return NXT_ERROR; +} diff --git a/src/nxt_cycle.h b/src/nxt_cycle.h new file mode 100644 index 00000000..18c944b4 --- /dev/null +++ b/src/nxt_cycle.h @@ -0,0 +1,159 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Valentin V. Bartenev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_CYCLE_H_INCLUDED_ +#define _NXT_CYCLE_H_INCLUDED_ + + +typedef enum { + NXT_PROCESS_SINGLE = 0, + NXT_PROCESS_MASTER, + NXT_PROCESS_WORKER, +} nxt_process_type_e; + + +typedef struct nxt_cycle_s nxt_cycle_t; +typedef void (*nxt_cycle_cont_t)(nxt_thread_t *thr, nxt_cycle_t *cycle); + + +struct nxt_cycle_s { + nxt_mem_pool_t *mem_pool; + + nxt_cycle_t *previous; + + nxt_array_t *inherited_sockets; /* of nxt_listen_socket_t */ + nxt_array_t *listen_sockets; /* of nxt_listen_socket_t */ + + nxt_array_t *services; /* of nxt_service_t */ + nxt_array_t *engines; /* of nxt_event_engine_t */ + + nxt_cycle_cont_t start; + + nxt_str_t *config_name; + + nxt_str_t *conf_prefix; + nxt_str_t *prefix; + + nxt_str_t hostname; + + nxt_file_name_t *pid_file; + nxt_file_name_t *oldbin_file; + nxt_pid_t new_binary; + +#if (NXT_THREADS) + nxt_array_t *thread_pools; /* of nxt_thread_pool_t */ + nxt_cycle_cont_t continuation; +#endif + + nxt_array_t *processes; /* of nxt_process_chan_t */ + + nxt_list_t *log_files; /* of nxt_file_t */ + + nxt_array_t *shm_zones; /* of nxt_cycle_shm_zone_t */ + + uint32_t process_generation; + uint32_t current_process; + uint32_t last_engine_id; + + nxt_process_type_e type; + + uint8_t test_config; /* 1 bit */ + uint8_t reconfiguring; /* 1 bit */ + + void **core_ctx; + + nxt_event_timer_t timer; + + uint8_t daemon; + uint8_t batch; + uint8_t master_process; + const char *engine; + uint32_t engine_connections; + uint32_t worker_processes; + uint32_t auxiliary_threads; + nxt_user_cred_t user_cred; + const char *group; + const char *pid; + const char *error_log; + nxt_sockaddr_t *listen; +}; + + +typedef struct { + void *addr; + size_t size; + nxt_uint_t page_size; + nxt_str_t name; +} nxt_cycle_shm_zone_t; + + + +typedef nxt_int_t (*nxt_module_init_t)(nxt_thread_t *thr, nxt_cycle_t *cycle); + + +nxt_thread_extern_data(nxt_cycle_t *, nxt_thread_cycle_data); + + +nxt_inline void +nxt_thread_cycle_set(nxt_cycle_t *cycle) +{ + nxt_cycle_t **p; + + p = nxt_thread_get_data(nxt_thread_cycle_data); + + *p = cycle; +} + + +nxt_inline nxt_cycle_t * +nxt_thread_cycle(void) +{ + nxt_cycle_t **p; + + p = nxt_thread_get_data(nxt_thread_cycle_data); + + return *p; +} + + +nxt_int_t nxt_cycle_create(nxt_thread_t *thr, nxt_cycle_t *previous, + nxt_cycle_cont_t start, nxt_str_t *config_name, nxt_bool_t test_config); +void nxt_cycle_quit(nxt_thread_t *thr, nxt_cycle_t *cycle); + +void nxt_cycle_event_engine_free(nxt_cycle_t *cycle); + +#if (NXT_THREADS) +nxt_int_t nxt_cycle_thread_pool_create(nxt_thread_t *thr, nxt_cycle_t *cycle, + nxt_uint_t max_threads, nxt_nsec_t timeout); +#endif + +/* STUB */ +nxt_str_t *nxt_current_directory(nxt_mem_pool_t *mp); + +nxt_int_t nxt_cycle_pid_file_create(nxt_file_name_t *pid_file, nxt_bool_t test); + +nxt_listen_socket_t *nxt_cycle_listen_socket_add(nxt_cycle_t *cycle, + nxt_sockaddr_t *sa); +nxt_int_t nxt_cycle_listen_sockets_enable(nxt_thread_t *thr, + nxt_cycle_t *cycle); +nxt_file_t *nxt_cycle_log_file_add(nxt_cycle_t *cycle, nxt_str_t *name); + +nxt_int_t nxt_cycle_shm_zone_add(nxt_cycle_t *cycle, nxt_str_t *name, + size_t size, nxt_uint_t page_size); + +/* STUB */ +void nxt_cdecl nxt_log_time_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...); + +nxt_int_t nxt_app_start(nxt_cycle_t *cycle); + + +extern nxt_module_init_t nxt_init_modules[]; +extern nxt_uint_t nxt_init_modules_n; + + +#endif /* _NXT_CYCLE_H_INCLIDED_ */ diff --git a/src/nxt_devpoll.c b/src/nxt_devpoll.c new file mode 100644 index 00000000..8fbe8184 --- /dev/null +++ b/src/nxt_devpoll.c @@ -0,0 +1,699 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * "/dev/poll" has been introduced in Solaris 7 (11/99), HP-UX 11.22 (named + * "eventport pseudo driver" internally, not to be confused with Solaris 10 + * event ports), IRIX 6.5.15, and Tru64 UNIX 5.1A. + * + * Although "/dev/poll" descriptor is a file descriptor, nevertheless + * it cannot be added to another poll set, Solaris poll(7d): + * + * The /dev/poll driver does not yet support polling. Polling on a + * /dev/poll file descriptor will result in POLLERR being returned + * in the revents field of pollfd structure. + */ + + +#define NXT_DEVPOLL_ADD 0 +#define NXT_DEVPOLL_UPDATE 1 +#define NXT_DEVPOLL_CHANGE 2 +#define NXT_DEVPOLL_DELETE 3 + + +static nxt_event_set_t *nxt_devpoll_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +static void nxt_devpoll_free(nxt_event_set_t *event_set); +static void nxt_devpoll_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_devpoll_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +#if (NXT_HPUX) +static void nxt_devpoll_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_devpoll_drop_changes(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +#endif +static void nxt_devpoll_enable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_enable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_disable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_disable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_block_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_block_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_oneshot_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_oneshot_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_devpoll_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_uint_t op, nxt_uint_t events); +static nxt_int_t nxt_devpoll_commit_changes(nxt_thread_t *thr, + nxt_devpoll_event_set_t *ds); +static void nxt_devpoll_change_error(nxt_thread_t *thr, + nxt_devpoll_event_set_t *ds, nxt_event_fd_t *ev); +static void nxt_devpoll_remove(nxt_thread_t *thr, nxt_devpoll_event_set_t *ds, + nxt_fd_t fd); +static nxt_int_t nxt_devpoll_write(nxt_thread_t *thr, int devpoll, + struct pollfd *pfd, size_t n); +static void nxt_devpoll_set_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout); + + +const nxt_event_set_ops_t nxt_devpoll_event_set = { + "devpoll", + nxt_devpoll_create, + nxt_devpoll_free, + nxt_devpoll_enable, + nxt_devpoll_disable, + nxt_devpoll_disable, +#if (NXT_HPUX) + nxt_devpoll_close, +#else + nxt_devpoll_disable, +#endif + nxt_devpoll_enable_read, + nxt_devpoll_enable_write, + nxt_devpoll_disable_read, + nxt_devpoll_disable_write, + nxt_devpoll_block_read, + nxt_devpoll_block_write, + nxt_devpoll_oneshot_read, + nxt_devpoll_oneshot_write, + nxt_devpoll_enable_read, + NULL, + NULL, + NULL, + NULL, + nxt_devpoll_set_poll, + + &nxt_unix_event_conn_io, + + NXT_NO_FILE_EVENTS, + NXT_NO_SIGNAL_EVENTS, +}; + + +static nxt_event_set_t * +nxt_devpoll_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + nxt_event_set_t *event_set; + nxt_devpoll_event_set_t *ds; + + event_set = nxt_zalloc(sizeof(nxt_devpoll_event_set_t)); + if (event_set == NULL) { + return NULL; + } + + ds = &event_set->devpoll; + + ds->devpoll = -1; + ds->mchanges = mchanges; + ds->mevents = mevents; + + ds->devpoll_changes = nxt_malloc(sizeof(nxt_devpoll_change_t) * mchanges); + if (ds->devpoll_changes == NULL) { + goto fail; + } + + /* + * NXT_DEVPOLL_CHANGE requires two struct pollfd's: + * for POLLREMOVE and subsequent POLLIN or POLLOUT. + */ + ds->changes = nxt_malloc(2 * sizeof(struct pollfd) * mchanges); + if (ds->changes == NULL) { + goto fail; + } + + ds->events = nxt_malloc(sizeof(struct pollfd) * mevents); + if (ds->events == NULL) { + goto fail; + } + + ds->devpoll = open("/dev/poll", O_RDWR); + if (ds->devpoll == -1) { + nxt_main_log_emerg("open(/dev/poll) failed %E", nxt_errno); + goto fail; + } + + nxt_main_log_debug("open(/dev/poll): %d", ds->devpoll); + + return event_set; + +fail: + + nxt_devpoll_free(event_set); + + return NULL; +} + + +static void +nxt_devpoll_free(nxt_event_set_t *event_set) +{ + nxt_devpoll_event_set_t *ds; + + ds = &event_set->devpoll; + + nxt_main_log_debug("devpoll %d free", ds->devpoll); + + if (ds->devpoll != -1) { + if (close(ds->devpoll) != 0) { + nxt_main_log_emerg("devpoll close(%d) failed %E", + ds->devpoll, nxt_errno); + } + } + + nxt_free(ds->events); + nxt_free(ds->changes); + nxt_free(ds->devpoll_changes); + nxt_event_set_fd_hash_destroy(&ds->fd_hash); + nxt_free(ds); +} + + +static void +nxt_devpoll_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_DEFAULT; + ev->write = NXT_EVENT_DEFAULT; + + nxt_devpoll_change(event_set, ev, NXT_DEVPOLL_ADD, POLLIN | POLLOUT); +} + + +/* + * Solaris does not automatically remove a closed file descriptor from + * a "/dev/poll" set: ioctl(DP_ISPOLLED) for the descriptor returns 1, + * significative of active descriptor. POLLREMOVE can remove already + * closed file descriptor, so the removal can be batched, Solaris poll(7d): + * + * When using the "/dev/poll" driver, you should remove a closed file + * descriptor from a monitored poll set. Failure to do so may result + * in a POLLNVAL revents being returned for the closed file descriptor. + * When a file descriptor is closed but not removed from the monitored + * set, and is reused in subsequent open of a different device, you + * will be polling the device associated with the reused file descriptor. + * In a multithreaded application, careful coordination among threads + * doing close and DP_POLL ioctl is recommended for consistent results. + * + * Besides Solaris and HP-UX allow to add invalid descriptors to an + * "/dev/poll" set, although the descriptors are not marked as polled, + * that is, ioctl(DP_ISPOLLED) returns 0. + */ + +static void +nxt_devpoll_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE || ev->write != NXT_EVENT_INACTIVE) { + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_devpoll_change(event_set, ev, NXT_DEVPOLL_DELETE, POLLREMOVE); + } +} + + +#if (NXT_HPUX) + +/* + * HP-UX poll(7): + * + * When a polled file descriptor is closed, it is automatically + * deregistered. + */ + +static void +nxt_devpoll_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_devpoll_drop_changes(event_set, ev); +} + + +static void +nxt_devpoll_drop_changes(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_devpoll_change_t *dst, *src, *end; + nxt_devpoll_event_set_t *ds; + + ds = &event_set->devpoll; + + dst = ds->devpoll_changes; + end = dst + ds->nchanges; + + for (src = dst; src < end; src++) { + + if (src->event == ev) { + continue; + } + + if (dst != src) { + *dst = *src; + } + + dst++; + } + + ds->nchanges -= end - dst; +} + +#endif + + +/* + * Solaris poll(7d): + * + * The fd field specifies the file descriptor being polled. The events + * field indicates the interested poll events on the file descriptor. + * If a pollfd array contains multiple pollfd entries with the same fd field, + * the "events" field in each pollfd entry is OR'ed. A special POLLREMOVE + * event in the events field of the pollfd structure removes the fd from + * the monitored set. The revents field is not used. + */ + +static void +nxt_devpoll_enable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + if (ev->read != NXT_EVENT_BLOCKED) { + + events = POLLIN; + + if (ev->write == NXT_EVENT_INACTIVE) { + op = NXT_DEVPOLL_ADD; + + } else if (ev->write == NXT_EVENT_BLOCKED) { + ev->write = NXT_EVENT_INACTIVE; + op = NXT_DEVPOLL_CHANGE; + + } else { + op = NXT_DEVPOLL_UPDATE; + events = POLLIN | POLLOUT; + } + + nxt_devpoll_change(event_set, ev, op, events); + } + + ev->read = NXT_EVENT_DEFAULT; +} + + +static void +nxt_devpoll_enable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + if (ev->write != NXT_EVENT_BLOCKED) { + + events = POLLOUT; + + if (ev->read == NXT_EVENT_INACTIVE) { + op = NXT_DEVPOLL_ADD; + + } else if (ev->read == NXT_EVENT_BLOCKED) { + ev->read = NXT_EVENT_INACTIVE; + op = NXT_DEVPOLL_CHANGE; + + } else { + op = NXT_DEVPOLL_UPDATE; + events = POLLIN | POLLOUT; + } + + nxt_devpoll_change(event_set, ev, op, events); + } + + ev->write = NXT_EVENT_DEFAULT; +} + + +static void +nxt_devpoll_disable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->read = NXT_EVENT_INACTIVE; + + if (ev->write <= NXT_EVENT_BLOCKED) { + ev->write = NXT_EVENT_INACTIVE; + op = NXT_DEVPOLL_DELETE; + events = POLLREMOVE; + + } else { + op = NXT_DEVPOLL_CHANGE; + events = POLLOUT; + } + + nxt_devpoll_change(event_set, ev, op, events); +} + + +static void +nxt_devpoll_disable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->write = NXT_EVENT_INACTIVE; + + if (ev->read <= NXT_EVENT_BLOCKED) { + ev->read = NXT_EVENT_INACTIVE; + op = NXT_DEVPOLL_DELETE; + events = POLLREMOVE; + + } else { + op = NXT_DEVPOLL_CHANGE; + events = POLLIN; + } + + nxt_devpoll_change(event_set, ev, op, events); +} + + +static void +nxt_devpoll_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_devpoll_block_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write != NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_devpoll_oneshot_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_devpoll_enable_read(event_set, ev); + + ev->read = NXT_EVENT_ONESHOT; +} + + +static void +nxt_devpoll_oneshot_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_devpoll_enable_write(event_set, ev); + + ev->write = NXT_EVENT_ONESHOT; +} + + +static void +nxt_devpoll_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_uint_t op, nxt_uint_t events) +{ + nxt_devpoll_change_t *ch; + nxt_devpoll_event_set_t *ds; + + ds = &event_set->devpoll; + + nxt_log_debug(ev->log, "devpoll %d change fd:%d op:%ui ev:%04Xi", + ds->devpoll, ev->fd, op, events); + + if (ds->nchanges >= ds->mchanges) { + (void) nxt_devpoll_commit_changes(nxt_thread(), ds); + } + + ch = &ds->devpoll_changes[ds->nchanges++]; + ch->op = op; + ch->fd = ev->fd; + ch->events = events; + ch->event = ev; +} + + +static nxt_int_t +nxt_devpoll_commit_changes(nxt_thread_t *thr, nxt_devpoll_event_set_t *ds) +{ + size_t n; + nxt_int_t ret, retval; + struct pollfd *pfd; + nxt_devpoll_change_t *ch, *end; + + nxt_log_debug(thr->log, "devpoll %d changes:%ui", + ds->devpoll, ds->nchanges); + + retval = NXT_OK; + n = 0; + ch = ds->devpoll_changes; + end = ch + ds->nchanges; + + do { + nxt_log_debug(thr->log, "devpoll fd:%d op:%d ev:%04Xd", + ch->fd, ch->op, ch->events); + + if (ch->op == NXT_DEVPOLL_CHANGE) { + pfd = &ds->changes[n++]; + pfd->fd = ch->fd; + pfd->events = POLLREMOVE; + pfd->revents = 0; + } + + pfd = &ds->changes[n++]; + pfd->fd = ch->fd; + pfd->events = ch->events; + pfd->revents = 0; + + ch++; + + } while (ch < end); + + ch = ds->devpoll_changes; + end = ch + ds->nchanges; + + ret = nxt_devpoll_write(thr, ds->devpoll, ds->changes, n); + + if (nxt_slow_path(ret != NXT_OK)) { + do { + nxt_devpoll_change_error(thr, ds, ch->event); + ch++; + } while (ch < end); + + ds->nchanges = 0; + + return NXT_ERROR; + } + + do { + if (ch->op == NXT_DEVPOLL_ADD) { + ret = nxt_event_set_fd_hash_add(&ds->fd_hash, ch->fd, ch->event); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_devpoll_change_error(thr, ds, ch->event); + retval = NXT_ERROR; + } + + } else if (ch->op == NXT_DEVPOLL_DELETE) { + nxt_event_set_fd_hash_delete(&ds->fd_hash, ch->fd, 0); + } + + /* Nothing tp do for NXT_DEVPOLL_UPDATE and NXT_DEVPOLL_CHANGE. */ + + ch++; + + } while (ch < end); + + ds->nchanges = 0; + + return retval; +} + + +static void +nxt_devpoll_change_error(nxt_thread_t *thr, nxt_devpoll_event_set_t *ds, + nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + ev->error_handler, ev, ev->data, ev->log); + + nxt_event_set_fd_hash_delete(&ds->fd_hash, ev->fd, 1); + + nxt_devpoll_remove(thr, ds, ev->fd); +} + + +static void +nxt_devpoll_remove(nxt_thread_t *thr, nxt_devpoll_event_set_t *ds, nxt_fd_t fd) +{ + int n; + struct pollfd pfd; + + pfd.fd = fd; + pfd.events = 0; + pfd.revents = 0; + + n = ioctl(ds->devpoll, DP_ISPOLLED, &pfd); + + nxt_log_debug(thr->log, "ioctl(%d, DP_ISPOLLED, %d): %d", + ds->devpoll, fd, n); + + if (n == 0) { + /* The file descriptor is not in the set. */ + return; + } + + if (n == -1) { + nxt_log_alert(thr->log, "ioctl(%d, DP_ISPOLLED, %d) failed %E", + ds->devpoll, fd, nxt_errno); + /* Fall through. */ + } + + /* n == 1: the file descriptor is in the set. */ + + nxt_log_debug(thr->log, "devpoll %d remove fd:%d", ds->devpoll, fd); + + pfd.fd = fd; + pfd.events = POLLREMOVE; + pfd.revents = 0; + + nxt_devpoll_write(thr, ds->devpoll, &pfd, 1); +} + + +static nxt_int_t +nxt_devpoll_write(nxt_thread_t *thr, int devpoll, struct pollfd *pfd, + size_t n) +{ + nxt_log_debug(thr->log, "devpoll write(%d) changes:%uz", devpoll, n); + + n *= sizeof(struct pollfd); + + if (nxt_slow_path(write(devpoll, pfd, n) == (ssize_t) n)) { + return NXT_OK; + } + + nxt_log_alert(thr->log, "devpoll write(%d) failed %E", + devpoll, nxt_errno); + + return NXT_ERROR; +} + + +static void +nxt_devpoll_set_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout) +{ + int nevents; + nxt_fd_t fd; + nxt_int_t i; + nxt_err_t err; + nxt_uint_t events, level; + struct dvpoll dvp; + struct pollfd *pfd; + nxt_event_fd_t *ev; + nxt_devpoll_event_set_t *ds; + + ds = &event_set->devpoll; + + if (ds->nchanges != 0) { + if (nxt_devpoll_commit_changes(thr, ds) != NXT_OK) { + /* Error handlers have been enqueued on failure. */ + timeout = 0; + } + } + + nxt_log_debug(thr->log, "ioctl(%d, DP_POLL) timeout:%M", + ds->devpoll, timeout); + + dvp.dp_fds = ds->events; + dvp.dp_nfds = ds->mevents; + dvp.dp_timeout = timeout; + + nevents = ioctl(ds->devpoll, DP_POLL, &dvp); + + err = (nevents == -1) ? nxt_errno : 0; + + nxt_thread_time_update(thr); + + nxt_log_debug(thr->log, "ioctl(%d, DP_POLL): %d", ds->devpoll, nevents); + + if (nevents == -1) { + level = (err == NXT_EINTR) ? NXT_LOG_INFO : NXT_LOG_ALERT; + nxt_log_error(level, thr->log, "ioctl(%d, DP_POLL) failed %E", + ds->devpoll, err); + return; + } + + for (i = 0; i < nevents; i++) { + + pfd = &ds->events[i]; + fd = pfd->fd; + events = pfd->revents; + + ev = nxt_event_set_fd_hash_get(&ds->fd_hash, fd); + + if (nxt_slow_path(ev == NULL)) { + nxt_log_alert(thr->log, "ioctl(%d, DP_POLL) returned invalid " + "fd:%d ev:%04Xd rev:%04uXi", + ds->devpoll, fd, pfd->events, events); + + nxt_devpoll_remove(thr, ds, fd); + continue; + } + + nxt_log_debug(ev->log, "devpoll: fd:%d ev:%04uXi rd:%d wr:%d", + fd, events, ev->read, ev->write); + + if (nxt_slow_path(events & (POLLERR | POLLHUP | POLLNVAL)) != 0) { + nxt_log_alert(ev->log, + "ioctl(%d, DP_POLL) error fd:%d ev:%04Xd rev:%04uXi", + ds->devpoll, fd, pfd->events, events); + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + ev->error_handler, ev, ev->data, ev->log); + continue; + } + + if (events & POLLIN) { + ev->read_ready = 1; + + if (ev->read != NXT_EVENT_BLOCKED) { + + if (ev->read == NXT_EVENT_ONESHOT) { + nxt_devpoll_disable_read(event_set, ev); + } + + nxt_thread_work_queue_add(thr, ev->read_work_queue, + ev->read_handler, + ev, ev->data, ev->log); + } + } + + if (events & POLLOUT) { + ev->write_ready = 1; + + if (ev->write != NXT_EVENT_BLOCKED) { + + if (ev->write == NXT_EVENT_ONESHOT) { + nxt_devpoll_disable_write(event_set, ev); + } + + nxt_thread_work_queue_add(thr, ev->write_work_queue, + ev->write_handler, + ev, ev->data, ev->log); + } + } + } +} diff --git a/src/nxt_djb_hash.c b/src/nxt_djb_hash.c new file mode 100644 index 00000000..cd315869 --- /dev/null +++ b/src/nxt_djb_hash.c @@ -0,0 +1,45 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +uint32_t +nxt_djb_hash(const void *data, size_t len) +{ + uint32_t hash; + const u_char *p; + + p = data; + hash = NXT_DJB_HASH_INIT; + + while (len != 0) { + hash = nxt_djb_hash_add(hash, *p++); + len--; + } + + return hash; +} + + +uint32_t +nxt_djb_hash_lowcase(const void *data, size_t len) +{ + u_char c; + uint32_t hash; + const u_char *p; + + p = data; + hash = NXT_DJB_HASH_INIT; + + while (len != 0) { + c = *p++; + hash = nxt_djb_hash_add(hash, nxt_lowcase(c)); + len--; + } + + return hash; +} diff --git a/src/nxt_djb_hash.h b/src/nxt_djb_hash.h new file mode 100644 index 00000000..c7ba6fdb --- /dev/null +++ b/src/nxt_djb_hash.h @@ -0,0 +1,26 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_DJB_HASH_H_INCLUDED_ +#define _NXT_DJB_HASH_H_INCLUDED_ + + +/* A fast and simple hash function by Daniel J. Bernstein. */ + + +NXT_EXPORT uint32_t nxt_djb_hash(const void *data, size_t len); +NXT_EXPORT uint32_t nxt_djb_hash_lowcase(const void *data, size_t len); + + +#define NXT_DJB_HASH_INIT 5381 + + +#define \ +nxt_djb_hash_add(hash, val) \ + ((uint32_t) ((((hash) << 5) + (hash)) ^ (uint32_t) (val))) + + +#endif /* _NXT_DJB_HASH_H_INCLUDED_ */ diff --git a/src/nxt_dyld.c b/src/nxt_dyld.c new file mode 100644 index 00000000..63e6be14 --- /dev/null +++ b/src/nxt_dyld.c @@ -0,0 +1,86 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_int_t +nxt_dyld_load(nxt_dyld_t *dyld) +{ + const char *err; + + dyld->handle = dlopen(dyld->name, RTLD_NOW | RTLD_GLOBAL); + + if (dyld->handle != NULL) { + nxt_thread_log_debug("dlopen(\"%s\")", dyld->name); + return NXT_OK; + } + + err = dlerror(); + if (err == NULL) { + err = "(null)"; + } + + nxt_thread_log_alert("dlopen(\"%s\") failed: %s", dyld->name, err); + + return NXT_ERROR; +} + + +void * +nxt_dyld_symbol(nxt_dyld_t *dyld, const char *symbol) +{ + void *handle, *s; + const char *name; + const char *err; + + if (dyld == NXT_DYLD_ANY) { + handle = RTLD_DEFAULT; + name = "RTLD_DEFAULT"; + + } else { + handle = dyld->handle; + name = dyld->name; + } + + s = dlsym(handle, symbol); + + if (s != NULL) { + nxt_thread_log_debug("dlsym(\"%s\", \"%s\")", name, symbol); + return s; + } + + err = dlerror(); + if (err == NULL) { + err = "(null)"; + } + + nxt_thread_log_alert("dlsym(\"%s\", \"%s\") failed: %s", name, symbol, err); + + return s; +} + + +nxt_int_t +nxt_dyld_unload(nxt_dyld_t *dyld) +{ + const char *err; + + if (dlclose(dyld->handle) == 0) { + nxt_thread_log_debug("dlclose(\"%s\")", dyld->name); + return NXT_OK; + } + + err = dlerror(); + + if (err == NULL) { + err = "(null)"; + } + + nxt_thread_log_alert("dlclose(\"%s\") failed: %s", dyld->name, err); + + return NXT_ERROR; +} diff --git a/src/nxt_dyld.h b/src/nxt_dyld.h new file mode 100644 index 00000000..a0cbeda3 --- /dev/null +++ b/src/nxt_dyld.h @@ -0,0 +1,30 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_DYLD_H_INCLUDED_ +#define _NXT_UNIX_DYLD_H_INCLUDED_ + + +typedef struct { + void *handle; + char *name; +} nxt_dyld_t; + + +#define NXT_DYLD_ANY RTLD_DEFAULT + + +#define \ +nxt_dyld_is_valid(dyld) \ + ((dyld)->handle != NULL) + + +NXT_EXPORT nxt_int_t nxt_dyld_load(nxt_dyld_t *dyld); +NXT_EXPORT void *nxt_dyld_symbol(nxt_dyld_t *dyld, const char *symbol); +NXT_EXPORT nxt_int_t nxt_dyld_unload(nxt_dyld_t *dyld); + + +#endif /* _NXT_UNIX_DYLD_H_INCLUDED_ */ diff --git a/src/nxt_epoll.c b/src/nxt_epoll.c new file mode 100644 index 00000000..65e9eb8d --- /dev/null +++ b/src/nxt_epoll.c @@ -0,0 +1,1167 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * The first epoll version has been introduced in Linux 2.5.44. The + * interface was changed several times since then and the final version + * of epoll_create(), epoll_ctl(), epoll_wait(), and EPOLLET mode has + * been introduced in Linux 2.6.0 and is supported since glibc 2.3.2. + * + * EPOLLET mode did not work reliable in early implementaions and in + * Linux 2.4 backport. + * + * EPOLLONESHOT Linux 2.6.2, glibc 2.3. + * EPOLLRDHUP Linux 2.6.17, glibc 2.8. + * epoll_pwait() Linux 2.6.19, glibc 2.6. + * signalfd() Linux 2.6.22, glibc 2.7. + * eventfd() Linux 2.6.22, glibc 2.7. + * timerfd_create() Linux 2.6.25, glibc 2.8. + * epoll_create1() Linux 2.6.27, glibc 2.9. + * signalfd4() Linux 2.6.27, glibc 2.9. + * eventfd2() Linux 2.6.27, glibc 2.9. + * accept4() Linux 2.6.28, glibc 2.10. + * eventfd2(EFD_SEMAPHORE) Linux 2.6.30, glibc 2.10. + */ + + +#if (NXT_HAVE_EPOLL_EDGE) +static nxt_event_set_t *nxt_epoll_edge_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +#endif +static nxt_event_set_t *nxt_epoll_level_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +static nxt_event_set_t *nxt_epoll_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents, nxt_event_conn_io_t *io, + uint32_t mode); +static void nxt_epoll_test_accept4(nxt_event_conn_io_t *io); +static void nxt_epoll_free(nxt_event_set_t *event_set); +static void nxt_epoll_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_epoll_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_epoll_delete(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_epoll_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_epoll_enable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_enable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_disable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_disable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_block_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_block_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_oneshot_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_oneshot_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_enable_accept(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_epoll_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + int op, uint32_t events); +static nxt_int_t nxt_epoll_commit_changes(nxt_thread_t *thr, + nxt_epoll_event_set_t *es); +static void nxt_epoll_error_handler(nxt_thread_t *thr, void *obj, + void *data); +#if (NXT_HAVE_SIGNALFD) +static nxt_int_t nxt_epoll_add_signal(nxt_epoll_event_set_t *es, + nxt_event_signals_t *signals); +static void nxt_epoll_signalfd_handler(nxt_thread_t *thr, void *obj, + void *data); +#endif +#if (NXT_HAVE_EVENTFD) +static nxt_int_t nxt_epoll_enable_post(nxt_event_set_t *event_set, + nxt_work_handler_t handler); +static void nxt_epoll_eventfd_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_epoll_signal(nxt_event_set_t *event_set, nxt_uint_t signo); +#endif +static void nxt_epoll_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout); + +#if (NXT_HAVE_ACCEPT4) +static void nxt_epoll_event_conn_io_accept4(nxt_thread_t *thr, void *obj, + void *data); +#endif + + +#if (NXT_HAVE_EPOLL_EDGE) + +static void nxt_epoll_edge_event_conn_io_connect(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_epoll_edge_event_conn_connected(nxt_thread_t *thr, void *obj, + void *data); +static ssize_t nxt_epoll_edge_event_conn_io_recvbuf(nxt_event_conn_t *c, + nxt_buf_t *b); + + +static nxt_event_conn_io_t nxt_epoll_edge_event_conn_io = { + nxt_epoll_edge_event_conn_io_connect, + nxt_event_conn_io_accept, + + nxt_event_conn_io_read, + nxt_epoll_edge_event_conn_io_recvbuf, + nxt_event_conn_io_recv, + + nxt_event_conn_io_write, + nxt_event_conn_io_write_chunk, + +#if (NXT_HAVE_LINUX_SENDFILE) + nxt_linux_event_conn_io_sendfile, +#else + nxt_event_conn_io_sendbuf, +#endif + + nxt_event_conn_io_writev, + nxt_event_conn_io_send, + + nxt_event_conn_io_shutdown, +}; + + +const nxt_event_set_ops_t nxt_epoll_edge_event_set = { + "epoll_edge", + nxt_epoll_edge_create, + nxt_epoll_free, + nxt_epoll_enable, + nxt_epoll_disable, + nxt_epoll_delete, + nxt_epoll_close, + nxt_epoll_enable_read, + nxt_epoll_enable_write, + nxt_epoll_disable_read, + nxt_epoll_disable_write, + nxt_epoll_block_read, + nxt_epoll_block_write, + nxt_epoll_oneshot_read, + nxt_epoll_oneshot_write, + nxt_epoll_enable_accept, + NULL, + NULL, +#if (NXT_HAVE_EVENTFD) + nxt_epoll_enable_post, + nxt_epoll_signal, +#else + NULL, + NULL, +#endif + nxt_epoll_poll, + + &nxt_epoll_edge_event_conn_io, + +#if (NXT_HAVE_INOTIFY) + NXT_FILE_EVENTS, +#else + NXT_NO_FILE_EVENTS, +#endif + +#if (NXT_HAVE_SIGNALFD) + NXT_SIGNAL_EVENTS, +#else + NXT_NO_SIGNAL_EVENTS, +#endif +}; + +#endif + + +const nxt_event_set_ops_t nxt_epoll_level_event_set = { + "epoll_level", + nxt_epoll_level_create, + nxt_epoll_free, + nxt_epoll_enable, + nxt_epoll_disable, + nxt_epoll_delete, + nxt_epoll_close, + nxt_epoll_enable_read, + nxt_epoll_enable_write, + nxt_epoll_disable_read, + nxt_epoll_disable_write, + nxt_epoll_block_read, + nxt_epoll_block_write, + nxt_epoll_oneshot_read, + nxt_epoll_oneshot_write, + nxt_epoll_enable_accept, + NULL, + NULL, +#if (NXT_HAVE_EVENTFD) + nxt_epoll_enable_post, + nxt_epoll_signal, +#else + NULL, + NULL, +#endif + nxt_epoll_poll, + + &nxt_unix_event_conn_io, + +#if (NXT_HAVE_INOTIFY) + NXT_FILE_EVENTS, +#else + NXT_NO_FILE_EVENTS, +#endif + +#if (NXT_HAVE_SIGNALFD) + NXT_SIGNAL_EVENTS, +#else + NXT_NO_SIGNAL_EVENTS, +#endif +}; + + +#if (NXT_HAVE_EPOLL_EDGE) + +static nxt_event_set_t * +nxt_epoll_edge_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + return nxt_epoll_create(signals, mchanges, mevents, + &nxt_epoll_edge_event_conn_io, + EPOLLET | EPOLLRDHUP); +} + +#endif + + +static nxt_event_set_t * +nxt_epoll_level_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + return nxt_epoll_create(signals, mchanges, mevents, + &nxt_unix_event_conn_io, 0); +} + + +static nxt_event_set_t * +nxt_epoll_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents, nxt_event_conn_io_t *io, uint32_t mode) +{ + nxt_event_set_t *event_set; + nxt_epoll_event_set_t *es; + + event_set = nxt_zalloc(sizeof(nxt_epoll_event_set_t)); + if (event_set == NULL) { + return NULL; + } + + es = &event_set->epoll; + + es->epoll = -1; + es->mode = mode; + es->mchanges = mchanges; + es->mevents = mevents; +#if (NXT_HAVE_SIGNALFD) + es->signalfd.fd = -1; +#endif + + es->changes = nxt_malloc(sizeof(nxt_epoll_change_t) * mchanges); + if (es->changes == NULL) { + goto fail; + } + + es->events = nxt_malloc(sizeof(struct epoll_event) * mevents); + if (es->events == NULL) { + goto fail; + } + + es->epoll = epoll_create(1); + if (es->epoll == -1) { + nxt_main_log_emerg("epoll_create() failed %E", nxt_errno); + goto fail; + } + + nxt_main_log_debug("epoll_create(): %d", es->epoll); + +#if (NXT_HAVE_SIGNALFD) + + if (signals != NULL) { + if (nxt_epoll_add_signal(es, signals) != NXT_OK) { + goto fail; + } + } + +#endif + + nxt_epoll_test_accept4(io); + + return event_set; + +fail: + + nxt_epoll_free(event_set); + + return NULL; +} + + +static void +nxt_epoll_test_accept4(nxt_event_conn_io_t *io) +{ + static nxt_work_handler_t handler; + + if (handler == NULL) { + + handler = io->accept; + +#if (NXT_HAVE_ACCEPT4) + + (void) accept4(-1, NULL, NULL, SOCK_NONBLOCK); + + if (nxt_errno != NXT_ENOSYS) { + handler = nxt_epoll_event_conn_io_accept4; + + } else { + nxt_main_log_error(NXT_LOG_NOTICE, "accept4() failed %E", + NXT_ENOSYS); + } + +#endif + } + + io->accept = handler; +} + + +static void +nxt_epoll_free(nxt_event_set_t *event_set) +{ + nxt_epoll_event_set_t *es; + + es = &event_set->epoll; + + nxt_main_log_debug("epoll %d free", es->epoll); + +#if (NXT_HAVE_SIGNALFD) + + if (es->signalfd.fd != -1) { + if (close(es->signalfd.fd) != 0) { + nxt_main_log_emerg("signalfd close(%d) failed %E", + es->signalfd.fd, nxt_errno); + } + } + +#endif + +#if (NXT_HAVE_EVENTFD) + + if (es->eventfd.fd != -1) { + if (close(es->eventfd.fd) != 0) { + nxt_main_log_emerg("eventfd close(%d) failed %E", + es->eventfd.fd, nxt_errno); + } + } + +#endif + + if (es->epoll != -1) { + if (close(es->epoll) != 0) { + nxt_main_log_emerg("epoll close(%d) failed %E", + es->epoll, nxt_errno); + } + } + + nxt_free(es->events); + nxt_free(es); +} + + +static void +nxt_epoll_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_DEFAULT; + ev->write = NXT_EVENT_DEFAULT; + + nxt_epoll_change(event_set, ev, EPOLL_CTL_ADD, + EPOLLIN | EPOLLOUT | event_set->epoll.mode); +} + + +static void +nxt_epoll_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read > NXT_EVENT_DISABLED || ev->write > NXT_EVENT_DISABLED) { + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_epoll_change(event_set, ev, EPOLL_CTL_DEL, 0); + } +} + + +static void +nxt_epoll_delete(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE || ev->write != NXT_EVENT_INACTIVE) { + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_epoll_change(event_set, ev, EPOLL_CTL_DEL, 0); + } +} + + +/* + * Although calling close() on a file descriptor will remove any epoll + * events that reference the descriptor, in this case the close() acquires + * the kernel global "epmutex" while epoll_ctl(EPOLL_CTL_DEL) does not + * acquire the "epmutex" since Linux 3.13 if the file descriptor presents + * only in one epoll set. Thus removing events explicitly before closing + * eliminates possible lock contention. + */ + +static void +nxt_epoll_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_epoll_event_set_t *es; + + nxt_epoll_delete(event_set, ev); + + es = &event_set->epoll; + + if (es->nchanges != 0) { + (void) nxt_epoll_commit_changes(nxt_thread(), &event_set->epoll); + } +} + + +static void +nxt_epoll_enable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + int op; + uint32_t events; + + if (ev->read != NXT_EVENT_BLOCKED) { + + op = EPOLL_CTL_MOD; + events = EPOLLIN | event_set->epoll.mode; + + if (ev->read == NXT_EVENT_INACTIVE && ev->write == NXT_EVENT_INACTIVE) { + op = EPOLL_CTL_ADD; + + } else if (ev->write >= NXT_EVENT_BLOCKED) { + events |= EPOLLOUT; + } + + nxt_epoll_change(event_set, ev, op, events); + } + + ev->read = NXT_EVENT_DEFAULT; +} + + +static void +nxt_epoll_enable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + int op; + uint32_t events; + + if (ev->write != NXT_EVENT_BLOCKED) { + + op = EPOLL_CTL_MOD; + events = EPOLLOUT | event_set->epoll.mode; + + if (ev->read == NXT_EVENT_INACTIVE && ev->write == NXT_EVENT_INACTIVE) { + op = EPOLL_CTL_ADD; + + } else if (ev->read >= NXT_EVENT_BLOCKED) { + events |= EPOLLIN; + } + + nxt_epoll_change(event_set, ev, op, events); + } + + ev->write = NXT_EVENT_DEFAULT; +} + + +static void +nxt_epoll_disable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + int op; + uint32_t events; + + ev->read = NXT_EVENT_INACTIVE; + + if (ev->write <= NXT_EVENT_DISABLED) { + ev->write = NXT_EVENT_INACTIVE; + op = EPOLL_CTL_DEL; + events = 0; + + } else { + op = EPOLL_CTL_MOD; + events = EPOLLOUT | event_set->epoll.mode; + } + + nxt_epoll_change(event_set, ev, op, events); +} + + +static void +nxt_epoll_disable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + int op; + uint32_t events; + + ev->write = NXT_EVENT_INACTIVE; + + if (ev->read <= NXT_EVENT_DISABLED) { + ev->write = NXT_EVENT_INACTIVE; + op = EPOLL_CTL_DEL; + events = 0; + + } else { + op = EPOLL_CTL_MOD; + events = EPOLLIN | event_set->epoll.mode; + } + + nxt_epoll_change(event_set, ev, op, events); +} + + +static void +nxt_epoll_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_epoll_block_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write != NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_BLOCKED; + } +} + + +/* + * NXT_EVENT_DISABLED state is used to track whether EPOLLONESHOT + * event should be added or modified, epoll_ctl(2): + * + * EPOLLONESHOT (since Linux 2.6.2) + * Sets the one-shot behavior for the associated file descriptor. + * This means that after an event is pulled out with epoll_wait(2) + * the associated file descriptor is internally disabled and no + * other events will be reported by the epoll interface. The user + * must call epoll_ctl() with EPOLL_CTL_MOD to rearm the file + * descriptor with a new event mask. + */ + +static void +nxt_epoll_oneshot_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + int op; + + op = (ev->read == NXT_EVENT_INACTIVE && ev->write == NXT_EVENT_INACTIVE) ? + EPOLL_CTL_ADD : EPOLL_CTL_MOD; + + ev->read = NXT_EVENT_ONESHOT; + ev->write = NXT_EVENT_INACTIVE; + + nxt_epoll_change(event_set, ev, op, EPOLLIN | EPOLLONESHOT); +} + + +static void +nxt_epoll_oneshot_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + int op; + + op = (ev->read == NXT_EVENT_INACTIVE && ev->write == NXT_EVENT_INACTIVE) ? + EPOLL_CTL_ADD : EPOLL_CTL_MOD; + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_ONESHOT; + + nxt_epoll_change(event_set, ev, op, EPOLLOUT | EPOLLONESHOT); +} + + +static void +nxt_epoll_enable_accept(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_DEFAULT; + + nxt_epoll_change(event_set, ev, EPOLL_CTL_ADD, EPOLLIN); +} + + +/* + * epoll changes are batched to improve instruction and data cache + * locality of several epoll_ctl() calls followed by epoll_wait() call. + */ + +static void +nxt_epoll_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, int op, + uint32_t events) +{ + nxt_epoll_change_t *ch; + nxt_epoll_event_set_t *es; + + es = &event_set->epoll; + + nxt_log_debug(ev->log, "epoll %d set event: fd:%d op:%d ev:%XD", + es->epoll, ev->fd, op, events); + + if (es->nchanges >= es->mchanges) { + (void) nxt_epoll_commit_changes(nxt_thread(), es); + } + + ch = &es->changes[es->nchanges++]; + ch->op = op; + ch->fd = ev->fd; + ch->event.events = events; + ch->event.data.ptr = ev; +} + + +static nxt_int_t +nxt_epoll_commit_changes(nxt_thread_t *thr, nxt_epoll_event_set_t *es) +{ + nxt_int_t ret; + nxt_event_fd_t *ev; + nxt_epoll_change_t *ch, *end; + + nxt_log_debug(thr->log, "epoll %d changes:%ui", es->epoll, es->nchanges); + + ret = NXT_OK; + ch = es->changes; + end = ch + es->nchanges; + + do { + ev = ch->event.data.ptr; + + nxt_log_debug(ev->log, "epoll_ctl(%d): fd:%d op:%d ev:%XD", + es->epoll, ch->fd, ch->op, ch->event.events); + + if (epoll_ctl(es->epoll, ch->op, ch->fd, &ch->event) != 0) { + nxt_log_alert(ev->log, "epoll_ctl(%d, %d, %d) failed %E", + es->epoll, ch->op, ch->fd, nxt_errno); + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_epoll_error_handler, + ev, ev->data, ev->log); + + ret = NXT_ERROR; + } + + ch++; + + } while (ch < end); + + es->nchanges = 0; + + return ret; +} + + +static void +nxt_epoll_error_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_fd_t *ev; + + ev = obj; + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + ev->error_handler(thr, ev, data); +} + + +#if (NXT_HAVE_SIGNALFD) + +static nxt_int_t +nxt_epoll_add_signal(nxt_epoll_event_set_t *es, nxt_event_signals_t *signals) +{ + int fd; + struct epoll_event ee; + + if (sigprocmask(SIG_BLOCK, &signals->sigmask, NULL) != 0) { + nxt_main_log_alert("sigprocmask(SIG_BLOCK) failed %E", nxt_errno); + return NXT_ERROR; + } + + /* + * Glibc signalfd() wrapper always has the flags argument. Glibc 2.7 + * and 2.8 signalfd() wrappers call the original signalfd() syscall + * without the flags argument. Glibc 2.9+ signalfd() wrapper at first + * tries to call signalfd4() syscall and if it fails then calls the + * original signalfd() syscall. For this reason the non-blocking mode + * is set separately. + */ + + fd = signalfd(-1, &signals->sigmask, 0); + + if (fd == -1) { + nxt_main_log_emerg("signalfd(%d) failed %E", + es->signalfd.fd, nxt_errno); + return NXT_ERROR; + } + + es->signalfd.fd = fd; + + if (nxt_fd_nonblocking(fd) != NXT_OK) { + return NXT_ERROR; + } + + nxt_main_log_debug("signalfd(): %d", fd); + + es->signalfd.data = signals->handler; + es->signalfd.read_work_queue = nxt_thread_main_work_queue(); + es->signalfd.read_handler = nxt_epoll_signalfd_handler; + es->signalfd.log = &nxt_main_log; + + ee.events = EPOLLIN; + ee.data.ptr = &es->signalfd; + + if (epoll_ctl(es->epoll, EPOLL_CTL_ADD, fd, &ee) != 0) { + nxt_main_log_alert("epoll_ctl(%d, %d, %d) failed %E", + es->epoll, EPOLL_CTL_ADD, fd, nxt_errno); + + return NXT_ERROR; + } + + return NXT_OK; +} + + +static void +nxt_epoll_signalfd_handler(nxt_thread_t *thr, void *obj, void *data) +{ + int n; + nxt_event_fd_t *ev; + nxt_work_handler_t handler; + struct signalfd_siginfo sfd; + + ev = obj; + handler = data; + + nxt_log_debug(thr->log, "signalfd handler"); + + n = read(ev->fd, &sfd, sizeof(struct signalfd_siginfo)); + + nxt_log_debug(thr->log, "read signalfd(%d): %d", ev->fd, n); + + if (n != sizeof(struct signalfd_siginfo)) { + nxt_log_alert(thr->log, "read signalfd(%d) failed %E", + ev->fd, nxt_errno); + } + + nxt_log_debug(thr->log, "signalfd(%d) signo:%d", ev->fd, sfd.ssi_signo); + + handler(thr, (void *) (uintptr_t) sfd.ssi_signo, NULL); +} + +#endif + + +#if (NXT_HAVE_EVENTFD) + +static nxt_int_t +nxt_epoll_enable_post(nxt_event_set_t *event_set, nxt_work_handler_t handler) +{ + struct epoll_event ee; + nxt_epoll_event_set_t *es; + + es = &event_set->epoll; + es->post_handler = handler; + + /* + * Glibc eventfd() wrapper always has the flags argument. Glibc 2.7 + * and 2.8 eventfd() wrappers call the original eventfd() syscall + * without the flags argument. Glibc 2.9+ eventfd() wrapper at first + * tries to call eventfd2() syscall and if it fails then calls the + * original eventfd() syscall. For this reason the non-blocking mode + * is set separately. + */ + + es->eventfd.fd = eventfd(0, 0); + + if (es->eventfd.fd == -1) { + nxt_main_log_emerg("eventfd() failed %E", nxt_errno); + return NXT_ERROR; + } + + if (nxt_fd_nonblocking(es->eventfd.fd) != NXT_OK) { + return NXT_ERROR; + } + + nxt_main_log_debug("eventfd(): %d", es->eventfd.fd); + + es->eventfd.read_work_queue = nxt_thread_main_work_queue(); + es->eventfd.read_handler = nxt_epoll_eventfd_handler; + es->eventfd.data = es; + es->eventfd.log = &nxt_main_log; + + ee.events = EPOLLIN | EPOLLET; + ee.data.ptr = &es->eventfd; + + if (epoll_ctl(es->epoll, EPOLL_CTL_ADD, es->eventfd.fd, &ee) == 0) { + return NXT_OK; + } + + nxt_main_log_alert("epoll_ctl(%d, %d, %d) failed %E", + es->epoll, EPOLL_CTL_ADD, es->eventfd.fd, nxt_errno); + + return NXT_ERROR; +} + + +static void +nxt_epoll_eventfd_handler(nxt_thread_t *thr, void *obj, void *data) +{ + int n; + uint64_t events; + nxt_epoll_event_set_t *es; + + es = data; + + nxt_log_debug(thr->log, "eventfd handler, times:%ui", es->neventfd); + + /* + * The maximum value after write() to a eventfd() descriptor will + * block or return EAGAIN is 0xfffffffffffffffe, so the descriptor + * can be read once per many notifications, for example, once per + * 2^32-2 noticifcations. Since the eventfd() file descriptor is + * always registered in EPOLLET mode, epoll returns event about + * only the latest write() to the descriptor. + */ + + if (es->neventfd++ >= 0xfffffffe) { + es->neventfd = 0; + + n = read(es->eventfd.fd, &events, sizeof(uint64_t)); + + nxt_log_debug(thr->log, "read(%d): %d events:%uL", + es->eventfd.fd, n, events); + + if (n != sizeof(uint64_t)) { + nxt_log_alert(thr->log, "read eventfd(%d) failed %E", + es->eventfd.fd, nxt_errno); + } + } + + es->post_handler(thr, NULL, NULL); +} + + +static void +nxt_epoll_signal(nxt_event_set_t *event_set, nxt_uint_t signo) +{ + uint64_t event; + nxt_epoll_event_set_t *es; + + es = &event_set->epoll; + + /* + * eventfd() presents along with signalfd(), so the function + * is used only to post events and the signo argument is ignored. + */ + + event = 1; + + if (write(es->eventfd.fd, &event, sizeof(uint64_t)) != sizeof(uint64_t)) { + nxt_thread_log_alert("write(%d) to eventfd failed %E", + es->eventfd.fd, nxt_errno); + } +} + +#endif + + +static void +nxt_epoll_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout) +{ + int nevents; + uint32_t events; + nxt_int_t i; + nxt_err_t err; + nxt_bool_t error; + nxt_uint_t level; + nxt_event_fd_t *ev; + struct epoll_event *event; + nxt_epoll_event_set_t *es; + + es = &event_set->epoll; + + if (es->nchanges != 0) { + if (nxt_epoll_commit_changes(thr, es) != NXT_OK) { + /* Error handlers have been enqueued on failure. */ + timeout = 0; + } + } + + nxt_log_debug(thr->log, "epoll_wait(%d) timeout:%M", es->epoll, timeout); + + nevents = epoll_wait(es->epoll, es->events, es->mevents, timeout); + + err = (nevents == -1) ? nxt_errno : 0; + + nxt_thread_time_update(thr); + + nxt_log_debug(thr->log, "epoll_wait(%d): %d", es->epoll, nevents); + + if (nevents == -1) { + level = (err == NXT_EINTR) ? NXT_LOG_INFO : NXT_LOG_ALERT; + nxt_log_error(level, thr->log, "epoll_wait(%d) failed %E", + es->epoll, err); + return; + } + + for (i = 0; i < nevents; i++) { + + event = &es->events[i]; + events = event->events; + ev = event->data.ptr; + + nxt_log_debug(ev->log, "epoll: fd:%d ev:%04XD d:%p rd:%d wr:%d", + ev->fd, events, ev, ev->read, ev->write); + + /* + * On error epoll may set EPOLLERR and EPOLLHUP only without EPOLLIN or + * EPOLLOUT, so the "error" variable enqueues only one active handler. + */ + error = ((events & (EPOLLERR | EPOLLHUP)) != 0); + ev->epoll_error = error; + +#if (NXT_HAVE_EPOLL_EDGE) + + ev->epoll_eof = ((events & EPOLLRDHUP) != 0); + +#endif + + if ((events & EPOLLIN) || error) { + ev->read_ready = 1; + + if (ev->read != NXT_EVENT_BLOCKED) { + + if (ev->read == NXT_EVENT_ONESHOT) { + ev->read = NXT_EVENT_DISABLED; + } + + error = 0; + + nxt_thread_work_queue_add(thr, ev->read_work_queue, + ev->read_handler, + ev, ev->data, ev->log); + + } else if (event_set->epoll.mode == 0) { + /* Level-triggered mode. */ + nxt_epoll_disable_read(event_set, ev); + } + } + + if ((events & EPOLLOUT) || error) { + ev->write_ready = 1; + + if (ev->write != NXT_EVENT_BLOCKED) { + + if (ev->write == NXT_EVENT_ONESHOT) { + ev->write = NXT_EVENT_DISABLED; + } + + error = 0; + + nxt_thread_work_queue_add(thr, ev->write_work_queue, + ev->write_handler, + ev, ev->data, ev->log); + + } else if (event_set->epoll.mode == 0) { + /* Level-triggered mode. */ + nxt_epoll_disable_write(event_set, ev); + } + } + + if (error) { + ev->read_ready = 1; + ev->write_ready = 1; + } + } +} + + +#if (NXT_HAVE_ACCEPT4) + +static void +nxt_epoll_event_conn_io_accept4(nxt_thread_t *thr, void *obj, void *data) +{ + socklen_t len; + nxt_socket_t s; + struct sockaddr *sa; + nxt_event_conn_t *c; + nxt_event_conn_listen_t *cls; + + cls = obj; + c = data; + + cls->ready--; + cls->socket.read_ready = (cls->ready != 0); + + len = nxt_socklen(c->remote); + + if (len >= sizeof(struct sockaddr)) { + sa = &c->remote->u.sockaddr; + + } else { + sa = NULL; + len = 0; + } + + s = accept4(cls->socket.fd, sa, &len, SOCK_NONBLOCK); + + if (s != -1) { + c->socket.fd = s; + + nxt_log_debug(thr->log, "accept4(%d): %d", cls->socket.fd, s); + + nxt_event_conn_accept(thr, cls, c); + return; + } + + nxt_event_conn_accept_error(thr, cls, "accept4", nxt_errno); +} + +#endif + + +#if (NXT_HAVE_EPOLL_EDGE) + +/* + * nxt_epoll_edge_event_conn_io_connect() eliminates the getsockopt() + * syscall to test pending connect() error. Although this special + * interface can work in both edge-triggered and level-triggered + * modes it is enabled only for the former mode because this mode is + * available in all modern Linux distributions. For the latter mode + * it is required to create additional nxt_epoll_level_event_conn_io + * with single non-generic connect() interface. + */ + +static void +nxt_epoll_edge_event_conn_io_connect(nxt_thread_t *thr, void *obj, + void *data) +{ + nxt_event_conn_t *c; + nxt_work_handler_t handler; + const nxt_event_conn_state_t *state; + + c = obj; + + state = c->write_state; + + switch (nxt_socket_connect(c->socket.fd, c->remote) ){ + + case NXT_OK: + c->socket.write_ready = 1; + handler = state->ready_handler; + break; + + case NXT_AGAIN: + c->socket.write_handler = nxt_epoll_edge_event_conn_connected; + c->socket.error_handler = nxt_event_conn_connect_error; + + nxt_event_conn_timer(thr->engine, c, state, &c->write_timer); + + nxt_epoll_enable(thr->engine->event_set, &c->socket); + c->socket.read = NXT_EVENT_BLOCKED; + return; + +#if 0 + case NXT_AGAIN: + nxt_event_conn_timer(thr->engine, c, state, &c->write_timer); + + /* Fall through. */ + + case NXT_OK: + /* + * Mark both read and write directions as ready and try to perform + * I/O operations before receiving readiness notifications. + * On unconnected socket Linux send() and recv() return EAGAIN + * instead of ENOTCONN. + */ + c->socket.read_ready = 1; + c->socket.write_ready = 1; + /* + * Enabling both read and write notifications on a getting + * connected socket eliminates one epoll_ctl() syscall. + */ + c->socket.write_handler = nxt_epoll_edge_event_conn_connected; + c->socket.error_handler = state->error_handler; + + nxt_epoll_enable(thr->engine->event_set, &c->socket); + c->socket.read = NXT_EVENT_BLOCKED; + + handler = state->ready_handler; + break; +#endif + + case NXT_ERROR: + handler = state->error_handler; + break; + + default: /* NXT_DECLINED: connection refused. */ + handler = state->close_handler; + break; + } + + nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); +} + + +static void +nxt_epoll_edge_event_conn_connected(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "epoll event conn connected fd:%d", c->socket.fd); + + if (!c->socket.epoll_error) { + c->socket.write = NXT_EVENT_BLOCKED; + + if (c->write_state->autoreset_timer) { + nxt_event_timer_disable(&c->write_timer); + } + + nxt_event_conn_io_handle(thr, c->write_work_queue, + c->write_state->ready_handler, c, data); + return; + } + + nxt_event_conn_connect_test(thr, c, data); +} + + +/* + * nxt_epoll_edge_event_conn_io_recvbuf() is just wrapper around + * standard nxt_event_conn_io_recvbuf() to enforce to read a pending EOF + * in edge-triggered mode. + */ + +static ssize_t +nxt_epoll_edge_event_conn_io_recvbuf(nxt_event_conn_t *c, nxt_buf_t *b) +{ + ssize_t n; + + n = nxt_event_conn_io_recvbuf(c, b); + + if (n > 0 && c->socket.epoll_eof) { + c->socket.read_ready = 1; + } + + return n; +} + +#endif diff --git a/src/nxt_errno.c b/src/nxt_errno.c new file mode 100644 index 00000000..64e043e5 --- /dev/null +++ b/src/nxt_errno.c @@ -0,0 +1,152 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * The strerror() messages are copied because: + * + * 1) strerror() and strerror_r() functions are not Async-Signal-Safe, + * therefore, they can not be used in signal handlers; + * + * 2) a direct sys_errlist[] array may be used instead of these functions, + * but Linux linker warns about this usage: + * + * warning: `sys_errlist' is deprecated; use `strerror' or `strerror_r' instead + * warning: `sys_nerr' is deprecated; use `strerror' or `strerror_r' instead + * + * causing false bug reports. + */ + +static u_char *nxt_bootstrap_strerror(nxt_err_t err, u_char *errstr, + size_t size); +static u_char *nxt_runtime_strerror(nxt_err_t err, u_char *errstr, size_t size); + + +nxt_strerror_t nxt_strerror = nxt_bootstrap_strerror; +static nxt_str_t *nxt_sys_errlist; +static nxt_uint_t nxt_sys_nerr; + + +nxt_int_t +nxt_strerror_start(void) +{ + char *msg; + u_char *p; + size_t size, len, n; + nxt_uint_t err, invalid; + + /* The last entry. */ + size = sizeof("Unknown error") - 1; + + /* + * Linux has holes for error codes 41 and 58, so the loop + * stops only after 100 invalid codes in succession. + */ + + for (invalid = 0; invalid < 100 && nxt_sys_nerr < 65536; nxt_sys_nerr++) { + + nxt_set_errno(0); + msg = strerror((int) nxt_sys_nerr); + + /* + * strerror() behaviour on passing invalid error code depends + * on OS and version: + * Linux returns "Unknown error NN"; + * FreeBSD, NetBSD and OpenBSD return "Unknown error: NN" + * and set errno to EINVAL; + * Solaris 10 returns "Unknown error" and sets errno to EINVAL; + * Solaris 9 returns "Unknown error"; + * Solaris 2 returns NULL; + * MacOSX returns "Unknown error: NN"; + * AIX returns "Error NNN occurred."; + * HP-UX returns "Unknown error" for invalid codes lesser than 250 + * or empty string for larger codes. + */ + + if (msg == NULL) { + invalid++; + continue; + } + + len = nxt_strlen(msg); + size += len; + + if (len == 0 /* HP-UX empty strings. */ + || nxt_errno == NXT_EINVAL + || nxt_memcmp(msg, "Unknown error", 13) == 0) + { + invalid++; + continue; + } + +#if (NXT_AIX) + + if (nxt_memcmp(msg, "Error ", 6) == 0 + && nxt_memcmp(msg + len - 10, " occurred.", 9) == 0) + { + invalid++; + continue; + } + +#endif + } + + nxt_sys_nerr -= invalid; + + nxt_main_log_debug("sys_nerr: %d", nxt_sys_nerr); + + n = (nxt_sys_nerr + 1) * sizeof(nxt_str_t); + + nxt_sys_errlist = nxt_malloc(n + size); + if (nxt_sys_errlist == NULL) { + return NXT_ERROR; + } + + p = (u_char *) nxt_sys_errlist + n; + + for (err = 0; err < nxt_sys_nerr; err++) { + msg = strerror((int) err); + len = nxt_strlen(msg); + + nxt_sys_errlist[err].len = len; + nxt_sys_errlist[err].data = p; + + p = nxt_cpymem(p, msg, len); + } + + nxt_sys_errlist[err].len = 13; + nxt_sys_errlist[err].data = p; + nxt_memcpy(p, "Unknown error", 13); + + nxt_strerror = nxt_runtime_strerror; + + return NXT_OK; +} + + +static u_char * +nxt_bootstrap_strerror(nxt_err_t err, u_char *errstr, size_t size) +{ + return nxt_cpystrn(errstr, (u_char *) strerror(err), size); +} + + +static u_char * +nxt_runtime_strerror(nxt_err_t err, u_char *errstr, size_t size) +{ + nxt_str_t *msg; + nxt_uint_t n; + + n = nxt_min((nxt_uint_t) err, nxt_sys_nerr); + + msg = &nxt_sys_errlist[n]; + + size = nxt_min(size, msg->len); + + return nxt_cpymem(errstr, msg->data, size); +} diff --git a/src/nxt_errno.h b/src/nxt_errno.h new file mode 100644 index 00000000..b3d7105a --- /dev/null +++ b/src/nxt_errno.h @@ -0,0 +1,88 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_ERRNO_H_INCLUDED_ +#define _NXT_UNIX_ERRNO_H_INCLUDED_ + + +typedef int nxt_err_t; + + +#define NXT_EPERM EPERM +#define NXT_ENOENT ENOENT +#define NXT_ENOPATH ENOENT +#define NXT_ESRCH ESRCH +#define NXT_EINTR EINTR +#define NXT_ECHILD ECHILD +#define NXT_ENOMEM ENOMEM +#define NXT_EACCES EACCES +#define NXT_EBUSY EBUSY +#define NXT_EEXIST EEXIST +#define NXT_EXDEV EXDEV +#define NXT_ENOTDIR ENOTDIR +#define NXT_EISDIR EISDIR +#define NXT_EINVAL EINVAL +#define NXT_ENOSPC ENOSPC +#define NXT_EPIPE EPIPE +#define NXT_EINPROGRESS EINPROGRESS +#define NXT_EOPNOTSUPP EOPNOTSUPP +#define NXT_EADDRINUSE EADDRINUSE +#define NXT_ECONNABORTED ECONNABORTED +#define NXT_ECONNRESET ECONNRESET +#define NXT_ENOTCONN ENOTCONN +#define NXT_ETIMEDOUT ETIMEDOUT +#define NXT_ECONNREFUSED ECONNREFUSED +#define NXT_ENAMETOOLONG ENAMETOOLONG +#define NXT_ENETDOWN ENETDOWN +#define NXT_ENETUNREACH ENETUNREACH +#define NXT_EHOSTDOWN EHOSTDOWN +#define NXT_EHOSTUNREACH EHOSTUNREACH +#define NXT_ENOSYS ENOSYS +#define NXT_ECANCELED ECANCELED +#define NXT_EILSEQ EILSEQ +#define NXT_ETIME ETIME +#define NXT_ENOMOREFILES 0 + +#if (NXT_HPUX) +/* HP-UX uses EWOULDBLOCK instead of EAGAIN. */ +#define NXT_EAGAIN EWOULDBLOCK +#else +#define NXT_EAGAIN EAGAIN +#endif + + +#define NXT_OK 0 +#define NXT_ERROR (-1) +#define NXT_AGAIN (-2) +#define NXT_DECLINED (-3) +#define NXT_DONE (-4) + + +#define \ +nxt_errno \ + errno + +#define \ +nxt_socket_errno \ + errno + +#define \ +nxt_set_errno(err) \ + errno = err + +#define \ +nxt_set_socket_errno(err) \ + errno = err + + +nxt_int_t nxt_strerror_start(void); + + +typedef u_char *(*nxt_strerror_t)(nxt_err_t err, u_char *errstr, size_t size); +extern nxt_strerror_t nxt_strerror; + + +#endif /* _NXT_UNIX_ERRNO_H_INCLUDED_ */ diff --git a/src/nxt_event_conn.c b/src/nxt_event_conn.c new file mode 100644 index 00000000..a516bbac --- /dev/null +++ b/src/nxt_event_conn.c @@ -0,0 +1,234 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_event_conn_shutdown_socket(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_close_socket(nxt_thread_t *thr, void *obj, + void *data); + + +nxt_event_conn_io_t nxt_unix_event_conn_io = { + nxt_event_conn_io_connect, + nxt_event_conn_io_accept, + + nxt_event_conn_io_read, + nxt_event_conn_io_recvbuf, + nxt_event_conn_io_recv, + + nxt_event_conn_io_write, + nxt_event_conn_io_write_chunk, + +#if (NXT_HAVE_LINUX_SENDFILE) + nxt_linux_event_conn_io_sendfile, +#elif (NXT_HAVE_FREEBSD_SENDFILE) + nxt_freebsd_event_conn_io_sendfile, +#elif (NXT_HAVE_MACOSX_SENDFILE) + nxt_macosx_event_conn_io_sendfile, +#elif (NXT_HAVE_SOLARIS_SENDFILEV) + nxt_solaris_event_conn_io_sendfilev, +#elif (NXT_HAVE_AIX_SEND_FILE) + nxt_aix_event_conn_io_send_file, +#elif (NXT_HAVE_HPUX_SENDFILE) + nxt_hpux_event_conn_io_sendfile, +#else + nxt_event_conn_io_sendbuf, +#endif + + nxt_event_conn_io_writev, + nxt_event_conn_io_send, + + nxt_event_conn_io_shutdown, +}; + + +nxt_event_conn_t * +nxt_event_conn_create(nxt_mem_pool_t *mp, nxt_log_t *log) +{ + nxt_thread_t *thr; + nxt_event_conn_t *c; + static nxt_atomic_t ident = 1; + + c = nxt_mem_zalloc(mp, sizeof(nxt_event_conn_t)); + if (nxt_slow_path(c == NULL)) { + return NULL; + } + + c->mem_pool = mp; + + c->socket.fd = -1; + + c->socket.log = &c->log; + c->log = *log; + + /* The while loop skips possible uint32_t overflow. */ + + while (c->log.ident == 0) { + c->log.ident = (uint32_t) nxt_atomic_fetch_add(&ident, 1); + } + + thr = nxt_thread(); + thr->engine->connections++; + + c->io = thr->engine->event->io; + c->max_chunk = NXT_INT32_T_MAX; + c->sendfile = NXT_CONN_SENDFILE_UNSET; + + c->socket.read_work_queue = &thr->work_queue.main; + c->socket.write_work_queue = &thr->work_queue.main; + + nxt_event_conn_timer_init(&c->read_timer, c, c->socket.read_work_queue); + nxt_event_conn_timer_init(&c->write_timer, c, c->socket.write_work_queue); + + nxt_log_debug(&c->log, "event connections: %uD", thr->engine->connections); + + return c; +} + + +void +nxt_event_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) +{ + int ret; + socklen_t len; + struct linger linger; + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "event conn shutdown"); + + if (c->socket.timedout) { + /* + * A reset of timed out connection on close + * to release kernel memory associated with socket. + * This also causes sending TCP/IP RST to a peer. + */ + linger.l_onoff = 1; + linger.l_linger = 0; + len = sizeof(struct linger); + + ret = setsockopt(c->socket.fd, SOL_SOCKET, SO_LINGER, &linger, len); + + if (nxt_slow_path(ret != 0)) { + nxt_log_error(NXT_LOG_CRIT, thr->log, + "setsockopt(%d, SO_LINGER) failed %E", + c->socket.fd, nxt_socket_errno); + } + } + + c->write_state->close_handler(thr, c, data); +} + + +void +nxt_event_conn_close(nxt_thread_t *thr, nxt_event_conn_t *c) +{ + nxt_work_queue_t *wq; + nxt_work_handler_t handler; + + nxt_log_debug(thr->log, "event conn close fd:%d", c->socket.fd); + + nxt_thread_work_queue_drop(thr, c); + nxt_thread_work_queue_drop(thr, &c->read_timer); + nxt_thread_work_queue_drop(thr, &c->write_timer); + + nxt_event_timer_delete(thr->engine, &c->read_timer); + nxt_event_timer_delete(thr->engine, &c->write_timer); + + nxt_event_fd_close(thr->engine, &c->socket); + thr->engine->connections--; + + nxt_log_debug(thr->log, "event connections: %uD", thr->engine->connections); + + if (thr->engine->batch != 0) { + + if (c->socket.closed || c->socket.error != 0) { + wq = &thr->engine->close_work_queue; + handler = nxt_event_conn_close_socket; + + } else { + wq = &thr->engine->shutdown_work_queue; + handler = nxt_event_conn_shutdown_socket; + } + + nxt_thread_work_queue_add(thr, wq, handler, + (void *) (uintptr_t) c->socket.fd, NULL, + &nxt_main_log); + + } else { + nxt_socket_close(c->socket.fd); + } + + c->socket.fd = -1; +} + + +static void +nxt_event_conn_shutdown_socket(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_socket_t s; + + s = (nxt_socket_t) (uintptr_t) obj; + + nxt_socket_shutdown(s, SHUT_RDWR); + + nxt_thread_work_queue_add(thr, &thr->engine->close_work_queue, + nxt_event_conn_close_socket, + (void *) (uintptr_t) s, NULL, &nxt_main_log); +} + + +static void +nxt_event_conn_close_socket(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_socket_t s; + + s = (nxt_socket_t) (uintptr_t) obj; + + nxt_socket_close(s); +} + + +void +nxt_event_conn_timer(nxt_event_engine_t *engine, nxt_event_conn_t *c, + const nxt_event_conn_state_t *state, nxt_event_timer_t *tev) +{ + nxt_msec_t timer; + + if (state->timer_value != NULL) { + timer = state->timer_value(c, state->timer_data); + + if (timer != 0) { + tev->handler = state->timer_handler; + nxt_event_timer_add(engine, tev, timer); + } + } +} + + +void +nxt_event_conn_work_queue_set(nxt_event_conn_t *c, nxt_work_queue_t *wq) +{ +#if 0 + nxt_thread_t *thr; + nxt_work_queue_t *owq; + + thr = nxt_thread(); + owq = c->socket.work_queue; + + nxt_thread_work_queue_move(thr, owq, wq, c); + nxt_thread_work_queue_move(thr, owq, wq, &c->read_timer); + nxt_thread_work_queue_move(thr, owq, wq, &c->write_timer); +#endif + + c->read_work_queue = wq; + c->write_work_queue = wq; + c->read_timer.work_queue = wq; + c->write_timer.work_queue = wq; +} diff --git a/src/nxt_event_conn.h b/src/nxt_event_conn.h new file mode 100644 index 00000000..fe7d794a --- /dev/null +++ b/src/nxt_event_conn.h @@ -0,0 +1,382 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_EVENT_CONN_H_INCLUDED_ +#define _NXT_EVENT_CONN_H_INCLUDED_ + + +typedef nxt_msec_t (*nxt_event_conn_timer_val_t)(nxt_event_conn_t *c, + uintptr_t data); + + +#define NXT_EVENT_NO_BUF_PROCESS 0 +#define NXT_EVENT_BUF_PROCESS 1 +#define NXT_EVENT_BUF_COMPLETION 1 + +#define NXT_EVENT_TIMER_AUTORESET 1 +#define NXT_EVENT_TIMER_NO_AUTORESET 0 + + +typedef struct { + uint8_t process_buffers; + uint8_t autoreset_timer; + + nxt_work_handler_t ready_handler; + nxt_work_handler_t close_handler; + nxt_work_handler_t error_handler; + + nxt_work_handler_t timer_handler; + nxt_event_conn_timer_val_t timer_value; + uintptr_t timer_data; +} nxt_event_conn_state_t; + + +typedef struct { + double average; + size_t limit; + size_t limit_after; + size_t max_limit; + nxt_msec_t last; +} nxt_event_write_rate_t; + + +typedef struct { + + void (*connect)(nxt_thread_t *thr, void *obj, + void *data); + + void (*accept)(nxt_thread_t *thr, void *obj, + void *data); + + /* + * The read() with NULL c->read buffer waits readiness of a connection + * to avoid allocation of read buffer if the connection will time out + * or will be closed with error. The kqueue-specific read() can also + * detect case if a client did not sent anything and has just closed the + * connection without errors. In the latter case state's close_handler + * is called. + */ + void (*read)(nxt_thread_t *thr, void *obj, + void *data); + + ssize_t (*recvbuf)(nxt_event_conn_t *c, nxt_buf_t *b); + + ssize_t (*recv)(nxt_event_conn_t *c, void *buf, + size_t size, nxt_uint_t flags); + + /* + * The write() is an interface to write a buffer chain with a given rate + * limit. It calls write_chunk() in a cycle and handles write event timer. + */ + void (*write)(nxt_thread_t *thr, void *obj, + void *data); + + /* + * The write_chunk() interface writes a buffer chain with a given limit + * and toggles write event. SSL/TLS libraries' write_chunk() interface + * buffers data and calls the library specific send() interface to write + * the buffered data eventually. + */ + ssize_t (*write_chunk)(nxt_thread_t *thr, + nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + + /* + * The sendbuf() is an interface for OS-specific sendfile + * implementations or simple writev(). + */ + ssize_t (*sendbuf)(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + /* + * The writev() is an interface to write several nxt_iobuf_t buffers. + */ + ssize_t (*writev)(nxt_event_conn_t *c, + nxt_iobuf_t *iob, nxt_uint_t niob); + /* + * The send() is an interface to write a single buffer. SSL/TLS + * libraries' send() interface handles also the libraries' errors. + */ + ssize_t (*send)(nxt_event_conn_t *c, void *buf, + size_t size); + + void (*shutdown)(nxt_thread_t *thr, void *obj, + void *data); +} nxt_event_conn_io_t; + + +struct nxt_event_conn_s { + /* + * Must be the first field, since nxt_event_fd_t + * and nxt_event_conn_t are used interchangeably. + */ + nxt_event_fd_t socket; + + nxt_buf_t *read; + const nxt_event_conn_state_t *read_state; + nxt_work_queue_t *read_work_queue; + nxt_event_timer_t read_timer; + + nxt_buf_t *write; + const nxt_event_conn_state_t *write_state; + nxt_work_queue_t *write_work_queue; + nxt_event_write_rate_t *rate; + nxt_event_timer_t write_timer; + + nxt_off_t sent; + uint32_t max_chunk; + uint32_t nbytes; + + nxt_event_conn_io_t *io; + +#if (NXT_SSLTLS || NXT_THREADS) + /* SunC does not support "zero-sized struct/union". */ + + union { +#if (NXT_SSLTLS) + void *ssltls; +#endif +#if (NXT_THREADS) + nxt_thread_pool_t *thread_pool; +#endif + } u; + +#endif + + nxt_mem_pool_t *mem_pool; + + nxt_log_t log; + + nxt_listen_socket_t *listen; + nxt_sockaddr_t *remote; + nxt_sockaddr_t *local; + const char *action; + + uint8_t peek; + uint8_t blocked; /* 1 bit */ + uint8_t delayed; /* 1 bit */ + +#define NXT_CONN_SENDFILE_OFF 0 +#define NXT_CONN_SENDFILE_ON 1 +#define NXT_CONN_SENDFILE_UNSET 3 + + uint8_t sendfile; /* 2 bits */ + uint8_t tcp_nodelay; /* 1 bit */ + + nxt_queue_link_t link; +}; + + +/* + * The nxt_event_conn_listen_t is separated from nxt_listen_socket_t + * because nxt_listen_socket_t is one per process whilst each worker + * thread uses own nxt_event_conn_listen_t. + */ +typedef struct { + /* Must be the first field. */ + nxt_event_fd_t socket; + + uint32_t ready; + uint32_t batch; + + /* An accept() interface is cached to minimize memory accesses. */ + void (*accept)(nxt_thread_t *thr, void *obj, + void *data); + + nxt_listen_socket_t *listen; + + nxt_event_timer_t timer; + + nxt_queue_link_t link; +} nxt_event_conn_listen_t; + + +#define \ +nxt_event_conn_io_handle(thr, wq, handler, c, data) \ + do { \ + if (thr->engine->batch != 0) { \ + nxt_thread_work_queue_add(thr, wq, handler, c, data, thr->log); \ + \ + } else { \ + handler(thr, c, data); \ + } \ + } while (0) + + +#define \ +nxt_event_conn_timer_init(ev, c, wq) \ + do { \ + (ev)->work_queue = (wq); \ + (ev)->log = &(c)->log; \ + (ev)->precision = NXT_EVENT_TIMER_DEFAULT_PRECISION; \ + nxt_event_timer_ident((ev), (c)->socket.fd); \ + } while (0) + + +#define \ +nxt_event_read_timer_conn(ev) \ + nxt_event_timer_data(ev, nxt_event_conn_t, read_timer) + + +#define \ +nxt_event_write_timer_conn(ev) \ + nxt_event_timer_data(ev, nxt_event_conn_t, write_timer) + + +#if (NXT_HAVE_UNIX_DOMAIN) + +#define \ +nxt_event_conn_tcp_nodelay_on(c) \ + do { \ + nxt_int_t ret; \ + \ + if ((c)->remote->u.sockaddr.sa_family != AF_UNIX) { \ + ret = nxt_socket_setsockopt((c)->socket.fd, IPPROTO_TCP, \ + TCP_NODELAY, 1); \ + \ + (c)->tcp_nodelay = (ret == NXT_OK); \ + } \ + } while (0) + + +#else + +#define \ +nxt_event_conn_tcp_nodelay_on(c) \ + do { \ + nxt_int_t ret; \ + \ + ret = nxt_socket_setsockopt((c)->socket.fd, IPPROTO_TCP, \ + TCP_NODELAY, 1); \ + \ + (c)->tcp_nodelay = (ret == NXT_OK); \ + } while (0) + +#endif + + +NXT_EXPORT nxt_event_conn_t *nxt_event_conn_create(nxt_mem_pool_t *mp, + nxt_log_t *log); +void nxt_event_conn_io_shutdown(nxt_thread_t *thr, void *obj, + void *data); +NXT_EXPORT void nxt_event_conn_close(nxt_thread_t *thr, nxt_event_conn_t *c); + +NXT_EXPORT void nxt_event_conn_timer(nxt_event_engine_t *engine, + nxt_event_conn_t *c, const nxt_event_conn_state_t *state, + nxt_event_timer_t *tev); +NXT_EXPORT void nxt_event_conn_work_queue_set(nxt_event_conn_t *c, + nxt_work_queue_t *wq); + +NXT_EXPORT void nxt_event_conn_connect(nxt_thread_t *thr, nxt_event_conn_t *c); +void nxt_event_conn_batch_socket(nxt_thread_t *thr, void *obj, + void *data); +void nxt_event_conn_io_connect(nxt_thread_t *thr, void *obj, + void *data); +nxt_int_t nxt_event_conn_socket(nxt_thread_t *thr, + nxt_event_conn_t *c); +void nxt_event_conn_connect_test(nxt_thread_t *thr, void *obj, + void *data); +void nxt_event_conn_connect_error(nxt_thread_t *thr, void *obj, + void *data); + +NXT_EXPORT nxt_int_t nxt_event_conn_listen(nxt_thread_t *thr, + nxt_listen_socket_t *ls); +void nxt_event_conn_io_accept(nxt_thread_t *thr, void *obj, + void *data); +NXT_EXPORT void nxt_event_conn_accept(nxt_thread_t *thr, + nxt_event_conn_listen_t *cls, nxt_event_conn_t *c); +void nxt_event_conn_accept_error(nxt_thread_t *thr, + nxt_event_conn_listen_t *cls, const char *accept_syscall, nxt_err_t err); + +NXT_EXPORT void nxt_event_conn_read(nxt_thread_t *thr, nxt_event_conn_t *c); +void nxt_event_conn_io_read(nxt_thread_t *thr, void *obj, + void *data); +ssize_t nxt_event_conn_io_recvbuf(nxt_event_conn_t *c, nxt_buf_t *b); +ssize_t nxt_event_conn_io_recv(nxt_event_conn_t *c, void *buf, + size_t size, nxt_uint_t flags); + +NXT_EXPORT void nxt_event_conn_write(nxt_thread_t *thr, nxt_event_conn_t *c); +size_t nxt_event_conn_write_limit(nxt_event_conn_t *c); +nxt_bool_t nxt_event_conn_write_delayed(nxt_event_engine_t *engine, + nxt_event_conn_t *c, size_t sent); +void nxt_event_conn_io_write(nxt_thread_t *thr, void *obj, + void *data); +ssize_t nxt_event_conn_io_write_chunk(nxt_thread_t *thr, + nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); +ssize_t nxt_event_conn_io_writev(nxt_event_conn_t *c, + nxt_iobuf_t *iob, nxt_uint_t niob); +ssize_t nxt_event_conn_io_send(nxt_event_conn_t *c, void *buf, + size_t size); + +NXT_EXPORT void nxt_event_conn_job_sendfile(nxt_thread_t *thr, + nxt_event_conn_t *c); + + +#define \ +nxt_event_conn_connect_enqueue(thr, c) \ + nxt_thread_work_queue_add(thr, &thr->engine->socket_work_queue, \ + nxt_event_conn_batch_socket, \ + c, c->socket.data, c->socket.log) + + +#define \ +nxt_event_conn_read_enqueue(thr, c) \ + do { \ + c->socket.read_work_queue = &thr->engine->read_work_queue; \ + \ + nxt_thread_work_queue_add(thr, &thr->engine->read_work_queue, \ + c->io->read, c, c->socket.data, \ + c->socket.log); \ + } while (0) + + +#define \ +nxt_event_conn_write_enqueue(thr, c) \ + do { \ + c->socket.write_work_queue = &thr->engine->write_work_queue; \ + \ + nxt_thread_work_queue_add(thr, &thr->engine->write_work_queue, \ + c->io->write, c, c->socket.data, \ + c->socket.log); \ + } while (0) + + +extern nxt_event_conn_io_t nxt_unix_event_conn_io; + + +typedef struct { + /* + * Client and peer connections are not embedded because already + * existent connections can be switched to the event connection proxy. + */ + nxt_event_conn_t *client; + nxt_event_conn_t *peer; + nxt_buf_t *client_buffer; + nxt_buf_t *peer_buffer; + + size_t client_buffer_size; + size_t peer_buffer_size; + + nxt_msec_t client_wait_timeout; + nxt_msec_t connect_timeout; + nxt_msec_t reconnect_timeout; + nxt_msec_t peer_wait_timeout; + nxt_msec_t client_write_timeout; + nxt_msec_t peer_write_timeout; + + uint8_t connected; /* 1 bit */ + uint8_t delayed; /* 1 bit */ + uint8_t retries; /* 8 bits */ + + nxt_work_handler_t completion_handler; +} nxt_event_conn_proxy_t; + + +NXT_EXPORT nxt_event_conn_proxy_t *nxt_event_conn_proxy_create( + nxt_event_conn_t *c); +NXT_EXPORT void nxt_event_conn_proxy(nxt_event_conn_proxy_t *p); + + +#endif /* _NXT_EVENT_CONN_H_INCLUDED_ */ diff --git a/src/nxt_event_conn_accept.c b/src/nxt_event_conn_accept.c new file mode 100644 index 00000000..ef0eea2c --- /dev/null +++ b/src/nxt_event_conn_accept.c @@ -0,0 +1,367 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * A listen socket handler calls an event facility specific io_accept() + * method. The method accept()s a new connection and then calls + * nxt_event_conn_accept() to handle the new connection and to prepare + * for a next connection to avoid just dropping next accept()ed socket + * if no more connections allowed. If there are no available connections + * an idle connection would be closed. If there are no idle connections + * then new connections will not be accept()ed for 1 second. + */ + + +static nxt_event_conn_t *nxt_event_conn_accept_alloc(nxt_thread_t *thr, + nxt_event_conn_listen_t *cls); +static void nxt_event_conn_listen_handler(nxt_thread_t *thr, void *obj, + void *data); +static nxt_event_conn_t *nxt_event_conn_accept_next(nxt_thread_t *thr, + nxt_event_conn_listen_t *cls); +static nxt_int_t nxt_event_conn_accept_close_idle(nxt_thread_t *thr, + nxt_event_conn_listen_t *cls); +static void nxt_event_conn_listen_event_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_listen_timer_handler(nxt_thread_t *thr, void *obj, + void *data); + + +nxt_int_t +nxt_event_conn_listen(nxt_thread_t *thr, nxt_listen_socket_t *ls) +{ + nxt_event_conn_listen_t *cls; + + cls = nxt_zalloc(sizeof(nxt_event_conn_listen_t)); + + if (nxt_fast_path(cls != NULL)) { + cls->socket.fd = ls->socket; + + cls->batch = thr->engine->batch; + + if (cls->batch != 0) { + cls->socket.read_work_queue = &thr->engine->accept_work_queue; + + } else { + cls->socket.read_work_queue = &thr->work_queue.main; + cls->batch = 1; + } + + cls->socket.read_handler = nxt_event_conn_listen_handler; + cls->socket.error_handler = nxt_event_conn_listen_event_error; + cls->socket.log = &nxt_main_log; + + cls->accept = thr->engine->event->io->accept; + + cls->listen = ls; + + cls->timer.work_queue = &thr->work_queue.main; + cls->timer.handler = nxt_event_conn_listen_timer_handler; + cls->timer.log = &nxt_main_log; + + nxt_event_timer_ident(&cls->timer, cls->socket.fd); + + if (nxt_event_conn_accept_alloc(thr, cls) != NULL) { + nxt_event_fd_enable_accept(thr->engine, &cls->socket); + + nxt_queue_insert_head(&thr->engine->listen_connections, &cls->link); + } + + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_event_conn_t * +nxt_event_conn_accept_alloc(nxt_thread_t *thr, nxt_event_conn_listen_t *cls) +{ + nxt_sockaddr_t *sa, *remote; + nxt_mem_pool_t *mp; + nxt_event_conn_t *c; + nxt_listen_socket_t *ls; + + if (thr->engine->connections < thr->engine->max_connections) { + + mp = nxt_mem_pool_create(cls->listen->mem_pool_size); + + if (nxt_fast_path(mp != NULL)) { + /* This allocation cannot fail. */ + c = nxt_event_conn_create(mp, cls->socket.log); + + cls->socket.data = c; + c->socket.read_work_queue = cls->socket.read_work_queue; + c->socket.write_ready = 1; + + ls = cls->listen; + c->listen = ls; + + /* This allocation cannot fail. */ + remote = nxt_sockaddr_alloc(mp, ls->socklen); + c->remote = remote; + + sa = ls->sockaddr; + remote->type = sa->type; + /* + * Set address family for unspecified Unix domain, + * because these sockaddr's are not be passed to accept(). + */ + remote->u.sockaddr.sa_family = sa->u.sockaddr.sa_family; + + return c; + } + } + + return NULL; +} + + +static void +nxt_event_conn_listen_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_listen_t *cls; + + cls = obj; + cls->ready = cls->batch; + + cls->accept(thr, cls, data); +} + + +void +nxt_event_conn_io_accept(nxt_thread_t *thr, void *obj, void *data) +{ + socklen_t len; + nxt_socket_t s; + struct sockaddr *sa; + nxt_event_conn_t *c; + nxt_event_conn_listen_t *cls; + + cls = obj; + c = data; + + cls->ready--; + cls->socket.read_ready = (cls->ready != 0); + + len = nxt_socklen(c->remote); + + if (len >= sizeof(struct sockaddr)) { + sa = &c->remote->u.sockaddr; + + } else { + sa = NULL; + len = 0; + } + + s = accept(cls->socket.fd, sa, &len); + + if (s == -1) { + nxt_event_conn_accept_error(thr, cls, "accept", nxt_socket_errno); + return; + } + + c->socket.fd = s; + +#if (NXT_LINUX) + /* + * Linux does not inherit non-blocking mode + * from listen socket for accept()ed socket. + */ + if (nxt_slow_path(nxt_socket_nonblocking(s) != NXT_OK)) { + nxt_socket_close(s); + } + +#endif + + nxt_log_debug(thr->log, "accept(%d): %d", cls->socket.fd, s); + + nxt_event_conn_accept(thr, cls, c); +} + + +void +nxt_event_conn_accept(nxt_thread_t *thr, nxt_event_conn_listen_t *cls, + nxt_event_conn_t *c) +{ + nxt_event_conn_t *next; + + nxt_event_timer_ident(&c->read_timer, c->socket.fd); + nxt_event_timer_ident(&c->write_timer, c->socket.fd); + + /* This allocation cannot fail. */ + (void) nxt_sockaddr_text(c->mem_pool, c->remote, 0); + + nxt_log_debug(c->socket.log, "client: %*s", + c->remote->text_len, c->remote->text); + + nxt_queue_insert_head(&thr->engine->idle_connections, &c->link); + + c->read_work_queue = c->listen->work_queue; + c->write_work_queue = c->listen->work_queue; + + if (c->listen->read_after_accept) { + + //c->socket.read_ready = 1; + thr->log = c->socket.log; + c->listen->handler(thr, c, NULL); + thr->log = cls->socket.log; + + } else { + nxt_thread_work_queue_add(thr, c->write_work_queue, + c->listen->handler, c, NULL, c->socket.log); + } + + next = nxt_event_conn_accept_next(thr, cls); + + if (next != NULL && cls->socket.read_ready) { + nxt_thread_work_queue_add(thr, cls->socket.read_work_queue, + cls->accept, cls, next, cls->socket.log); + } +} + + +static nxt_event_conn_t * +nxt_event_conn_accept_next(nxt_thread_t *thr, nxt_event_conn_listen_t *cls) +{ + nxt_event_conn_t *c; + + cls->socket.data = NULL; + + do { + c = nxt_event_conn_accept_alloc(thr, cls); + + if (nxt_fast_path(c != NULL)) { + return c; + } + + } while (nxt_event_conn_accept_close_idle(thr, cls) == NXT_OK); + + nxt_log_alert(cls->socket.log, "no available connections, " + "new connections are not accepted within 1s"); + + return NULL; +} + + +static nxt_int_t +nxt_event_conn_accept_close_idle(nxt_thread_t *thr, + nxt_event_conn_listen_t *cls) +{ + nxt_queue_t *idle; + nxt_queue_link_t *link; + nxt_event_conn_t *c; + + static nxt_log_moderation_t nxt_idle_close_log_moderation = { + NXT_LOG_INFO, 2, "idle connections closed", NXT_LOG_MODERATION + }; + + idle = &thr->engine->idle_connections; + + for (link = nxt_queue_last(idle); + link != nxt_queue_head(idle); + link = nxt_queue_next(link)) + { + c = nxt_queue_link_data(link, nxt_event_conn_t, link); + + if (!c->socket.read_ready) { + nxt_log_moderate(&nxt_idle_close_log_moderation, NXT_LOG_INFO, + thr->log, "no available connections, " + "close idle connection"); + nxt_queue_remove(link); + nxt_event_conn_close(thr, c); + + return NXT_OK; + } + } + + nxt_event_timer_add(thr->engine, &cls->timer, 1000); + nxt_event_fd_disable_read(thr->engine, &cls->socket); + + return NXT_DECLINED; +} + + +void +nxt_event_conn_accept_error(nxt_thread_t *thr, nxt_event_conn_listen_t *cls, + const char *accept_syscall, nxt_err_t err) +{ + static nxt_log_moderation_t nxt_accept_log_moderation = { + NXT_LOG_INFO, 2, "accept() failed", NXT_LOG_MODERATION + }; + + cls->socket.read_ready = 0; + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(thr->log, "%s(%d) %E", + accept_syscall, cls->socket.fd, err); + return; + + case ECONNABORTED: + nxt_log_moderate(&nxt_accept_log_moderation, NXT_LOG_INFO, + thr->log, "%s(%d) failed %E", + accept_syscall, cls->socket.fd, err); + return; + + case EMFILE: + case ENFILE: + case ENOBUFS: + case ENOMEM: + if (nxt_event_conn_accept_close_idle(thr, cls) != NXT_OK) { + nxt_log_alert(thr->log, "%s(%d) failed %E, " + "new connections are not accepted within 1s", + accept_syscall, cls->socket.fd, err); + } + + return; + + default: + nxt_log_alert(thr->log, "%s(%d) failed %E", + accept_syscall, cls->socket.fd, err); + return; + } +} + + +static void +nxt_event_conn_listen_timer_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_event_timer_t *ev; + nxt_event_conn_listen_t *cls; + + ev = obj; + + cls = nxt_event_timer_data(ev, nxt_event_conn_listen_t, timer); + c = cls->socket.data; + + if (c == NULL) { + c = nxt_event_conn_accept_next(thr, cls); + + if (c == NULL) { + return; + } + } + + nxt_event_fd_enable_accept(thr->engine, &cls->socket); + + cls->accept(thr, cls, c); +} + + +static void +nxt_event_conn_listen_event_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_fd_t *ev; + + ev = obj; + + nxt_log_alert(thr->log, "accept(%d) event error", ev->fd); +} diff --git a/src/nxt_event_conn_connect.c b/src/nxt_event_conn_connect.c new file mode 100644 index 00000000..f614dcda --- /dev/null +++ b/src/nxt_event_conn_connect.c @@ -0,0 +1,213 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +void +nxt_event_conn_connect(nxt_thread_t *thr, nxt_event_conn_t *c) +{ + void *data; + + data = c->socket.data; + + if (thr->engine->batch != 0) { + nxt_thread_work_queue_add(thr, &thr->engine->socket_work_queue, + nxt_event_conn_batch_socket, c, data, + c->socket.log); + return; + } + + if (nxt_event_conn_socket(thr, c) == NXT_OK) { + c->io->connect(thr, c, data); + return; + } + + c->write_state->error_handler(thr, c, data); +} + + +void +nxt_event_conn_batch_socket(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_work_handler_t handler; + + c = obj; + + if (nxt_event_conn_socket(thr, c) == NXT_OK) { + c->socket.write_work_queue = c->write_work_queue; + handler = c->io->connect; + + } else { + handler = c->write_state->error_handler; + } + + nxt_thread_work_queue_add(thr, &thr->engine->connect_work_queue, + handler, c, data, thr->log); +} + + +void +nxt_event_conn_io_connect(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_work_handler_t handler; + const nxt_event_conn_state_t *state; + + c = obj; + + state = c->write_state; + + switch (nxt_socket_connect(c->socket.fd, c->remote)) { + + case NXT_OK: + c->socket.write_ready = 1; + handler = state->ready_handler; + break; + + case NXT_AGAIN: + c->socket.write_handler = nxt_event_conn_connect_test; + c->socket.error_handler = state->error_handler; + + nxt_event_conn_timer(thr->engine, c, state, &c->write_timer); + + nxt_event_fd_enable_write(thr->engine, &c->socket); + return; + + case NXT_DECLINED: + handler = state->close_handler; + break; + + default: /* NXT_ERROR */ + handler = state->error_handler; + break; + } + + nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); +} + + +nxt_int_t +nxt_event_conn_socket(nxt_thread_t *thr, nxt_event_conn_t *c) +{ + nxt_uint_t family; + nxt_socket_t s; + + nxt_log_debug(thr->log, "event conn socket"); + + family = c->remote->u.sockaddr.sa_family; + + s = nxt_socket_create(family, c->remote->type, 0, NXT_NONBLOCK); + + if (nxt_slow_path(s == -1)) { + return NXT_ERROR; + } + + c->sendfile = 1; + +#if (NXT_HAVE_UNIX_DOMAIN && NXT_SOLARIS) + + if (family == AF_UNIX) { + /* Solaris AF_UNIX does not support sendfilev(). */ + c->sendfile = 0; + } + +#endif + + c->socket.fd = s; + nxt_event_timer_ident(&c->read_timer, s); + nxt_event_timer_ident(&c->write_timer, s); + + if (c->local != NULL) { + if (nxt_slow_path(nxt_socket_bind(s, c->local, 0) != NXT_OK)) { + nxt_socket_close(s); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +void +nxt_event_conn_connect_test(nxt_thread_t *thr, void *obj, void *data) +{ + int ret, err; + socklen_t len; + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "event connect test fd:%d", c->socket.fd); + + nxt_event_fd_block_write(thr->engine, &c->socket); + + if (c->write_state->autoreset_timer) { + nxt_event_timer_disable(&c->write_timer); + } + + err = 0; + len = sizeof(int); + + /* + * Linux and BSDs return 0 and store a pending error in the err argument; + * Solaris returns -1 and sets the errno. + */ + + ret = getsockopt(c->socket.fd, SOL_SOCKET, SO_ERROR, (void *) &err, &len); + + if (nxt_slow_path(ret == -1)) { + err = nxt_errno; + } + + if (err == 0) { + nxt_event_conn_io_handle(thr, c->write_work_queue, + c->write_state->ready_handler, c, data); + return; + } + + c->socket.error = err; + + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), thr->log, + "connect(%d, %*s) failed %E", + c->socket.fd, c->remote->text_len, c->remote->text, err); + + nxt_event_conn_connect_error(thr, c, data); +} + + +void +nxt_event_conn_connect_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_work_handler_t handler; + const nxt_event_conn_state_t *state; + + c = obj; + + state = c->write_state; + + switch (c->socket.error) { + + case NXT_ECONNREFUSED: +#if (NXT_LINUX) + case NXT_EAGAIN: + /* + * Linux returns EAGAIN instead of ECONNREFUSED + * for UNIX sockets if a listen queue is full. + */ +#endif + handler = state->close_handler; + break; + + default: + handler = state->error_handler; + break; + } + + nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); +} diff --git a/src/nxt_event_conn_job_sendfile.c b/src/nxt_event_conn_job_sendfile.c new file mode 100644 index 00000000..140febab --- /dev/null +++ b/src/nxt_event_conn_job_sendfile.c @@ -0,0 +1,268 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#if (NXT_THREADS) + +typedef struct { + nxt_job_t job; + nxt_buf_t *out; + size_t sent; + size_t limit; + nxt_work_handler_t ready_handler; +} nxt_job_sendfile_t; + + +static void nxt_event_conn_job_sendfile_start(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_job_sendfile_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_job_sendfile_return(nxt_thread_t *thr, void *obj, + void *data); +static nxt_buf_t *nxt_event_conn_job_sendfile_completion(nxt_thread_t *thr, + nxt_event_conn_t *c, nxt_buf_t *b); + + +void +nxt_event_conn_job_sendfile(nxt_thread_t *thr, nxt_event_conn_t *c) +{ + nxt_event_fd_disable(thr->engine, &c->socket); + + /* A work item data is not used in nxt_event_conn_job_sendfile_start(). */ + nxt_event_conn_job_sendfile_start(thr, c, NULL); +} + + +static void +nxt_event_conn_job_sendfile_start(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_iobuf_t b; + nxt_event_conn_t *c; + nxt_job_sendfile_t *jbs; + nxt_sendbuf_coalesce_t sb; + + c = obj; + + nxt_log_debug(thr->log, "event conn sendfile fd:%d", c->socket.fd); + + jbs = nxt_job_create(c->mem_pool, sizeof(nxt_job_sendfile_t)); + + if (nxt_slow_path(jbs == NULL)) { + c->write_state->error_handler(thr, c, NULL); + return; + } + + c->socket.write_handler = nxt_event_conn_job_sendfile_start; + c->socket.error_handler = c->write_state->error_handler; + + jbs->job.data = c; + nxt_job_set_name(&jbs->job, "job sendfile"); + + jbs->limit = nxt_event_conn_write_limit(c); + + if (jbs->limit != 0) { + + sb.buf = c->write; + sb.iobuf = &b; + sb.nmax = 1; + sb.sync = 0; + sb.size = 0; + sb.limit = jbs->limit; + + if (nxt_sendbuf_mem_coalesce(&sb) != 0 || !sb.sync) { + + jbs->job.thread_pool = c->u.thread_pool; + jbs->job.log = c->socket.log; + jbs->out = c->write; + c->write = NULL; + jbs->ready_handler = nxt_event_conn_job_sendfile_return; + + c->blocked = 1; + + if (c->write_timer.state != NXT_EVENT_TIMER_DISABLED) { + c->write_timer.state = NXT_EVENT_TIMER_BLOCKED; + } + + nxt_job_start(thr, &jbs->job, nxt_event_conn_job_sendfile_handler); + return; + } + } + + nxt_event_conn_job_sendfile_return(thr, jbs, c); +} + + +static void +nxt_event_conn_job_sendfile_handler(nxt_thread_t *thr, void *obj, void *data) +{ + ssize_t ret; + nxt_buf_t *b; + nxt_bool_t first; + nxt_event_conn_t *c; + nxt_job_sendfile_t *jbs; + + jbs = obj; + c = data; + + nxt_log_debug(thr->log, "event conn job sendfile fd:%d", c->socket.fd); + + first = c->socket.write_ready; + b = jbs->out; + + do { + ret = c->io->sendbuf(c, b, jbs->limit); + + if (ret == NXT_AGAIN) { + break; + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + goto done; + } + + jbs->sent += ret; + jbs->limit -= ret; + + b = nxt_sendbuf_update(b, ret); + + if (b == NULL) { + goto done; + } + + if (jbs->limit == 0) { + + if (c->rate == NULL) { + jbs->limit = c->max_chunk; + goto fast; + } + + goto done; + } + + } while (c->socket.write_ready); + + if (first && thr->thread_pool->work_queue.head != NULL) { + goto fast; + } + +done: + + nxt_job_return(thr, &jbs->job, jbs->ready_handler); + return; + +fast: + + nxt_thread_pool_post(thr->thread_pool, nxt_event_conn_job_sendfile_handler, + jbs, c, thr->log); +} + + +static void +nxt_event_conn_job_sendfile_return(nxt_thread_t *thr, void *obj, void *data) +{ + size_t sent; + nxt_buf_t *b; + nxt_bool_t done; + nxt_event_conn_t *c; + nxt_job_sendfile_t *jbs; + + jbs = obj; + c = data; + + c->blocked = 0; + + sent = jbs->sent; + c->sent += sent; + + nxt_log_debug(thr->log, "event conn sendfile sent:%z", sent); + + b = jbs->out; + + /* The job must be destroyed before connection error handler. */ + nxt_job_destroy(jbs); + + if (c->write_state->process_buffers) { + b = nxt_event_conn_job_sendfile_completion(thr, c, b); + + done = (b == NULL); + + /* Add data which might be added after sendfile job has started. */ + nxt_buf_chain_add(&b, c->write); + c->write = b; + + if (done) { + /* All data has been sent. */ + + if (b != NULL) { + /* But new data has been added. */ + nxt_event_conn_job_sendfile_start(thr, c, NULL); + } + + return; + } + } + + if (sent != 0 && c->write_state->autoreset_timer) { + nxt_event_timer_disable(&c->write_timer); + + } else if (c->write_timer.state == NXT_EVENT_TIMER_BLOCKED) { + c->write_timer.state = NXT_EVENT_TIMER_ACTIVE; + } + + if (c->socket.error == 0 + && !nxt_event_conn_write_delayed(thr->engine, c, sent)) + { + nxt_event_conn_timer(thr->engine, c, c->write_state, &c->write_timer); + + nxt_event_fd_oneshot_write(thr->engine, &c->socket); + } + + if (sent != 0) { + nxt_event_conn_io_handle(thr, c->write_work_queue, + c->write_state->ready_handler, + c, c->socket.data); + /* + * Fall through if first operations were + * successful but the last one failed. + */ + } + + if (nxt_slow_path(c->socket.error != 0)) { + nxt_event_conn_io_handle(thr, c->write_work_queue, + c->write_state->error_handler, + c, c->socket.data); + } +} + + +static nxt_buf_t * +nxt_event_conn_job_sendfile_completion(nxt_thread_t *thr, nxt_event_conn_t *c, + nxt_buf_t *b) +{ + while (b != NULL) { + + nxt_prefetch(b->next); + + if (nxt_buf_is_mem(b) && b->mem.pos != b->mem.free) { + break; + + } else if (nxt_buf_is_file(b) && b->file_pos != b->file_end) { + break; + } + + nxt_thread_work_queue_add(thr, c->write_work_queue, + b->completion_handler, + b, b->parent, thr->log); + + b = b->next; + } + + return b; +} + +#endif diff --git a/src/nxt_event_conn_proxy.c b/src/nxt_event_conn_proxy.c new file mode 100644 index 00000000..0b9afe95 --- /dev/null +++ b/src/nxt_event_conn_proxy.c @@ -0,0 +1,1034 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_event_conn_proxy_client_buffer_alloc(nxt_thread_t *thr, + void *obj, void *data); +static void nxt_event_conn_proxy_peer_connect(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_connected(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_peer_read(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_client_read_ready(nxt_thread_t *thr, + void *obj, void *data); +static void nxt_event_conn_proxy_peer_read_ready(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_read_process(nxt_thread_t *thr, + nxt_event_conn_proxy_t *p, nxt_event_conn_t *source, + nxt_event_conn_t *sink); +static void nxt_event_conn_proxy_write_add(nxt_event_conn_t *c, nxt_buf_t *b); +static void nxt_event_conn_proxy_read(nxt_thread_t *thr, void *obj, void *data); +static void nxt_event_conn_proxy_client_write_ready(nxt_thread_t *thr, + void *obj, void *data); +static void nxt_event_conn_proxy_peer_write_ready(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_write_process(nxt_thread_t *thr, + nxt_event_conn_proxy_t *p, nxt_event_conn_t *sink, + nxt_event_conn_t *source); +static void nxt_event_conn_proxy_read_add(nxt_event_conn_t *c, nxt_buf_t *b); +static void nxt_event_conn_proxy_close(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_read_timeout(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_write_timeout(nxt_thread_t *thr, void *obj, + void *data); +static nxt_msec_t nxt_event_conn_proxy_timeout_value(nxt_event_conn_t *c, + uintptr_t data); +static void nxt_event_conn_proxy_refused(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_reconnect_handler(nxt_thread_t *thr, + void *obj, void *data); +static void nxt_event_conn_proxy_shutdown(nxt_thread_t *thr, + nxt_event_conn_proxy_t *p, nxt_event_conn_t *source, + nxt_event_conn_t *sink); +static void nxt_event_conn_proxy_read_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_write_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_conn_proxy_complete(nxt_thread_t *thr, + nxt_event_conn_proxy_t *p); + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_client_wait_state; +static const nxt_event_conn_state_t + nxt_event_conn_proxy_client_first_read_state; +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_connect_state; +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_wait_state; +static const nxt_event_conn_state_t nxt_event_conn_proxy_client_read_state; +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_read_state; +static const nxt_event_conn_state_t nxt_event_conn_proxy_client_write_state; +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_write_state; + + +nxt_event_conn_proxy_t * +nxt_event_conn_proxy_create(nxt_event_conn_t *client) +{ + nxt_thread_t *thr; + nxt_event_conn_t *peer; + nxt_event_conn_proxy_t *p; + + p = nxt_mem_zalloc(client->mem_pool, sizeof(nxt_event_conn_proxy_t)); + if (nxt_slow_path(p == NULL)) { + return NULL; + } + + peer = nxt_event_conn_create(client->mem_pool, client->socket.log); + if (nxt_slow_path(peer == NULL)) { + return NULL; + } + + thr = nxt_thread(); + + client->socket.read_work_queue = &thr->engine->read_work_queue; + client->socket.write_work_queue = &thr->engine->write_work_queue; + peer->socket.read_work_queue = &thr->engine->read_work_queue; + peer->socket.write_work_queue = &thr->engine->write_work_queue; + + peer->socket.data = client->socket.data; + + peer->read_work_queue = client->read_work_queue; + peer->write_work_queue = client->write_work_queue; + peer->read_timer.work_queue = client->read_work_queue; + peer->write_timer.work_queue = client->write_work_queue; + + p->client = client; + p->peer = peer; + + return p; +} + + +void +nxt_event_conn_proxy(nxt_event_conn_proxy_t *p) +{ + nxt_thread_t *thr; + nxt_event_conn_t *peer; + + thr = nxt_thread(); + + /* + * Peer read event: not connected, disabled. + * Peer write event: not connected, disabled. + */ + + if (p->client_wait_timeout == 0) { + /* + * Peer write event: waiting for connection + * to be established with connect_timeout. + */ + peer = p->peer; + peer->write_state = &nxt_event_conn_proxy_peer_connect_state; + + nxt_event_conn_connect_enqueue(thr, peer); + } + + /* + * Client read event: waiting for client data with + * client_wait_timeout before buffer allocation. + */ + p->client->read_state = &nxt_event_conn_proxy_client_wait_state; + + nxt_event_conn_read(thr, p->client); +} + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_client_wait_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_NO_AUTORESET, + + nxt_event_conn_proxy_client_buffer_alloc, + nxt_event_conn_proxy_close, + nxt_event_conn_proxy_error, + + nxt_event_conn_proxy_read_timeout, + nxt_event_conn_proxy_timeout_value, + offsetof(nxt_event_conn_proxy_t, client_wait_timeout), +}; + + +static void +nxt_event_conn_proxy_client_buffer_alloc(nxt_thread_t *thr, void *obj, + void *data) +{ + nxt_buf_t *b; + nxt_event_conn_t *client; + nxt_event_conn_proxy_t *p; + + client = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy client first read fd:%d", + client->socket.fd); + + b = nxt_buf_mem_alloc(client->mem_pool, p->client_buffer_size, + NXT_MEM_BUF_CUTBACK | NXT_MEM_BUF_USABLE); + + if (nxt_slow_path(b == NULL)) { + /* An error completion. */ + nxt_event_conn_proxy_complete(thr, p); + return; + } + + p->client_buffer = b; + client->read = b; + + if (p->peer->socket.fd != -1) { + /* + * Client read event: waiting, no timeout. + * Client write event: blocked. + * Peer read event: disabled. + * Peer write event: waiting for connection to be established + * or blocked after the connection has established. + */ + client->read_state = &nxt_event_conn_proxy_client_read_state; + + } else { + /* + * Client read event: waiting for data with client_wait_timeout + * before connecting to a peer. + * Client write event: blocked. + * Peer read event: not connected, disabled. + * Peer write event: not connected, disabled. + */ + client->read_state = &nxt_event_conn_proxy_client_first_read_state; + } + + nxt_event_conn_read(thr, client); +} + + +static const nxt_event_conn_state_t + nxt_event_conn_proxy_client_first_read_state nxt_aligned(64) = +{ + NXT_EVENT_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_event_conn_proxy_peer_connect, + nxt_event_conn_proxy_close, + nxt_event_conn_proxy_error, + + nxt_event_conn_proxy_read_timeout, + nxt_event_conn_proxy_timeout_value, + offsetof(nxt_event_conn_proxy_t, client_wait_timeout), +}; + + +static void +nxt_event_conn_proxy_peer_connect(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *client; + nxt_event_conn_proxy_t *p; + + client = obj; + p = data; + + /* + * Client read event: waiting, no timeout. + * Client write event: blocked. + * Peer read event: disabled. + * Peer write event: waiting for connection to be established + * with connect_timeout. + */ + client->read_state = &nxt_event_conn_proxy_client_read_state; + + p->peer->write_state = &nxt_event_conn_proxy_peer_connect_state; + + nxt_event_conn_connect(thr, p->peer); +} + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_connect_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_event_conn_proxy_connected, + nxt_event_conn_proxy_refused, + nxt_event_conn_proxy_error, + + nxt_event_conn_proxy_write_timeout, + nxt_event_conn_proxy_timeout_value, + offsetof(nxt_event_conn_proxy_t, connect_timeout), +}; + + +static void +nxt_event_conn_proxy_connected(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *client, *peer; + nxt_event_conn_proxy_t *p; + + peer = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy connected fd:%d", + peer->socket.fd); + + p->connected = 1; + + nxt_event_conn_tcp_nodelay_on(peer); + nxt_event_conn_tcp_nodelay_on(p->client); + + /* Peer read event: waiting with peer_wait_timeout. */ + + peer->read_state = &nxt_event_conn_proxy_peer_wait_state; + peer->write_state = &nxt_event_conn_proxy_peer_write_state; + + nxt_event_conn_read_enqueue(thr, peer); + + if (p->client_buffer != NULL) { + client = p->client; + + client->read_state = &nxt_event_conn_proxy_client_read_state; + client->write_state = &nxt_event_conn_proxy_client_write_state; + /* + * Send a client read data to the connected peer. + * Client write event: blocked. + */ + nxt_event_conn_proxy_read_process(thr, p, client, peer); + } +} + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_wait_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_NO_AUTORESET, + + nxt_event_conn_proxy_peer_read, + nxt_event_conn_proxy_close, + nxt_event_conn_proxy_error, + + nxt_event_conn_proxy_read_timeout, + nxt_event_conn_proxy_timeout_value, + offsetof(nxt_event_conn_proxy_t, peer_wait_timeout), +}; + + +static void +nxt_event_conn_proxy_peer_read(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_event_conn_t *peer; + nxt_event_conn_proxy_t *p; + + peer = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy peer read fd:%d", + peer->socket.fd); + + b = nxt_buf_mem_alloc(peer->mem_pool, p->peer_buffer_size, + NXT_MEM_BUF_CUTBACK | NXT_MEM_BUF_USABLE); + + if (nxt_slow_path(b == NULL)) { + /* An error completion. */ + nxt_event_conn_proxy_complete(thr, p); + return; + } + + p->peer_buffer = b; + peer->read = b; + + p->client->write_state = &nxt_event_conn_proxy_client_write_state; + peer->read_state = &nxt_event_conn_proxy_peer_read_state; + peer->write_state = &nxt_event_conn_proxy_peer_write_state; + + /* + * Client read event: waiting, no timeout. + * Client write event: blocked. + * Peer read event: waiting with possible peer_wait_timeout. + * Peer write event: blocked. + */ + nxt_event_conn_read(thr, peer); +} + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_client_read_state + nxt_aligned(64) = +{ + NXT_EVENT_BUF_PROCESS, + NXT_EVENT_TIMER_NO_AUTORESET, + + nxt_event_conn_proxy_client_read_ready, + nxt_event_conn_proxy_close, + nxt_event_conn_proxy_read_error, + + NULL, + NULL, + 0, +}; + + +static void +nxt_event_conn_proxy_client_read_ready(nxt_thread_t *thr, void *obj, + void *data) +{ + nxt_event_conn_t *client; + nxt_event_conn_proxy_t *p; + + client = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy client read ready fd:%d", + client->socket.fd); + + nxt_event_conn_proxy_read_process(thr, p, client, p->peer); +} + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_read_state + nxt_aligned(64) = +{ + NXT_EVENT_BUF_PROCESS, + NXT_EVENT_TIMER_NO_AUTORESET, + + nxt_event_conn_proxy_peer_read_ready, + nxt_event_conn_proxy_close, + nxt_event_conn_proxy_read_error, + + NULL, + NULL, + 0, +}; + + +static void +nxt_event_conn_proxy_peer_read_ready(nxt_thread_t *thr, void *obj, + void *data) +{ + nxt_event_conn_t *peer; + nxt_event_conn_proxy_t *p; + + peer = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy peer read ready fd:%d", + peer->socket.fd); + + nxt_event_conn_proxy_read_process(thr, p, peer, p->client); +} + + +static void +nxt_event_conn_proxy_read_process(nxt_thread_t *thr, nxt_event_conn_proxy_t *p, + nxt_event_conn_t *source, nxt_event_conn_t *sink) +{ + nxt_buf_t *rb, *wb; + + if (sink->socket.error != 0) { + nxt_log_debug(thr->log, "event conn proxy sink fd:%d error:%d", + sink->socket.fd, sink->socket.error); + + nxt_event_conn_proxy_write_error(thr, sink, sink->socket.data); + return; + } + + while (source->read != NULL) { + + rb = source->read; + + if (rb->mem.pos != rb->mem.free) { + + /* Add a read part to a write chain. */ + + wb = nxt_buf_mem_alloc(source->mem_pool, 0, 0); + if (wb == NULL) { + /* An error completion. */ + nxt_event_conn_proxy_complete(thr, p); + return; + } + + wb->mem.pos = rb->mem.pos; + wb->mem.free = rb->mem.free; + wb->mem.start = rb->mem.pos; + wb->mem.end = rb->mem.free; + + rb->mem.pos = rb->mem.free; + rb->mem.start = rb->mem.free; + + nxt_event_conn_proxy_write_add(sink, wb); + } + + if (rb->mem.start != rb->mem.end) { + nxt_thread_work_queue_push(thr, source->read_work_queue, + nxt_event_conn_proxy_read, + source, source->socket.data, + source->socket.log); + break; + } + + source->read = rb->next; + nxt_buf_free(source->mem_pool, rb); + } + + if (p->connected) { + nxt_event_conn_write_enqueue(thr, sink); + } +} + + +static void +nxt_event_conn_proxy_write_add(nxt_event_conn_t *c, nxt_buf_t *b) +{ + nxt_buf_t *first, *second, *prev; + + first = c->write; + + if (first == NULL) { + c->write = b; + return; + } + + /* + * A event conn proxy maintains a buffer per each direction. + * The buffer is divided by read and write parts. These parts are + * linked in buffer chains. There can be no more than two buffers + * in write chain at any time, because an added buffer is coalesced + * with the last buffer if possible. + */ + + second = first->next; + + if (second == NULL) { + + if (first->mem.end != b->mem.start) { + first->next = b; + return; + } + + /* + * The first buffer is just before the added buffer, so + * expand the first buffer to the end of the added buffer. + */ + prev = first; + + } else { + if (second->mem.end != b->mem.start) { + nxt_thread_log_alert("event conn proxy write: second buffer end:%p " + "is not equal to added buffer start:%p", + second->mem.end, b->mem.start); + return; + } + + /* + * "second->mem.end == b->mem.start" must be always true here, + * that is the second buffer is just before the added buffer, + * so expand the second buffer to the end of added buffer. + */ + prev = second; + } + + prev->mem.free = b->mem.end; + prev->mem.end = b->mem.end; + + nxt_buf_free(c->mem_pool, b); +} + + +static void +nxt_event_conn_proxy_read(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *source, *sink; + nxt_event_conn_proxy_t *p; + + source = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy read fd:%d", source->socket.fd); + + if (!source->socket.closed) { + sink = (source == p->client) ? p->peer : p->client; + + if (sink->socket.error == 0) { + nxt_event_conn_read(thr, source); + } + } +} + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_client_write_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_event_conn_proxy_client_write_ready, + NULL, + nxt_event_conn_proxy_write_error, + + nxt_event_conn_proxy_write_timeout, + nxt_event_conn_proxy_timeout_value, + offsetof(nxt_event_conn_proxy_t, client_write_timeout), +}; + + +static void +nxt_event_conn_proxy_client_write_ready(nxt_thread_t *thr, void *obj, + void *data) +{ + nxt_event_conn_t *client; + nxt_event_conn_proxy_t *p; + + client = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy client write ready fd:%d", + client->socket.fd); + + nxt_event_conn_proxy_write_process(thr, p, client, p->peer); +} + + +static const nxt_event_conn_state_t nxt_event_conn_proxy_peer_write_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_event_conn_proxy_peer_write_ready, + NULL, + nxt_event_conn_proxy_write_error, + + nxt_event_conn_proxy_write_timeout, + nxt_event_conn_proxy_timeout_value, + offsetof(nxt_event_conn_proxy_t, peer_write_timeout), +}; + + +static void +nxt_event_conn_proxy_peer_write_ready(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *peer; + nxt_event_conn_proxy_t *p; + + peer = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy peer write ready fd:%d", + peer->socket.fd); + + nxt_event_conn_proxy_write_process(thr, p, peer, p->client); +} + + +static void +nxt_event_conn_proxy_write_process(nxt_thread_t *thr, nxt_event_conn_proxy_t *p, + nxt_event_conn_t *sink, nxt_event_conn_t *source) +{ + nxt_buf_t *rb, *wb; + + while (sink->write != NULL) { + + wb = sink->write; + + if (nxt_buf_is_sync(wb)) { + + /* A sync buffer marks the end of stream. */ + + sink->write = NULL; + nxt_buf_free(sink->mem_pool, wb); + nxt_event_conn_proxy_shutdown(thr, p, source, sink); + return; + } + + if (wb->mem.start != wb->mem.pos) { + + /* Add a written part to a read chain. */ + + rb = nxt_buf_mem_alloc(sink->mem_pool, 0, 0); + if (rb == NULL) { + /* An error completion. */ + nxt_event_conn_proxy_complete(thr, p); + return; + } + + rb->mem.pos = wb->mem.start; + rb->mem.free = wb->mem.start; + rb->mem.start = wb->mem.start; + rb->mem.end = wb->mem.pos; + + wb->mem.start = wb->mem.pos; + + nxt_event_conn_proxy_read_add(source, rb); + } + + if (wb->mem.pos != wb->mem.free) { + nxt_event_conn_write_enqueue(thr, sink); + + break; + } + + sink->write = wb->next; + nxt_buf_free(sink->mem_pool, wb); + } + + nxt_thread_work_queue_push(thr, source->read_work_queue, + nxt_event_conn_proxy_read, source, + source->socket.data, source->socket.log); +} + + +static void +nxt_event_conn_proxy_read_add(nxt_event_conn_t *c, nxt_buf_t *b) +{ + nxt_buf_t *first, *second; + + first = c->read; + + if (first == NULL) { + c->read = b; + return; + } + + /* + * A event conn proxy maintains a buffer per each direction. + * The buffer is divided by read and write parts. These parts are + * linked in buffer chains. There can be no more than two buffers + * in read chain at any time, because an added buffer is coalesced + * with the last buffer if possible. The first and the second + * buffers are also coalesced if possible. + */ + + second = first->next; + + if (second == NULL) { + + if (first->mem.start == b->mem.end) { + /* + * The added buffer is just before the first buffer, so expand + * the first buffer to the beginning of the added buffer. + */ + first->mem.pos = b->mem.start; + first->mem.free = b->mem.start; + first->mem.start = b->mem.start; + + } else if (first->mem.end == b->mem.start) { + /* + * The added buffer is just after the first buffer, so + * expand the first buffer to the end of the added buffer. + */ + first->mem.end = b->mem.end; + + } else { + first->next = b; + return; + } + + } else { + if (second->mem.end != b->mem.start) { + nxt_thread_log_alert("event conn proxy read: second buffer end:%p " + "is not equal to added buffer start:%p", + second->mem.end, b->mem.start); + return; + } + + /* + * The added buffer is just after the second buffer, so + * expand the second buffer to the end of the added buffer. + */ + second->mem.end = b->mem.end; + + if (first->mem.start == second->mem.end) { + /* + * The second buffer is just before the first buffer, so expand + * the first buffer to the beginning of the second buffer. + */ + first->mem.pos = second->mem.start; + first->mem.free = second->mem.start; + first->mem.start = second->mem.start; + first->next = NULL; + + nxt_buf_free(c->mem_pool, second); + } + } + + nxt_buf_free(c->mem_pool, b); +} + + +static void +nxt_event_conn_proxy_close(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_event_conn_t *source, *sink; + nxt_event_conn_proxy_t *p; + + source = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy close fd:%d", source->socket.fd); + + sink = (source == p->client) ? p->peer : p->client; + + if (sink->write == NULL) { + nxt_event_conn_proxy_shutdown(thr, p, source, sink); + return; + } + + b = nxt_buf_sync_alloc(source->mem_pool, 0); + if (b == NULL) { + /* An error completion. */ + nxt_event_conn_proxy_complete(thr, p); + return; + } + + nxt_buf_chain_add(&sink->write, b); +} + + +static void +nxt_event_conn_proxy_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_event_conn_proxy_t *p; + + c = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy error fd:%d", c->socket.fd); + + nxt_event_conn_proxy_close(thr, c, p); +} + + +static void +nxt_event_conn_proxy_read_timeout(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_event_timer_t *ev; + + ev = obj; + + c = nxt_event_read_timer_conn(ev); + c->socket.timedout = 1; + c->socket.closed = 1; + + nxt_log_debug(thr->log, "event conn proxy read timeout fd:%d", + c->socket.fd); + + nxt_event_conn_proxy_close(thr, c, c->socket.data); +} + + +static void +nxt_event_conn_proxy_write_timeout(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_event_timer_t *ev; + + ev = obj; + + c = nxt_event_write_timer_conn(ev); + c->socket.timedout = 1; + c->socket.closed = 1; + + nxt_log_debug(thr->log, "event conn proxy write timeout fd:%d", + c->socket.fd); + + nxt_event_conn_proxy_close(thr, c, c->socket.data); +} + + +static nxt_msec_t +nxt_event_conn_proxy_timeout_value(nxt_event_conn_t *c, uintptr_t data) +{ + nxt_msec_t *timer; + nxt_event_conn_proxy_t *p; + + p = c->socket.data; + + timer = (nxt_msec_t *) ((char *) p + data); + + return *timer; +} + + +static void +nxt_event_conn_proxy_refused(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *peer; + nxt_event_conn_proxy_t *p; + + peer = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy refused fd:%d", peer->socket.fd); + + if (p->retries == 0) { + /* An error completion. */ + nxt_event_conn_proxy_complete(thr, p); + return; + } + + p->retries--; + + nxt_socket_close(peer->socket.fd); + peer->socket.fd = -1; + peer->socket.error = 0; + + p->delayed = 1; + + peer->write_timer.handler = nxt_event_conn_proxy_reconnect_handler; + nxt_event_timer_add(thr->engine, &peer->write_timer, p->reconnect_timeout); +} + + +static void +nxt_event_conn_proxy_reconnect_handler(nxt_thread_t *thr, void *obj, + void *data) +{ + nxt_event_conn_t *peer; + nxt_event_timer_t *ev; + nxt_event_conn_proxy_t *p; + + ev = obj; + + nxt_log_debug(thr->log, "event conn proxy reconnect timer"); + + peer = nxt_event_write_timer_conn(ev); + p = peer->socket.data; + + if (p->client->socket.closed) { + nxt_event_conn_proxy_complete(thr, p); + return; + } + + p->delayed = 0; + + peer->write_state = &nxt_event_conn_proxy_peer_connect_state; + /* + * Peer read event: disabled. + * Peer write event: waiting for connection with connect_timeout. + */ + nxt_event_conn_connect(thr, peer); +} + + +static void +nxt_event_conn_proxy_shutdown(nxt_thread_t *thr, nxt_event_conn_proxy_t *p, + nxt_event_conn_t *source, nxt_event_conn_t *sink) +{ + nxt_buf_t *b; + + nxt_log_debug(source->socket.log, + "event conn proxy shutdown source fd:%d cl:%d err:%d", + source->socket.fd, source->socket.closed, + source->socket.error); + + nxt_log_debug(sink->socket.log, + "event conn proxy shutdown sink fd:%d cl:%d err:%d", + sink->socket.fd, sink->socket.closed, sink->socket.error); + + if (!p->connected || p->delayed) { + nxt_event_conn_proxy_complete(thr, p); + return; + } + + if (sink->socket.error != 0 || sink->socket.closed) { + /* + * A socket is already closed or half-closed by + * remote side so the shutdown() syscall is surplus + * since the close() syscall also sends FIN. + */ + nxt_event_conn_close(thr, sink); + + } else { + nxt_socket_shutdown(sink->socket.fd, SHUT_WR); + } + + if (sink->socket.error != 0 + || (sink->socket.closed && source->write == NULL)) + { + /* The opposite direction also has been already closed. */ + nxt_event_conn_proxy_complete(thr, p); + return; + } + + /* Free the direction's buffer. */ + b = (source == p->client) ? p->client_buffer : p->peer_buffer; + nxt_mem_free(source->mem_pool, b); +} + + +static void +nxt_event_conn_proxy_read_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_event_conn_proxy_t *p; + + c = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy read error fd:%d", c->socket.fd); + + nxt_event_conn_proxy_close(thr, c, p); +} + + +static void +nxt_event_conn_proxy_write_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *source, *sink; + nxt_event_conn_proxy_t *p; + + sink = obj; + p = data; + + nxt_log_debug(thr->log, "event conn proxy write error fd:%d", + sink->socket.fd); + + /* Clear data for the direction sink. */ + sink->write = NULL; + + /* Block the direction source. */ + source = (sink == p->client) ? p->peer : p->client; + nxt_event_fd_block_read(thr->engine, &source->socket); + + if (source->write == NULL) { + /* + * There is no data for the opposite direction and + * the next read from the sink will most probably fail. + */ + nxt_event_conn_proxy_complete(thr, p); + } +} + + +static void +nxt_event_conn_proxy_complete(nxt_thread_t *thr, nxt_event_conn_proxy_t *p) +{ + nxt_log_debug(p->client->socket.log, "event conn proxy complete %d:%d", + p->client->socket.fd, p->peer->socket.fd); + + if (p->client->socket.fd != -1) { + nxt_event_conn_close(thr, p->client); + } + + if (p->peer->socket.fd != -1) { + nxt_event_conn_close(thr, p->peer); + + } else if (p->delayed) { + nxt_thread_work_queue_drop(thr, &p->peer->write_timer); + + nxt_queue_remove(&p->peer->link); + nxt_event_timer_delete(thr->engine, &p->peer->write_timer); + } + + nxt_mem_free(p->client->mem_pool, p->client_buffer); + nxt_mem_free(p->client->mem_pool, p->peer_buffer); + + p->completion_handler(thr, p, NULL); +} diff --git a/src/nxt_event_conn_read.c b/src/nxt_event_conn_read.c new file mode 100644 index 00000000..59d7d59c --- /dev/null +++ b/src/nxt_event_conn_read.c @@ -0,0 +1,259 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +void +nxt_event_conn_read(nxt_thread_t *thr, nxt_event_conn_t *c) +{ + nxt_work_queue_t *wq; + nxt_work_handler_t handler; + + handler = c->io->read; + + if (thr->engine->batch != 0) { + + wq = &thr->engine->read_work_queue; + c->socket.read_work_queue = wq; + + nxt_thread_work_queue_add(thr, wq, handler, c, c->socket.data, + c->socket.log); + return; + } + + handler(thr, c, c->socket.data); +} + + +void +nxt_event_conn_io_read(nxt_thread_t *thr, void *obj, void *data) +{ + ssize_t n; + nxt_buf_t *b; + nxt_bool_t batch; + nxt_event_conn_t *c; + nxt_work_handler_t handler; + const nxt_event_conn_state_t *state; + + c = obj; + + nxt_log_debug(thr->log, "event conn read fd:%d rdy:%d cl:%d", + c->socket.fd, c->socket.read_ready, c->socket.closed); + + batch = (thr->engine->batch != 0); + state = c->read_state; + + if (c->socket.read_ready) { + + b = c->read; + + if (b == NULL) { + /* Just test descriptor readiness. */ + goto ready; + } + + if (c->peek == 0) { + n = c->io->recvbuf(c, b); + + } else { + n = c->io->recv(c, b->mem.free, c->peek, MSG_PEEK); + } + + if (n > 0) { + c->nbytes = n; + + if (state->process_buffers) { + nxt_recvbuf_update(b, n); + + } else { + /* + * A ready_handler must not be queued, instead buffers + * must be processed by the ready_handler at once after + * recv() operation, otherwise two sequentially queued + * recv() operations will read in the same buffers. + */ + batch = 0; + } + + goto ready; + } + + if (n != NXT_AGAIN) { + nxt_event_fd_block_read(thr->engine, &c->socket); + nxt_event_timer_disable(&c->read_timer); + + if (n == 0) { + handler = state->close_handler; + goto done; + } + + /* n == NXT_ERROR */ + handler = state->error_handler; + goto done; + } + } + + /* + * Here c->io->read() is assigned instead of direct + * nxt_event_conn_io_read() because the function can + * be called by nxt_kqueue_event_conn_io_read(). + */ + c->socket.read_handler = c->io->read; + c->socket.error_handler = state->error_handler; + + if (c->read_timer.state == NXT_EVENT_TIMER_DISABLED + || nxt_event_fd_is_disabled(c->socket.read)) + { + /* Timer may be set or reset. */ + nxt_event_conn_timer(thr->engine, c, state, &c->read_timer); + + if (nxt_event_fd_is_disabled(c->socket.read)) { + nxt_event_fd_enable_read(thr->engine, &c->socket); + } + } + + return; + +ready: + + nxt_event_fd_block_read(thr->engine, &c->socket); + + if (state->autoreset_timer) { + nxt_event_timer_disable(&c->read_timer); + } + + handler = state->ready_handler; + +done: + + if (batch) { + nxt_thread_work_queue_add(thr, c->read_work_queue, handler, + c, data, thr->log); + } else { + handler(thr, c, data); + } +} + + +ssize_t +nxt_event_conn_io_recvbuf(nxt_event_conn_t *c, nxt_buf_t *b) +{ + ssize_t n; + nxt_err_t err; + nxt_uint_t niov; + struct iovec iov[NXT_IOBUF_MAX]; + nxt_recvbuf_coalesce_t rb; + + rb.buf = b; + rb.iobuf = iov; + rb.nmax = NXT_IOBUF_MAX; + rb.size = 0; + + niov = nxt_recvbuf_mem_coalesce(&rb); + + if (niov == 1) { + /* Disposal of surplus kernel iovec copy-in operation. */ + return nxt_event_conn_io_recv(c, iov->iov_base, iov->iov_len, 0); + } + + for ( ;; ) { + n = readv(c->socket.fd, iov, niov); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(c->socket.log, "readv(%d, %ui): %z", + c->socket.fd, niov, n); + + if (n > 0) { + if ((size_t) n < rb.size) { + c->socket.read_ready = 0; + } + + return n; + } + + if (n == 0) { + c->socket.closed = 1; + c->socket.read_ready = 0; + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(c->socket.log, "readv() %E", err); + c->socket.read_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(c->socket.log, "readv() %E", err); + continue; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "readv(%d, %ui) failed %E", + c->socket.fd, niov, err); + return NXT_ERROR; + } + } +} + + +ssize_t +nxt_event_conn_io_recv(nxt_event_conn_t *c, void *buf, size_t size, + nxt_uint_t flags) +{ + ssize_t n; + nxt_err_t err; + + for ( ;; ) { + n = recv(c->socket.fd, buf, size, flags); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(c->socket.log, "recv(%d, %p, %uz, 0x%ui): %z", + c->socket.fd, buf, size, flags, n); + + if (n > 0) { + if ((size_t) n < size) { + c->socket.read_ready = 0; + } + + return n; + } + + if (n == 0) { + c->socket.closed = 1; + c->socket.read_ready = 0; + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(c->socket.log, "recv() %E", err); + c->socket.read_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(c->socket.log, "recv() %E", err); + continue; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "recv(%d, %p, %uz, %ui) failed %E", + c->socket.fd, buf, size, flags, err); + return NXT_ERROR; + } + } +} diff --git a/src/nxt_event_conn_write.c b/src/nxt_event_conn_write.c new file mode 100644 index 00000000..0f35b5f3 --- /dev/null +++ b/src/nxt_event_conn_write.c @@ -0,0 +1,431 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_event_conn_average_rate_update(nxt_event_write_rate_t *rate, + size_t sent, nxt_msec_t now); +NXT_LIB_UNIT_TEST_STATIC double + nxt_event_conn_exponential_approximation(double n); +static void nxt_event_conn_write_timer_handler(nxt_thread_t *thr, void *obj, + void *data); + + +void +nxt_event_conn_write(nxt_thread_t *thr, nxt_event_conn_t *c) +{ + if (thr->engine->batch != 0) { + nxt_event_conn_write_enqueue(thr, c); + + } else { + c->io->write(thr, c, c->socket.data); + } +} + + +void +nxt_event_conn_io_write(nxt_thread_t *thr, void *obj, void *data) +{ + size_t sent, limit; + ssize_t ret; + nxt_buf_t *b; + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "event conn write fd:%d", c->socket.fd); + + if (!c->socket.write_ready || c->delayed || c->write == NULL) { + return; + } + + c->socket.write_handler = nxt_event_conn_io_write; + c->socket.error_handler = c->write_state->error_handler; + + ret = NXT_DECLINED; + sent = 0; + b = c->write; + + limit = nxt_event_conn_write_limit(c); + + while (limit != 0) { + + ret = c->io->write_chunk(thr, c, b, limit); + + if (ret < 0) { + /* ret == NXT_AGAIN || ret == NXT_ERROR. */ + break; + } + + sent += ret; + limit -= ret; + + if (c->write_state->process_buffers) { + b = nxt_sendbuf_completion(thr, c->write_work_queue, b, ret); + c->write = b; + + } else { + b = nxt_sendbuf_update(b, ret); + } + + if (b == NULL) { + nxt_event_fd_block_write(thr->engine, &c->socket); + break; + } + + if (!c->socket.write_ready) { + ret = NXT_AGAIN; + break; + } + } + + nxt_log_debug(thr->log, "event conn: %i sent:%z", ret, sent); + + if (sent != 0) { + if (c->write_state->autoreset_timer) { + nxt_event_timer_disable(&c->write_timer); + } + } + + if (ret != NXT_ERROR + && !nxt_event_conn_write_delayed(thr->engine, c, sent)) + { + if (limit == 0) { + /* + * Postpone writing until next event poll to allow to + * process other recevied events and to get new events. + */ + c->write_timer.handler = nxt_event_conn_write_timer_handler; + nxt_event_timer_add(thr->engine, &c->write_timer, 0); + + } else if (ret == NXT_AGAIN) { + /* + * SSL libraries can require to toggle either write or read + * event if renegotiation occurs during SSL write operation. + * This case is handled on the event_io->send() level. Timer + * can be set here because it should be set only for write + * direction. + */ + nxt_event_conn_timer(thr->engine, c, c->write_state, + &c->write_timer); + } + } + + if (ret == 0 || sent != 0) { + /* "ret == 0" means a sync buffer was processed. */ + c->sent += sent; + nxt_event_conn_io_handle(thr, c->write_work_queue, + c->write_state->ready_handler, c, data); + /* + * Fall through if first operations were + * successful but the last one failed. + */ + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_event_fd_block_write(thr->engine, &c->socket); + + nxt_event_conn_io_handle(thr, c->write_work_queue, + c->write_state->error_handler, c, data); + } +} + + +size_t +nxt_event_conn_write_limit(nxt_event_conn_t *c) +{ + ssize_t limit, correction; + nxt_event_write_rate_t *rate; + + rate = c->rate; + + if (rate == NULL) { + return c->max_chunk; + } + + limit = rate->limit; + correction = limit - (size_t) rate->average; + + nxt_log_debug(c->socket.log, "event conn correction:%z average:%0.3f", + correction, rate->average); + + limit += correction; + + if (limit <= 0) { + return 0; + } + + if (rate->limit_after != 0) { + limit += rate->limit_after; + limit = nxt_min((size_t) limit, rate->max_limit); + } + + return nxt_min((size_t) limit, c->max_chunk); +} + + +nxt_bool_t +nxt_event_conn_write_delayed(nxt_event_engine_t *engine, nxt_event_conn_t *c, + size_t sent) +{ + nxt_msec_t timer; + nxt_event_write_rate_t *rate; + + rate = c->rate; + + if (rate != NULL) { + nxt_event_conn_average_rate_update(rate, sent, engine->timers.now); + + if (rate->limit_after == 0) { + timer = sent * 1000 / rate->limit; + + } else if (rate->limit_after >= sent) { + timer = sent * 1000 / rate->max_limit; + rate->limit_after -= sent; + + } else { + sent -= rate->limit_after; + timer = rate->limit_after * 1000 / rate->max_limit + + sent * 1000 / rate->limit; + rate->limit_after = 0; + } + + nxt_log_debug(c->socket.log, "event conn timer: %M", timer); + + if (timer != 0) { + c->delayed = 1; + + nxt_event_fd_block_write(engine, &c->socket); + + c->write_timer.handler = nxt_event_conn_write_timer_handler; + nxt_event_timer_add(engine, &c->write_timer, timer); + + return 1; + } + } + + return 0; +} + + +/* Exponentially weighted moving average rate for a given interval. */ + +static void +nxt_event_conn_average_rate_update(nxt_event_write_rate_t *rate, size_t sent, + nxt_msec_t now) +{ + double weight, delta; + nxt_msec_t elapsed; + const nxt_uint_t interval = 10; /* 10s */ + + elapsed = now - rate->last; + + if (elapsed == 0) { + return; + } + + rate->last = now; + delta = (double) elapsed / 1000; + + weight = nxt_event_conn_exponential_approximation(-delta / interval); + + rate->average = (1 - weight) * sent / delta + weight * rate->average; + + nxt_thread_log_debug("event conn delta:%0.3f, weight:%0.3f, average:%0.3f", + delta, weight, rate->average); +} + + +/* + * exp() takes tens or hundreds nanoseconds on modern CPU. + * This is a faster exp() approximation based on IEEE-754 format + * layout and described in "A Fast, Compact Approximation of + * the Exponential Function" * by N. N. Schraudolph, 1999. + */ + +NXT_LIB_UNIT_TEST_STATIC double +nxt_event_conn_exponential_approximation(double x) +{ + union { + double d; + int64_t n; + } exp; + + if (x < -100) { + /* + * The approximation is correct in -700 to 700 range. + * The "x" argument is always negative. + */ + return 0; + } + + /* + * x * 2^52 / ln(2) + (1023 * 2^52 - 261140389990637.73 + * + * 52 is the number of mantissa bits; + * 1023 is the exponent bias; + * 261140389990637.73 is the adjustment parameter to + * improve the approximation. The parameter is equal to + * + * 2^52 * ln[ 3 / (8 * ln(2)) + 0.5 ] / ln(2) + * + * Only significant digits of the double float format + * are used to present the double float constants. + */ + exp.n = x * 4503599627370496.0 / 0.69314718055994530 + + (4607182418800017408.0 - 261140389990637.73); + + return exp.d; +} + + +static void +nxt_event_conn_write_timer_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_event_timer_t *ev; + + ev = obj; + + nxt_log_debug(thr->log, "event conn conn timer"); + + c = nxt_event_write_timer_conn(ev); + c->delayed = 0; + + c->io->write(thr, c, c->socket.data); +} + + +ssize_t +nxt_event_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, + nxt_buf_t *b, size_t limit) +{ + ssize_t ret; + + ret = c->io->sendbuf(c, b, limit); + + if ((ret == NXT_AGAIN || !c->socket.write_ready) + && nxt_event_fd_is_disabled(c->socket.write)) + { + nxt_event_fd_enable_write(thr->engine, &c->socket); + } + + return ret; +} + + +ssize_t +nxt_event_conn_io_sendbuf(nxt_event_conn_t *c, nxt_buf_t *b, size_t limit) +{ + nxt_uint_t niob; + struct iovec iob[NXT_IOBUF_MAX]; + nxt_sendbuf_coalesce_t sb; + + sb.buf = b; + sb.iobuf = iob; + sb.nmax = NXT_IOBUF_MAX; + sb.sync = 0; + sb.size = 0; + sb.limit = limit; + + niob = nxt_sendbuf_mem_coalesce(&sb); + + if (niob == 0 && sb.sync) { + return 0; + } + + return nxt_event_conn_io_writev(c, iob, niob); +} + + +ssize_t +nxt_event_conn_io_writev(nxt_event_conn_t *c, nxt_iobuf_t *iob, nxt_uint_t niob) +{ + ssize_t n; + nxt_err_t err; + + if (niob == 1) { + /* Disposal of surplus kernel iovec copy-in operation. */ + return nxt_event_conn_io_send(c, iob->iov_base, iob->iov_len); + } + + for ( ;; ) { + n = writev(c->socket.fd, iob, niob); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(c->socket.log, "writev(%d, %ui): %d", + c->socket.fd, niob, n); + + if (n > 0) { + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(c->socket.log, "writev() %E", err); + c->socket.write_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(c->socket.log, "writev() %E", err); + continue; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "writev(%d, %ui) failed %E", + c->socket.fd, niob, err); + return NXT_ERROR; + } + } +} + + +ssize_t +nxt_event_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) +{ + ssize_t n; + nxt_err_t err; + + for ( ;; ) { + n = send(c->socket.fd, buf, size, 0); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(c->socket.log, "send(%d, %p, %uz): %z", + c->socket.fd, buf, size, n); + + if (n > 0) { + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(c->socket.log, "send() %E", err); + c->socket.write_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(c->socket.log, "send() %E", err); + continue; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "send(%d, %p, %uz) failed %E", + c->socket.fd, buf, size, err); + return NXT_ERROR; + } + } +} diff --git a/src/nxt_event_engine.c b/src/nxt_event_engine.c new file mode 100644 index 00000000..dd0f5fe3 --- /dev/null +++ b/src/nxt_event_engine.c @@ -0,0 +1,526 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_event_engine_post_init(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static nxt_int_t nxt_event_engine_signal_pipe_create(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static void nxt_event_engine_signal_pipe_close(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_engine_signal_pipe(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_engine_post_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_engine_signal_pipe_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_event_engine_signal_handler(nxt_thread_t *thr, void *obj, + void *data); +static const nxt_event_sig_t *nxt_event_engine_signal_find(nxt_thread_t *thr, + nxt_uint_t signo); + + +nxt_event_engine_t * +nxt_event_engine_create(nxt_thread_t *thr, const nxt_event_set_ops_t *event_set, + const nxt_event_sig_t *signals, nxt_uint_t flags, nxt_uint_t batch) +{ + nxt_uint_t events; + nxt_event_engine_t *engine; + + engine = nxt_zalloc(sizeof(nxt_event_engine_t)); + if (engine == NULL) { + return NULL; + } + + engine->batch = batch; + + if (flags & NXT_ENGINE_FIBERS) { + engine->fibers = nxt_fiber_main_create(engine); + if (engine->fibers == NULL) { + goto fibers_fail; + } + } + + nxt_thread_work_queue_create(thr, 0); + + nxt_work_queue_name(&engine->accept_work_queue, "accept"); + nxt_work_queue_name(&engine->read_work_queue, "read"); + nxt_work_queue_name(&engine->socket_work_queue, "socket"); + nxt_work_queue_name(&engine->connect_work_queue, "connect"); + nxt_work_queue_name(&engine->write_work_queue, "write"); + nxt_work_queue_name(&engine->shutdown_work_queue, "shutdown"); + nxt_work_queue_name(&engine->close_work_queue, "close"); + +#if (NXT_THREADS) + + nxt_locked_work_queue_create(&engine->work_queue, 7); + +#endif + + if (signals != NULL) { + engine->signals = nxt_event_engine_signals(signals); + if (engine->signals == NULL) { + goto signals_fail; + } + + engine->signals->handler = nxt_event_engine_signal_handler; + + if (!event_set->signal_support) { + if (nxt_event_engine_signals_start(engine) != NXT_OK) { + goto signals_fail; + } + } + } + + /* + * Number of event set and timers changes should be at least twice + * more than number of events to avoid premature flushes of the changes. + * Fourfold is for sure. + */ + events = (batch != 0) ? batch : 32; + + engine->event_set = event_set->create(engine->signals, 4 * events, events); + if (engine->event_set == NULL) { + goto event_set_fail; + } + + engine->event = event_set; + + if (nxt_event_engine_post_init(thr, engine) != NXT_OK) { + goto post_fail; + } + + if (nxt_event_timers_init(&engine->timers, 4 * events) != NXT_OK) { + goto timers_fail; + } + + nxt_thread_time_update(thr); + engine->timers.now = nxt_thread_monotonic_time(thr) / 1000000; + + engine->max_connections = 0xffffffff; + + nxt_queue_init(&engine->listen_connections); + nxt_queue_init(&engine->idle_connections); + + thr->engine = engine; + thr->fiber = &engine->fibers->fiber; + +#if !(NXT_THREADS) + + if (engine->event->signal_support) { + thr->time.signal = -1; + } + +#endif + + return engine; + +timers_fail: +post_fail: + + event_set->free(engine->event_set); + +event_set_fail: +signals_fail: + + nxt_free(engine->signals); + nxt_thread_work_queue_destroy(thr); + nxt_free(engine->fibers); + +fibers_fail: + + nxt_free(engine); + return NULL; +} + + +static nxt_int_t +nxt_event_engine_post_init(nxt_thread_t *thr, nxt_event_engine_t *engine) +{ + if (engine->event->enable_post != NULL) { + return engine->event->enable_post(engine->event_set, + nxt_event_engine_post_handler); + } + +#if !(NXT_THREADS) + + /* Only signals may are posted in single-threaded mode. */ + + if (engine->event->signal_support) { + return NXT_OK; + } + +#endif + + if (nxt_event_engine_signal_pipe_create(thr, engine) != NXT_OK) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_event_engine_signal_pipe_create(nxt_thread_t *thr, + nxt_event_engine_t *engine) +{ + nxt_event_engine_pipe_t *pipe; + + pipe = nxt_zalloc(sizeof(nxt_event_engine_pipe_t)); + if (pipe == NULL) { + return NXT_ERROR; + } + + engine->pipe = pipe; + + /* + * An event engine pipe is in blocking mode for writer + * and in non-blocking node for reader. + */ + + if (nxt_pipe_create(pipe->fds, 1, 0) != NXT_OK) { + nxt_free(pipe); + return NXT_ERROR; + } + + pipe->event.fd = pipe->fds[0]; + pipe->event.read_work_queue = &thr->work_queue.main; + pipe->event.read_handler = nxt_event_engine_signal_pipe; + pipe->event.write_work_queue = &thr->work_queue.main; + pipe->event.error_handler = nxt_event_engine_signal_pipe_error; + pipe->event.log = &nxt_main_log; + + nxt_event_fd_enable_read(engine, &pipe->event); + + return NXT_OK; +} + + +static void +nxt_event_engine_signal_pipe_free(nxt_event_engine_t *engine) +{ + nxt_event_engine_pipe_t *pipe; + + pipe = engine->pipe; + + if (pipe != NULL) { + + if (pipe->event.read_work_queue != NULL) { + nxt_event_fd_close(engine, &pipe->event); + nxt_pipe_close(pipe->fds); + } + + nxt_free(pipe); + } +} + + +static void +nxt_event_engine_signal_pipe_close(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_engine_pipe_t *pipe; + + pipe = obj; + + nxt_pipe_close(pipe->fds); + nxt_free(pipe); +} + + +void +nxt_event_engine_post(nxt_event_engine_t *engine, nxt_work_handler_t handler, + void *obj, void *data, nxt_log_t *log) +{ + nxt_thread_log_debug("event engine post"); + + nxt_locked_work_queue_add(&engine->work_queue, handler, obj, data, log); + + nxt_event_engine_signal(engine, 0); +} + + +void +nxt_event_engine_signal(nxt_event_engine_t *engine, nxt_uint_t signo) +{ + u_char buf; + + nxt_thread_log_debug("event engine signal:%ui", signo); + + /* + * A signal number may be sent in a signal context, so the signal + * information cannot be passed via a locked work queue. + */ + + if (engine->event->signal != NULL) { + engine->event->signal(engine->event_set, signo); + return; + } + + buf = (u_char) signo; + (void) nxt_fd_write(engine->pipe->fds[1], &buf, 1); +} + + +static void +nxt_event_engine_signal_pipe(nxt_thread_t *thr, void *obj, void *data) +{ + int i, n; + u_char signo; + nxt_bool_t post; + nxt_event_fd_t *ev; + const nxt_event_sig_t *sigev; + u_char buf[128]; + + ev = obj; + + nxt_log_debug(thr->log, "engine signal pipe"); + + post = 0; + + do { + n = nxt_fd_read(ev->fd, buf, sizeof(buf)); + + for (i = 0; i < n; i++) { + signo = buf[i]; + + nxt_log_debug(thr->log, "engine pipe signo:%d", signo); + + if (signo == 0) { + /* A post should be processed only once. */ + post = 1; + + } else { + sigev = nxt_event_engine_signal_find(thr, signo); + + if (nxt_fast_path(sigev != NULL)) { + sigev->handler(thr, (void *) (uintptr_t) signo, + (void *) sigev->name); + } + } + } + + } while (n == sizeof(buf)); + + if (post) { + nxt_event_engine_post_handler(thr, NULL, NULL); + } +} + + +static void +nxt_event_engine_post_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_locked_work_queue_move(thr, &thr->engine->work_queue, + &thr->work_queue.main); +} + + +static void +nxt_event_engine_signal_pipe_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_fd_t *ev; + + ev = obj; + + nxt_log_alert(ev->log, "engine pipe(%FD:%FD) event error", + thr->engine->pipe->fds[0], thr->engine->pipe->fds[1]); + + nxt_event_fd_close(thr->engine, &thr->engine->pipe->event); + nxt_pipe_close(thr->engine->pipe->fds); +} + + +static void +nxt_event_engine_signal_handler(nxt_thread_t *thr, void *obj, void *data) +{ + uintptr_t signo; + const nxt_event_sig_t *sigev; + + signo = (uintptr_t) obj; + + sigev = nxt_event_engine_signal_find(thr, signo); + + if (nxt_fast_path(sigev != NULL)) { + sigev->handler(thr, (void *) (uintptr_t) signo, (void *) sigev->name); + } +} + + +static const nxt_event_sig_t * +nxt_event_engine_signal_find(nxt_thread_t *thr, nxt_uint_t signo) +{ + const nxt_event_sig_t *sigev; + + for (sigev = thr->engine->signals->sigev; sigev->signo != 0; sigev++) { + if (signo == (nxt_uint_t) sigev->signo) { + return sigev; + } + } + + nxt_log_alert(thr->log, "signal %ui handler not found", signo); + + return NULL; +} + + +nxt_int_t +nxt_event_engine_change(nxt_thread_t *thr, const nxt_event_set_ops_t *event_set, + nxt_uint_t batch) +{ + nxt_uint_t events; + nxt_event_engine_t *engine; + + engine = thr->engine; + engine->batch = batch; + + if (!engine->event->signal_support && event_set->signal_support) { + /* + * Block signal processing if the current event + * facility does not support signal processing. + */ + nxt_event_engine_signals_stop(engine); + + /* + * Add to thread main work queue the signal events possibly + * received before the blocking signal processing. + */ + nxt_event_engine_signal_pipe(thr, &engine->pipe->event, NULL); + } + + if (engine->pipe != NULL && event_set->enable_post != NULL) { + /* + * An engine pipe must be closed after all signal events + * added above to thread main work queue will be processed. + */ + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_event_engine_signal_pipe_close, + engine->pipe, NULL, &nxt_main_log); + + engine->pipe = NULL; + } + + engine->event->free(engine->event_set); + + events = (batch != 0) ? batch : 32; + + engine->event_set = event_set->create(engine->signals, 4 * events, events); + if (engine->event_set == NULL) { + return NXT_ERROR; + } + + engine->event = event_set; + + if (nxt_event_engine_post_init(thr, engine) != NXT_OK) { + return NXT_ERROR; + } + + if (engine->signals != NULL) { + + if (!engine->event->signal_support) { + return nxt_event_engine_signals_start(engine); + } + +#if (NXT_THREADS) + /* + * Reset the PID flag to start the signal thread if + * some future event facility will not support signals. + */ + engine->signals->process = 0; +#endif + } + + return NXT_OK; +} + + +void +nxt_event_engine_free(nxt_event_engine_t *engine) +{ + nxt_event_engine_signal_pipe_free(engine); + nxt_free(engine->signals); + + nxt_locked_work_queue_destroy(&engine->work_queue); + nxt_thread_work_queue_destroy(nxt_thread()); + + engine->event->free(engine->event_set); + + /* TODO: free timers */ + + nxt_free(engine); +} + + +void +nxt_event_engine_start(nxt_event_engine_t *engine) +{ + void *obj, *data; + nxt_msec_t timeout, now; + nxt_thread_t *thr; + nxt_work_handler_t handler; + + thr = nxt_thread(); + + if (engine->fibers) { + /* + * _setjmp() cannot be wrapped in a function since return from + * the function clobbers stack used by future _setjmp() returns. + */ + _setjmp(engine->fibers->fiber.jmp); + + /* A return point from fibers. */ + } + + for ( ;; ) { + + for ( ;; ) { + handler = nxt_thread_work_queue_pop(thr, &obj, &data, &thr->log); + + if (handler == NULL) { + break; + } + + handler(thr, obj, data); + + thr->log = &nxt_main_log; + } + + for ( ;; ) { + handler = nxt_thread_last_work_queue_pop(thr, &obj, &data, + &thr->log); + if (handler == NULL) { + break; + } + + handler(thr, obj, data); + + thr->log = &nxt_main_log; + } + + /* Attach some event engine work queues in preferred order. */ + + nxt_work_queue_attach(thr, &engine->accept_work_queue); + nxt_work_queue_attach(thr, &engine->read_work_queue); + + timeout = nxt_event_timer_find(engine); + + engine->event->poll(thr, engine->event_set, timeout); + + /* + * Look up expired timers only if a new zero timer has been + * just added before the event poll or if the event poll slept + * at least 1 millisecond, because all old eligible timers were + * processed in the previous iterations. + */ + + now = nxt_thread_monotonic_time(thr) / 1000000; + + if (timeout == 0 || now != engine->timers.now) { + nxt_event_timer_expire(thr, now); + } + } +} diff --git a/src/nxt_event_engine.h b/src/nxt_event_engine.h new file mode 100644 index 00000000..0cbfc89d --- /dev/null +++ b/src/nxt_event_engine.h @@ -0,0 +1,94 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_EVENT_ENGINE_H_INCLUDED_ +#define _NXT_EVENT_ENGINE_H_INCLUDED_ + + +#define NXT_ENGINE_FIBERS 1 + + +typedef struct { + nxt_fd_t fds[2]; + nxt_event_fd_t event; +} nxt_event_engine_pipe_t; + + +struct nxt_event_engine_s { + const nxt_event_set_ops_t *event; + nxt_event_set_t *event_set; + + nxt_event_timers_t timers; + + /* The engine ID, the main engine has ID 0. */ + uint32_t id; + + /* + * A pipe to pass event signals to the engine, if the engine's + * underlying event facility does not support user events. + */ + nxt_event_engine_pipe_t *pipe; + + nxt_work_queue_t accept_work_queue; + nxt_work_queue_t read_work_queue; + nxt_work_queue_t socket_work_queue; + nxt_work_queue_t connect_work_queue; + nxt_work_queue_t write_work_queue; + nxt_work_queue_t shutdown_work_queue; + nxt_work_queue_t close_work_queue; + + nxt_locked_work_queue_t work_queue; + + nxt_event_signals_t *signals; + + nxt_fiber_main_t *fibers; + + uint8_t shutdown; /* 1 bit */ + + uint32_t batch; + uint32_t connections; + uint32_t max_connections; + + nxt_queue_t listen_connections; + nxt_queue_t idle_connections; +}; + + +NXT_EXPORT nxt_event_engine_t *nxt_event_engine_create(nxt_thread_t *thr, + const nxt_event_set_ops_t *event_set, const nxt_event_sig_t *signals, + nxt_uint_t flags, nxt_uint_t batch); +NXT_EXPORT nxt_int_t nxt_event_engine_change(nxt_thread_t *thr, + const nxt_event_set_ops_t *event_set, nxt_uint_t batch); +NXT_EXPORT void nxt_event_engine_free(nxt_event_engine_t *engine); +NXT_EXPORT void nxt_event_engine_start(nxt_event_engine_t *engine); + +NXT_EXPORT void nxt_event_engine_post(nxt_event_engine_t *engine, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log); +NXT_EXPORT void nxt_event_engine_signal(nxt_event_engine_t *engine, + nxt_uint_t signo); + + +nxt_inline nxt_event_engine_t * +nxt_thread_event_engine(void) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + return thr->engine; +} + + +nxt_inline nxt_work_queue_t * +nxt_thread_main_work_queue(void) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + return &thr->work_queue.main; +} + + +#endif /* _NXT_EVENT_ENGINE_H_INCLUDED_ */ diff --git a/src/nxt_event_fd.h b/src/nxt_event_fd.h new file mode 100644 index 00000000..dd8d1c20 --- /dev/null +++ b/src/nxt_event_fd.h @@ -0,0 +1,110 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_EVENT_FD_H_INCLUDED_ +#define _NXT_EVENT_FD_H_INCLUDED_ + + +typedef enum { + /* A completely inactive event. */ + NXT_EVENT_INACTIVE = 0, + + /* + * An event presents in the kernel but disabled after oneshot. + * Used by epoll. + */ + NXT_EVENT_DISABLED, + + /* + * An event is active in the kernel but blocked by application. + * Used by kqueue, epoll, eventport, devpoll, and pollset. + */ + NXT_EVENT_BLOCKED, + + /* + * An active oneshot event. + * Used by epoll, devpoll, pollset, poll, and select. + */ + NXT_EVENT_ONESHOT, + + /* An active level-triggered event. Used by eventport. */ + NXT_EVENT_LEVEL, + + /* An active event. */ + NXT_EVENT_DEFAULT, +} nxt_event_fd_state_t; + + +#define \ +nxt_event_fd_is_disabled(state) \ + ((state) < NXT_EVENT_ONESHOT) + + +#define \ +nxt_event_fd_is_active(state) \ + ((state) >= NXT_EVENT_ONESHOT) + + +struct nxt_event_fd_s { + void *data; + + /* Both are int's. */ + nxt_socket_t fd; + nxt_err_t error; + + /* The flags should also be prefetched by nxt_work_queue_pop(). */ + +#if (NXT_64BIT) + uint8_t read; + uint8_t write; + uint8_t log_error; + uint8_t read_ready; + uint8_t write_ready; + uint8_t closed; + uint8_t timedout; +#if (NXT_HAVE_EPOLL) + uint8_t epoll_eof:1; + uint8_t epoll_error:1; +#endif +#if (NXT_HAVE_KQUEUE) + uint8_t kq_eof; +#endif + +#else /* NXT_32BIT */ + nxt_event_fd_state_t read:3; + nxt_event_fd_state_t write:3; + nxt_socket_error_level_t log_error:3; + unsigned read_ready:1; + unsigned write_ready:1; + unsigned closed:1; + unsigned timedout:1; +#if (NXT_HAVE_EPOLL) + unsigned epoll_eof:1; + unsigned epoll_error:1; +#endif +#if (NXT_HAVE_KQUEUE) + unsigned kq_eof:1; +#endif +#endif /* NXT_64BIT */ + +#if (NXT_HAVE_KQUEUE) + /* nxt_err_t is int. */ + nxt_err_t kq_errno; + /* struct kevent.data is intptr_t, however int32_t is enough. */ + int32_t kq_available; +#endif + + nxt_work_queue_t *read_work_queue; + nxt_work_handler_t read_handler; + nxt_work_queue_t *write_work_queue; + nxt_work_handler_t write_handler; + nxt_work_handler_t error_handler; + + nxt_log_t *log; +}; + + +#endif /* _NXT_EVENT_FD_H_INCLUDED_ */ diff --git a/src/nxt_event_file.h b/src/nxt_event_file.h new file mode 100644 index 00000000..8426f912 --- /dev/null +++ b/src/nxt_event_file.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_EVENT_FILE_H_INCLUDED_ +#define _NXT_EVENT_FILE_H_INCLUDED_ + + +typedef struct { + void *data; + nxt_file_t *file; + nxt_work_handler_t handler; +} nxt_event_file_t; + + +#endif /* _NXT_EVENT_FILE_H_INCLUDED_ */ diff --git a/src/nxt_event_set.c b/src/nxt_event_set.c new file mode 100644 index 00000000..2e75267b --- /dev/null +++ b/src/nxt_event_set.c @@ -0,0 +1,107 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_event_set_fd_hash_test(nxt_lvlhsh_query_t *lhq, + void *data); + + +static const nxt_lvlhsh_proto_t nxt_event_set_fd_hash_proto nxt_aligned(64) = +{ + NXT_LVLHSH_LARGE_MEMALIGN, + 0, + nxt_event_set_fd_hash_test, + nxt_lvlhsh_alloc, + nxt_lvlhsh_free, +}; + + +/* nxt_murmur_hash2() is unique for 4 bytes. */ + +static nxt_int_t +nxt_event_set_fd_hash_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + return NXT_OK; +} + + +nxt_int_t +nxt_event_set_fd_hash_add(nxt_lvlhsh_t *lh, nxt_fd_t fd, nxt_event_fd_t *ev) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_murmur_hash2(&fd, sizeof(nxt_fd_t)); + lhq.replace = 0; + lhq.value = ev; + lhq.proto = &nxt_event_set_fd_hash_proto; + + if (nxt_lvlhsh_insert(lh, &lhq) == NXT_OK) { + return NXT_OK; + } + + nxt_log_alert(ev->log, "event fd %d is already in hash", ev->fd); + return NXT_ERROR; +} + + +void * +nxt_event_set_fd_hash_get(nxt_lvlhsh_t *lh, nxt_fd_t fd) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_murmur_hash2(&fd, sizeof(nxt_fd_t)); + lhq.proto = &nxt_event_set_fd_hash_proto; + + if (nxt_lvlhsh_find(lh, &lhq) == NXT_OK) { + return lhq.value; + } + + nxt_thread_log_alert("event fd %d not found in hash", fd); + return NULL; +} + + +void +nxt_event_set_fd_hash_delete(nxt_lvlhsh_t *lh, nxt_fd_t fd, nxt_bool_t ignore) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_murmur_hash2(&fd, sizeof(nxt_fd_t)); + lhq.proto = &nxt_event_set_fd_hash_proto; + + if (nxt_lvlhsh_delete(lh, &lhq) != NXT_OK && !ignore) { + nxt_thread_log_alert("event fd %d not found in hash", fd); + } +} + + +void +nxt_event_set_fd_hash_destroy(nxt_lvlhsh_t *lh) +{ + nxt_event_fd_t *ev; + nxt_lvlhsh_each_t lhe; + nxt_lvlhsh_query_t lhq; + + nxt_memzero(&lhe, sizeof(nxt_lvlhsh_each_t)); + lhe.proto = &nxt_event_set_fd_hash_proto; + lhq.proto = &nxt_event_set_fd_hash_proto; + + for ( ;; ) { + ev = nxt_lvlhsh_each(lh, &lhe); + + if (ev == NULL) { + return; + } + + lhq.key_hash = nxt_murmur_hash2(&ev->fd, sizeof(nxt_fd_t)); + + if (nxt_lvlhsh_delete(lh, &lhq) != NXT_OK) { + nxt_thread_log_alert("event fd %d not found in hash", ev->fd); + } + } +} diff --git a/src/nxt_event_set.h b/src/nxt_event_set.h new file mode 100644 index 00000000..103a8125 --- /dev/null +++ b/src/nxt_event_set.h @@ -0,0 +1,473 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_EVENT_SET_H_INCLUDED_ +#define _NXT_EVENT_SET_H_INCLUDED_ + + +/* + * An event facility is kernel interface such as kqueue, epoll, etc. + * intended to get event notifications about file descriptor state, + * signals, etc. + * + * An event set provides generic interface to underlying event facility. + * Although event set and event facility are closely coupled with an event + * engine, nevertheless they are separated from an event engine to allow + * to add one event facility to another if underlying event facility allows + * this (Linux epoll, BSD kqueue, Solaris eventport). + */ + +typedef union nxt_event_set_u nxt_event_set_t; + + +#define NXT_FILE_EVENTS 1 +#define NXT_NO_FILE_EVENTS 0 + +#define NXT_SIGNAL_EVENTS 1 +#define NXT_NO_SIGNAL_EVENTS 0 + + +typedef struct { + + /* The canonical event set name. */ + const char *name; + + /* + * Create an event set. The mchanges argument is a maximum number of + * changes to send to the kernel. The mevents argument is a maximum + * number of events to retrieve from the kernel at once, if underlying + * event facility supports batch operations. + */ + nxt_event_set_t *(*create)(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); + + /* Close and free an event set. */ + void (*free)(nxt_event_set_t *data); + + /* + * Add a file descriptor to an event set and enable the most + * effective read and write event notification method provided + * by underlying event facility. + */ + void (*enable)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* Disable file descriptor event notifications. */ + void (*disable)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Delete a file descriptor from an event set. A possible usage + * is a moving of the file descriptor from one event set to another. + */ + void (*delete)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Delete a file descriptor from an event set before closing the + * file descriptor. The most event facilities such as Linux epoll, + * BSD kqueue, Solaris event ports, AIX pollset, and HP-UX /dev/poll + * delete a file descriptor automatically on the file descriptor close. + * Some facilities such as Solaris /dev/poll require to delete a file + * descriptor explicitly. + */ + void (*close)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Add a file descriptor to an event set and enable the most effective + * read event notification method provided by underlying event facility. + */ + void (*enable_read)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Add a file descriptor to an event set and enable the most effective + * write event notification method provided by underlying event facility. + */ + void (*enable_write)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* Disable file descriptor read event notifications. */ + void (*disable_read)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* Disable file descriptor write event notifications. */ + void (*disable_write)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* Block file descriptor read event notifications. */ + void (*block_read)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* Block file descriptor write event notifications. */ + void (*block_write)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Add a file descriptor to an event set and enable an oneshot + * read event notification method. + */ + void (*oneshot_read)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Add a file descriptor to an event set and enable an oneshot + * write event notification method. + */ + void (*oneshot_write)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Add a listening socket descriptor to an event set and enable + * a level-triggered read event notification method. + */ + void (*enable_accept)(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); + + /* + * Add a file to an event set and enable a file change notification + * events. + */ + void (*enable_file)(nxt_event_set_t *event_set, + nxt_event_file_t *fev); + + /* + * Delete a file from an event set before closing the file descriptor. + */ + void (*close_file)(nxt_event_set_t *event_set, + nxt_event_file_t *fev); + + /* + * Enable post event notifications and set a post handler to handle + * the zero signal. + */ + nxt_int_t (*enable_post)(nxt_event_set_t *event_set, + nxt_work_handler_t handler); + + /* + * Signal an event set. If a signal number is non-zero then + * a signal handler added to the event set is called. This is + * a way to route Unix signals to an event engine if underlying + * event facility does not support signal events. + * + * If a signal number is zero, then the post_handler of the event + * set is called. This has no relation to Unix signals but is + * a way to wake up the event set to process works posted to + * the event engine locked work queue. + */ + void (*signal)(nxt_event_set_t *event_set, + nxt_uint_t signo); + + /* Poll an event set for new event notifications. */ + void (*poll)(nxt_thread_t *thr, + nxt_event_set_t *event_set, + nxt_msec_t timeout); + + /* I/O operations suitable to underlying event facility. */ + nxt_event_conn_io_t *io; + + /* True if an event facility supports file change event notifications. */ + uint8_t file_support; /* 1 bit */ + + /* True if an event facility supports signal event notifications. */ + uint8_t signal_support; /* 1 bit */ +} nxt_event_set_ops_t; + + +#if (NXT_HAVE_KQUEUE) + +typedef struct { + int kqueue; + int nchanges; + int mchanges; + int mevents; + nxt_pid_t pid; + + nxt_work_handler_t post_handler; + + struct kevent *changes; + struct kevent *events; +} nxt_kqueue_event_set_t; + +extern const nxt_event_set_ops_t nxt_kqueue_event_set; + +#endif + + +#if (NXT_HAVE_EPOLL) + +typedef struct { + int op; + /* + * Although file descriptor can be obtained using pointer to a + * nxt_event_fd_t stored in event.data.ptr, nevertheless storing + * the descriptor right here avoid cache miss. Besides this costs + * no space because event.data must be anyway aligned to 64 bits. + */ + nxt_socket_t fd; + + struct epoll_event event; +} nxt_epoll_change_t; + + +typedef struct { + int epoll; + uint32_t mode; + nxt_uint_t nchanges; + nxt_uint_t mchanges; + int mevents; + + nxt_epoll_change_t *changes; + struct epoll_event *events; + +#if (NXT_HAVE_EVENTFD) + nxt_work_handler_t post_handler; + nxt_event_fd_t eventfd; + uint32_t neventfd; +#endif + +#if (NXT_HAVE_SIGNALFD) + nxt_event_fd_t signalfd; +#endif +} nxt_epoll_event_set_t; + + +extern const nxt_event_set_ops_t nxt_epoll_edge_event_set; +extern const nxt_event_set_ops_t nxt_epoll_level_event_set; + +#endif + + +#if (NXT_HAVE_EVENTPORT) + +typedef struct { + /* + * Although file descriptor can be obtained using pointer to a + * nxt_event_fd_t, nevertheless storing the descriptor right here + * avoid cache miss. Besides this costs no space on 64-bit platform. + */ + nxt_socket_t fd; + + int events; + nxt_event_fd_t *event; +} nxt_eventport_change_t; + + +typedef struct { + int port; + nxt_uint_t nchanges; + nxt_uint_t mchanges; + u_int mevents; + + nxt_eventport_change_t *changes; + port_event_t *events; + + nxt_work_handler_t post_handler; + nxt_work_handler_t signal_handler; +} nxt_eventport_event_set_t; + +extern const nxt_event_set_ops_t nxt_eventport_event_set; + +#endif + + +#if (NXT_HAVE_DEVPOLL) + +typedef struct { + uint8_t op; + short events; + + /* A file descriptor stored because nxt_event_fd_t may be already freed. */ + nxt_socket_t fd; + + nxt_event_fd_t *event; +} nxt_devpoll_change_t; + + +typedef struct { + int devpoll; + int nchanges; + int mchanges; + int mevents; + + nxt_devpoll_change_t *devpoll_changes; + struct pollfd *changes; + struct pollfd *events; + nxt_lvlhsh_t fd_hash; +} nxt_devpoll_event_set_t; + +extern const nxt_event_set_ops_t nxt_devpoll_event_set; + +#endif + + +#if (NXT_HAVE_POLLSET) + +typedef struct { + uint8_t op; + uint8_t cmd; + short events; + + /* A file descriptor stored because nxt_event_fd_t may be already freed. */ + nxt_socket_t fd; + + nxt_event_fd_t *event; +} nxt_pollset_change_t; + + +typedef struct { + pollset_t pollset; + int nchanges; + int mchanges; + int mevents; + + nxt_pollset_change_t *pollset_changes; + struct poll_ctl *changes; + struct pollfd *events; + nxt_lvlhsh_t fd_hash; +} nxt_pollset_event_set_t; + +extern const nxt_event_set_ops_t nxt_pollset_event_set; + +#endif + + +typedef struct { + uint8_t op; + short events; + + /* A file descriptor stored because nxt_event_fd_t may be already freed. */ + nxt_socket_t fd; + + nxt_event_fd_t *event; +} nxt_poll_change_t; + + +typedef struct { + nxt_uint_t max_nfds; + nxt_uint_t nfds; + + nxt_uint_t nchanges; + nxt_uint_t mchanges; + + nxt_poll_change_t *changes; + struct pollfd *poll_set; + + nxt_lvlhsh_t fd_hash; +} nxt_poll_event_set_t; + +extern const nxt_event_set_ops_t nxt_poll_event_set; + + +typedef struct { + int nfds; + uint32_t update_nfds; /* 1 bit */ + + nxt_event_fd_t **events; + + fd_set main_read_fd_set; + fd_set main_write_fd_set; + fd_set work_read_fd_set; + fd_set work_write_fd_set; +} nxt_select_event_set_t; + +extern const nxt_event_set_ops_t nxt_select_event_set; + + +union nxt_event_set_u { +#if (NXT_HAVE_KQUEUE) + nxt_kqueue_event_set_t kqueue; +#endif +#if (NXT_HAVE_EPOLL) + nxt_epoll_event_set_t epoll; +#endif +#if (NXT_HAVE_EVENTPORT) + nxt_eventport_event_set_t eventport; +#endif +#if (NXT_HAVE_DEVPOLL) + nxt_devpoll_event_set_t devpoll; +#endif +#if (NXT_HAVE_POLLSET) + nxt_pollset_event_set_t pollset; +#endif + nxt_poll_event_set_t poll; + nxt_select_event_set_t select; +}; + + +nxt_int_t nxt_event_set_fd_hash_add(nxt_lvlhsh_t *lh, nxt_fd_t fd, + nxt_event_fd_t *ev); +void *nxt_event_set_fd_hash_get(nxt_lvlhsh_t *lh, nxt_fd_t fd); +void nxt_event_set_fd_hash_delete(nxt_lvlhsh_t *lh, nxt_fd_t fd, + nxt_bool_t ignore); +void nxt_event_set_fd_hash_destroy(nxt_lvlhsh_t *lh); + + +#define \ +nxt_event_fd_disable(engine, ev) \ + (engine)->event->disable((engine)->event_set, ev) + + +#define \ +nxt_event_fd_close(engine, ev) \ + (engine)->event->close((engine)->event_set, ev) + + +#define \ +nxt_event_fd_enable_read(engine, ev) \ + (engine)->event->enable_read((engine)->event_set, ev) + + +#define \ +nxt_event_fd_enable_write(engine, ev) \ + (engine)->event->enable_write((engine)->event_set, ev) + + +#define \ +nxt_event_fd_disable_read(engine, ev) \ + (engine)->event->disable_read((engine)->event_set, ev) + + +#define \ +nxt_event_fd_disable_write(engine, ev) \ + (engine)->event->disable_write((engine)->event_set, ev) + + +#define \ +nxt_event_fd_block_read(engine, ev) \ + do { \ + if (nxt_event_fd_is_active((ev)->read)) { \ + (engine)->event->block_read((engine)->event_set, ev); \ + } \ + } while (0) + + +#define \ +nxt_event_fd_block_write(engine, ev) \ + do { \ + if (nxt_event_fd_is_active((ev)->write)) { \ + (engine)->event->block_write((engine)->event_set, ev); \ + } \ + } while (0) + + +#define \ +nxt_event_fd_oneshot_read(engine, ev) \ + (engine)->event->oneshot_read((engine)->event_set, ev) + + +#define \ +nxt_event_fd_oneshot_write(engine, ev) \ + (engine)->event->oneshot_write((engine)->event_set, ev) + + +#define \ +nxt_event_fd_enable_accept(engine, ev) \ + (engine)->event->enable_accept((engine)->event_set, ev) + + +#endif /* _NXT_EVENT_SET_H_INCLUDED_ */ diff --git a/src/nxt_event_timer.c b/src/nxt_event_timer.c new file mode 100644 index 00000000..29c225de --- /dev/null +++ b/src/nxt_event_timer.c @@ -0,0 +1,320 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * Timer operations are batched to improve instruction and data + * cache locality of rbtree operations. + * + * nxt_event_timer_add() adds a timer to the changes array to add or to + * modify the timer. The changes are processed by nxt_event_timer_find(). + * + * nxt_event_timer_disable() disables a timer. The disabled timer may + * however present in rbtree for a long time and may be eventually removed + * by nxt_event_timer_find() or nxt_event_timer_expire(). + * + * nxt_event_timer_delete() removes a timer at once from both the rbtree and + * the changes array and should be used only if the timer memory must be freed. + */ + +static nxt_int_t nxt_event_timer_rbtree_compare(nxt_rbtree_node_t *node1, + nxt_rbtree_node_t *node2); +static void nxt_event_timer_change(nxt_event_timers_t *timers, + nxt_event_timer_t *ev, nxt_msec_t time); +static void nxt_event_commit_timer_changes(nxt_event_timers_t *timers); +static void nxt_event_timer_drop_changes(nxt_event_timers_t *timers, + nxt_event_timer_t *ev); + + +nxt_int_t +nxt_event_timers_init(nxt_event_timers_t *timers, nxt_uint_t mchanges) +{ + nxt_rbtree_init(&timers->tree, nxt_event_timer_rbtree_compare, NULL); + + timers->mchanges = mchanges; + + timers->changes = nxt_malloc(sizeof(nxt_event_timer_change_t) * mchanges); + + if (nxt_fast_path(timers->changes != NULL)) { + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_event_timer_rbtree_compare(nxt_rbtree_node_t *node1, + nxt_rbtree_node_t *node2) +{ + nxt_event_timer_t *ev1, *ev2; + + ev1 = (nxt_event_timer_t *) node1; + ev2 = (nxt_event_timer_t *) node2; + + /* + * Timer values are distributed in small range, usually several minutes + * and overflow every 49 days if nxt_msec_t is stored in 32 bits. + * This signed comparison takes into account that overflow. + */ + /* ev1->time < ev2->time */ + return nxt_msec_diff(ev1->time, ev2->time); +} + + +void +nxt_event_timer_add(nxt_event_engine_t *engine, nxt_event_timer_t *ev, + nxt_msec_t timer) +{ + int32_t diff; + uint32_t time; + + time = engine->timers.now + timer; + + if (nxt_event_timer_is_in_tree(ev)) { + + diff = nxt_msec_diff(time, ev->time); + + /* + * Use the previous timer if difference between it and the + * new timer is less than required precision milliseconds: + * this decreases rbtree operations for fast connections. + */ + + if (nxt_abs(diff) < ev->precision) { + nxt_log_debug(ev->log, "event timer previous: %D: %d:%M", + ev->ident, ev->state, time); + + if (ev->state == NXT_EVENT_TIMER_DISABLED) { + ev->state = NXT_EVENT_TIMER_ACTIVE; + } + + return; + } + + nxt_log_debug(ev->log, "event timer change: %D: %d:%M", + ev->ident, ev->state, ev->time); + + } else { + /* + * The timer's time is updated here just to log a correct + * value by debug logging in nxt_event_timer_disable(). + * It could be updated only in nxt_event_commit_timer_changes() + * just before nxt_rbtree_insert(). + */ + ev->time = time; + + nxt_log_debug(ev->log, "event timer add: %D: %M:%M", + ev->ident, timer, time); + } + + nxt_event_timer_change(&engine->timers, ev, time); +} + + +static void +nxt_event_timer_change(nxt_event_timers_t *timers, nxt_event_timer_t *ev, + nxt_msec_t time) +{ + nxt_event_timer_change_t *ch; + + if (timers->nchanges >= timers->mchanges) { + nxt_event_commit_timer_changes(timers); + } + + ev->state = NXT_EVENT_TIMER_ACTIVE; + + ch = &timers->changes[timers->nchanges]; + ch->time = time; + ch->event = ev; + timers->nchanges++; +} + + +#if (NXT_DEBUG) + +void +nxt_event_timer_disable(nxt_event_timer_t *ev) +{ + nxt_log_debug(ev->log, "event timer disable: %D: %d:%M", + ev->ident, ev->state, ev->time); + + ev->state = NXT_EVENT_TIMER_DISABLED; +} + +#endif + + +void +nxt_event_timer_delete(nxt_event_engine_t *engine, nxt_event_timer_t *ev) +{ + if (nxt_event_timer_is_in_tree(ev)) { + nxt_log_debug(ev->log, "event timer delete: %D: %d:%M", + ev->ident, ev->state, ev->time); + + nxt_rbtree_delete(&engine->timers.tree, &ev->node); + nxt_event_timer_in_tree_clear(ev); + ev->state = NXT_EVENT_TIMER_DISABLED; + } + + nxt_event_timer_drop_changes(&engine->timers, ev); +} + + +static void +nxt_event_timer_drop_changes(nxt_event_timers_t *timers, nxt_event_timer_t *ev) +{ + nxt_event_timer_change_t *dst, *src, *end; + + dst = timers->changes; + end = dst + timers->nchanges; + + for (src = dst; src < end; src++) { + + if (src->event == ev) { + continue; + } + + if (dst != src) { + *dst = *src; + } + + dst++; + } + + timers->nchanges -= end - dst; +} + + +static void +nxt_event_commit_timer_changes(nxt_event_timers_t *timers) +{ + nxt_event_timer_t *ev; + nxt_event_timer_change_t *ch, *end; + + nxt_thread_log_debug("event timers changes: %ui", timers->nchanges); + + ch = timers->changes; + end = ch + timers->nchanges; + + while (ch < end) { + ev = ch->event; + + if (ev->state != NXT_EVENT_TIMER_DISABLED) { + + if (nxt_event_timer_is_in_tree(ev)) { + nxt_log_debug(ev->log, "event timer delete: %D: %d:%M", + ev->ident, ev->state, ev->time); + + nxt_rbtree_delete(&timers->tree, &ev->node); + + ev->time = ch->time; + } + + nxt_log_debug(ev->log, "event timer add: %D: %M", + ev->ident, ev->time); + + nxt_rbtree_insert(&timers->tree, &ev->node); + nxt_event_timer_in_tree_set(ev); + } + + ch++; + } + + timers->nchanges = 0; +} + + +nxt_msec_t +nxt_event_timer_find(nxt_event_engine_t *engine) +{ + int32_t time; + nxt_rbtree_node_t *node, *next; + nxt_event_timer_t *ev; + + if (engine->timers.nchanges != 0) { + nxt_event_commit_timer_changes(&engine->timers); + } + + for (node = nxt_rbtree_min(&engine->timers.tree); + nxt_rbtree_is_there_successor(&engine->timers.tree, node); + node = next) + { + next = nxt_rbtree_node_successor(&engine->timers.tree, node); + + ev = (nxt_event_timer_t *) node; + + if (ev->state != NXT_EVENT_TIMER_DISABLED) { + + if (ev->state == NXT_EVENT_TIMER_BLOCKED) { + nxt_log_debug(ev->log, "event timer blocked: %D: %M", + ev->ident, ev->time); + continue; + } + + time = nxt_msec_diff(ev->time, engine->timers.now); + + return (nxt_msec_t) nxt_max(time, 0); + } + + /* Delete disabled timer. */ + + nxt_log_debug(ev->log, "event timer delete: %D: 0:%M", + ev->ident, ev->time); + + nxt_rbtree_delete(&engine->timers.tree, &ev->node); + nxt_event_timer_in_tree_clear(ev); + } + + return NXT_INFINITE_MSEC; +} + + +void +nxt_event_timer_expire(nxt_thread_t *thr, nxt_msec_t now) +{ + nxt_rbtree_t *tree; + nxt_rbtree_node_t *node, *next; + nxt_event_timer_t *ev; + + thr->engine->timers.now = now; + tree = &thr->engine->timers.tree; + + for (node = nxt_rbtree_min(tree); + nxt_rbtree_is_there_successor(tree, node); + node = next) + { + ev = (nxt_event_timer_t *) node; + + /* ev->time > now */ + if (nxt_msec_diff(ev->time, now) > 0) { + return; + } + + next = nxt_rbtree_node_successor(tree, node); + + if (ev->state == NXT_EVENT_TIMER_BLOCKED) { + nxt_log_debug(ev->log, "event timer blocked: %D: %M", + ev->ident, ev->time); + continue; + } + + nxt_log_debug(ev->log, "event timer delete: %D: %d:%M", + ev->ident, ev->state, ev->time); + + nxt_rbtree_delete(tree, &ev->node); + nxt_event_timer_in_tree_clear(ev); + + if (ev->state != NXT_EVENT_TIMER_DISABLED) { + ev->state = NXT_EVENT_TIMER_DISABLED; + + nxt_thread_work_queue_add(thr, ev->work_queue, ev->handler, + ev, NULL, ev->log); + } + } +} diff --git a/src/nxt_event_timer.h b/src/nxt_event_timer.h new file mode 100644 index 00000000..c20c2c17 --- /dev/null +++ b/src/nxt_event_timer.h @@ -0,0 +1,146 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_EVENT_TIMER_H_INCLUDED_ +#define _NXT_EVENT_TIMER_H_INCLUDED_ + + +/* Valid values are between 1ms to 255ms. */ +#define NXT_EVENT_TIMER_DEFAULT_PRECISION 100 +//#define NXT_EVENT_TIMER_DEFAULT_PRECISION 1 + + +#if (NXT_DEBUG) +#define NXT_EVENT_TIMER { NXT_RBTREE_NODE_INIT, 0, 0, 0, \ + NULL, NULL, NULL, -1 } + +#else +#define NXT_EVENT_TIMER { NXT_RBTREE_NODE_INIT, 0, 0, 0, \ + NULL, NULL, NULL } +#endif + + +typedef struct { + /* The rbtree node must be the first field. */ + NXT_RBTREE_NODE (node); + + uint8_t state; + uint8_t precision; + nxt_msec_t time; + + nxt_work_queue_t *work_queue; + nxt_work_handler_t handler; + + nxt_log_t *log; +#if (NXT_DEBUG) + int32_t ident; +#endif +} nxt_event_timer_t; + + +typedef struct { + nxt_msec_t time; + nxt_event_timer_t *event; +} nxt_event_timer_change_t; + + +typedef struct { + nxt_rbtree_t tree; + + /* An overflown milliseconds counter. */ + nxt_msec_t now; + + nxt_uint_t mchanges; + nxt_uint_t nchanges; + + nxt_event_timer_change_t *changes; +} nxt_event_timers_t; + + +#define \ +nxt_event_timer_data(ev, type, timer) \ + nxt_container_of(ev, type, timer) + + +/* + * When timer resides in rbtree all links of its node are not NULL. + * A parent link is the nearst to other timer flags. + */ + +#define \ +nxt_event_timer_is_in_tree(ev) \ + ((ev)->node.parent != NULL) + +#define \ +nxt_event_timer_in_tree_set(ev) + /* Noop, because rbtree insertion sets a node's parent link. */ + +#define \ +nxt_event_timer_in_tree_clear(ev) \ + (ev)->node.parent = NULL + + +#define NXT_EVENT_TIMER_DISABLED 0 +#define NXT_EVENT_TIMER_BLOCKED 1 +#define NXT_EVENT_TIMER_ACTIVE 2 + + +#if (NXT_DEBUG) + +#define \ +nxt_event_timer_ident(ev, val) \ + (ev)->ident = (val) + +#else + +#define \ +nxt_event_timer_ident(ev, val) + +#endif + + +nxt_inline nxt_event_timer_t * +nxt_event_timer_create(int32_t ident) +{ + nxt_event_timer_t *ev; + + ev = nxt_zalloc(sizeof(nxt_event_timer_t)); + if (ev == NULL) { + return NULL; + } + + ev->precision = NXT_EVENT_TIMER_DEFAULT_PRECISION; +#if (NXT_DEBUG) + ev->ident = ident; +#endif + + return ev; +} + + +nxt_int_t nxt_event_timers_init(nxt_event_timers_t *timers, + nxt_uint_t mchanges); +NXT_EXPORT void nxt_event_timer_add(nxt_event_engine_t *engine, + nxt_event_timer_t *ev, nxt_msec_t timer); +NXT_EXPORT void nxt_event_timer_delete(nxt_event_engine_t *engine, + nxt_event_timer_t *ev); +nxt_msec_t nxt_event_timer_find(nxt_event_engine_t *engine); +void nxt_event_timer_expire(nxt_thread_t *thr, nxt_msec_t now); + +#if (NXT_DEBUG) + +NXT_EXPORT void nxt_event_timer_disable(nxt_event_timer_t *ev); + +#else + +#define \ +nxt_event_timer_disable(ev) \ + (ev)->state = NXT_EVENT_TIMER_DISABLED + +#endif + + +#endif /* _NXT_EVENT_TIMER_H_INCLUDED_ */ diff --git a/src/nxt_eventport.c b/src/nxt_eventport.c new file mode 100644 index 00000000..02984573 --- /dev/null +++ b/src/nxt_eventport.c @@ -0,0 +1,646 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * The event ports have been introduced in Solaris 10. + * The PORT_SOURCE_MQ and PORT_SOURCE_FILE sources have + * been added in OpenSolaris. + */ + + +static nxt_event_set_t *nxt_eventport_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +static void nxt_eventport_free(nxt_event_set_t *event_set); +static void nxt_eventport_enable(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_disable(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_eventport_drop_changes(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_enable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_enable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_enable_event(nxt_event_set_t *event_set, + nxt_event_fd_t *ev, nxt_uint_t events); +static void nxt_eventport_disable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_disable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_disable_event(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static nxt_int_t nxt_eventport_commit_changes(nxt_thread_t *thr, + nxt_eventport_event_set_t *es); +static void nxt_eventport_error_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_eventport_block_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_block_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_oneshot_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_oneshot_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_eventport_enable_accept(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static nxt_int_t nxt_eventport_enable_post(nxt_event_set_t *event_set, + nxt_work_handler_t handler); +static void nxt_eventport_signal(nxt_event_set_t *event_set, nxt_uint_t signo); +static void nxt_eventport_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout); + + +const nxt_event_set_ops_t nxt_eventport_event_set = { + "eventport", + nxt_eventport_create, + nxt_eventport_free, + nxt_eventport_enable, + nxt_eventport_disable, + nxt_eventport_disable, + nxt_eventport_close, + nxt_eventport_enable_read, + nxt_eventport_enable_write, + nxt_eventport_disable_read, + nxt_eventport_disable_write, + nxt_eventport_block_read, + nxt_eventport_block_write, + nxt_eventport_oneshot_read, + nxt_eventport_oneshot_write, + nxt_eventport_enable_accept, + NULL, + NULL, + nxt_eventport_enable_post, + nxt_eventport_signal, + nxt_eventport_poll, + + &nxt_unix_event_conn_io, + + NXT_NO_FILE_EVENTS, + NXT_NO_SIGNAL_EVENTS, +}; + + +static nxt_event_set_t * +nxt_eventport_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + nxt_event_set_t *event_set; + nxt_eventport_event_set_t *es; + + event_set = nxt_zalloc(sizeof(nxt_eventport_event_set_t)); + if (event_set == NULL) { + return NULL; + } + + es = &event_set->eventport; + + es->port = -1; + es->mchanges = mchanges; + es->mevents = mevents; + + es->changes = nxt_malloc(sizeof(nxt_eventport_change_t) * mchanges); + if (es->changes == NULL) { + goto fail; + } + + es->events = nxt_malloc(sizeof(port_event_t) * mevents); + if (es->events == NULL) { + goto fail; + } + + es->port = port_create(); + if (es->port == -1) { + nxt_main_log_emerg("port_create() failed %E", nxt_errno); + goto fail; + } + + nxt_main_log_debug("port_create(): %d", es->port); + + if (signals != NULL) { + es->signal_handler = signals->handler; + } + + return event_set; + +fail: + + nxt_eventport_free(event_set); + + return NULL; +} + + +static void +nxt_eventport_free(nxt_event_set_t *event_set) +{ + nxt_eventport_event_set_t *es; + + es = &event_set->eventport; + + nxt_main_log_debug("eventport %d free", es->port); + + if (es->port != -1) { + if (close(es->port) != 0) { + nxt_main_log_emerg("eventport close(%d) failed %E", + es->port, nxt_errno); + } + } + + nxt_free(es->events); + nxt_free(es); +} + + +static void +nxt_eventport_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_DEFAULT; + ev->write = NXT_EVENT_DEFAULT; + + nxt_eventport_enable_event(event_set, ev, POLLIN | POLLOUT); +} + + +static void +nxt_eventport_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE || ev->write != NXT_EVENT_INACTIVE) { + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_eventport_disable_event(event_set, ev); + } +} + + +static void +nxt_eventport_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_eventport_drop_changes(event_set, ev); +} + + +static void +nxt_eventport_drop_changes(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_eventport_change_t *dst, *src, *end; + nxt_eventport_event_set_t *es; + + es = &event_set->eventport; + + dst = es->changes; + end = dst + es->nchanges; + + for (src = dst; src < end; src++) { + + if (src->event == ev) { + continue; + } + + if (dst != src) { + *dst = *src; + } + + dst++; + } + + es->nchanges -= end - dst; +} + + +static void +nxt_eventport_enable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t events; + + if (ev->read != NXT_EVENT_BLOCKED) { + events = (ev->write == NXT_EVENT_INACTIVE) ? POLLIN: + (POLLIN | POLLOUT); + nxt_eventport_enable_event(event_set, ev, events); + } + + ev->read = NXT_EVENT_DEFAULT; +} + + +static void +nxt_eventport_enable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t events; + + if (ev->write != NXT_EVENT_BLOCKED) { + events = (ev->read == NXT_EVENT_INACTIVE) ? POLLOUT: + (POLLIN | POLLOUT); + nxt_eventport_enable_event(event_set, ev, events); + } + + ev->write = NXT_EVENT_DEFAULT; +} + + +/* + * eventport changes are batched to improve instruction and data + * cache locality of several port_associate() and port_dissociate() + * calls followed by port_getn() call. + */ + +static void +nxt_eventport_enable_event(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_uint_t events) +{ + nxt_eventport_change_t *ch; + nxt_eventport_event_set_t *es; + + es = &event_set->eventport; + + nxt_log_debug(ev->log, "port %d set event: fd:%d ev:%04XD u:%p", + es->port, ev->fd, events, ev); + + if (es->nchanges >= es->mchanges) { + (void) nxt_eventport_commit_changes(nxt_thread(), es); + } + + ch = &es->changes[es->nchanges++]; + ch->fd = ev->fd; + ch->events = events; + ch->event = ev; +} + + +static void +nxt_eventport_disable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + + if (ev->write == NXT_EVENT_INACTIVE) { + nxt_eventport_disable_event(event_set, ev); + + } else { + nxt_eventport_enable_event(event_set, ev, POLLOUT); + } +} + + +static void +nxt_eventport_disable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->write = NXT_EVENT_INACTIVE; + + if (ev->read == NXT_EVENT_INACTIVE) { + nxt_eventport_disable_event(event_set, ev); + + } else { + nxt_eventport_enable_event(event_set, ev, POLLIN); + } +} + + +static void +nxt_eventport_disable_event(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_eventport_change_t *ch; + nxt_eventport_event_set_t *es; + + es = &event_set->eventport; + + nxt_log_debug(ev->log, "port %d disable event : fd:%d", es->port, ev->fd); + + if (es->nchanges >= es->mchanges) { + (void) nxt_eventport_commit_changes(nxt_thread(), es); + } + + ch = &es->changes[es->nchanges++]; + ch->fd = ev->fd; + ch->events = 0; + ch->event = ev; +} + + +static nxt_int_t +nxt_eventport_commit_changes(nxt_thread_t *thr, nxt_eventport_event_set_t *es) +{ + int ret; + nxt_int_t retval; + nxt_event_fd_t *ev; + nxt_eventport_change_t *ch, *end; + + nxt_log_debug(thr->log, "eventport %d changes:%ui", es->port, es->nchanges); + + retval = NXT_OK; + ch = es->changes; + end = ch + es->nchanges; + + do { + ev = ch->event; + + if (ch->events != 0) { + nxt_log_debug(ev->log, "port_associate(%d): fd:%d ev:%04XD u:%p", + es->port, ch->fd, ch->events, ev); + + ret = port_associate(es->port, PORT_SOURCE_FD, ch->fd, + ch->events, ev); + if (ret == 0) { + goto next; + } + + nxt_log_alert(ev->log, + "port_associate(%d, %d, %d, %04XD) failed %E", + es->port, PORT_SOURCE_FD, ch->fd, ch->events, + nxt_errno); + + } else { + nxt_log_debug(ev->log, "port_dissociate(%d): fd:%d", + es->port, ch->fd); + + if (port_dissociate(es->port, PORT_SOURCE_FD, ch->fd) == 0) { + goto next; + } + + nxt_log_alert(ev->log, "port_dissociate(%d, %d, %d) failed %E", + es->port, PORT_SOURCE_FD, ch->fd, nxt_errno); + } + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_eventport_error_handler, + ev, ev->data, ev->log); + + retval = NXT_ERROR; + + next: + + ch++; + + } while (ch < end); + + es->nchanges = 0; + + return retval; +} + + +static void +nxt_eventport_error_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_fd_t *ev; + + ev = obj; + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + ev->error_handler(thr, ev, data); +} + + +static void +nxt_eventport_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_eventport_block_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write != NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_eventport_oneshot_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read == NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_DEFAULT; + + nxt_eventport_enable_event(event_set, ev, POLLIN); + } +} + + +static void +nxt_eventport_oneshot_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write == NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_DEFAULT; + + nxt_eventport_enable_event(event_set, ev, POLLOUT); + } +} + + +static void +nxt_eventport_enable_accept(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_LEVEL; + + nxt_eventport_enable_event(event_set, ev, POLLIN); +} + + +static nxt_int_t +nxt_eventport_enable_post(nxt_event_set_t *event_set, + nxt_work_handler_t handler) +{ + event_set->eventport.post_handler = handler; + + return NXT_OK; +} + + +static void +nxt_eventport_signal(nxt_event_set_t *event_set, nxt_uint_t signo) +{ + nxt_eventport_event_set_t *es; + + es = &event_set->eventport; + + nxt_thread_log_debug("port_send(%d, %ui)", es->port, signo); + + if (port_send(es->port, signo, NULL) != 0) { + nxt_thread_log_alert("port_send(%d) failed %E", es->port, nxt_errno); + } +} + + +static void +nxt_eventport_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout) +{ + int n, events, signo; + uint_t nevents; + nxt_err_t err; + nxt_uint_t i, level; + timespec_t ts, *tp; + port_event_t *event; + nxt_event_fd_t *ev; + nxt_work_handler_t handler; + nxt_eventport_event_set_t *es; + + es = &event_set->eventport; + + if (es->nchanges != 0) { + if (nxt_eventport_commit_changes(thr, es) != NXT_OK) { + /* Error handlers have been enqueued on failure. */ + timeout = 0; + } + } + + if (timeout == NXT_INFINITE_MSEC) { + tp = NULL; + + } else { + ts.tv_sec = timeout / 1000; + ts.tv_nsec = (timeout % 1000) * 1000000; + tp = &ts; + } + + nxt_log_debug(thr->log, "port_getn(%d) timeout: %M", es->port, timeout); + + /* + * A trap for possible error when Solaris does not update nevents + * if ETIME or EINTR is returned. This issue will be logged as + * "unexpected port_getn() event". + * + * The details are in OpenSolaris mailing list thread "port_getn() + * and timeouts - is this a bug or an undocumented feature?" + */ + event = &es->events[0]; + event->portev_events = -1; /* invalid port events */ + event->portev_source = -1; /* invalid port source */ + event->portev_object = -1; + event->portev_user = (void *) -1; + + nevents = 1; + n = port_getn(es->port, es->events, es->mevents, &nevents, tp); + + /* + * 32-bit port_getn() on Solaris 10 x86 returns large negative + * values instead of 0 when returning immediately. + */ + err = (n < 0) ? nxt_errno : 0; + + nxt_thread_time_update(thr); + + if (n == -1) { + if (err == NXT_ETIME || err == NXT_EINTR) { + if (nevents != 0) { + nxt_log_alert(thr->log, "port_getn(%d) failed %E, events:%ud", + es->port, err, nevents); + } + } + + if (err != NXT_ETIME) { + level = (err == NXT_EINTR) ? NXT_LOG_INFO : NXT_LOG_ALERT; + nxt_log_error(level, thr->log, "port_getn(%d) failed %E", + es->port, err); + + if (err != NXT_EINTR) { + return; + } + } + } + + nxt_log_debug(thr->log, "port_getn(%d) events: %d", es->port, nevents); + + for (i = 0; i < nevents; i++) { + event = &es->events[i]; + + switch (event->portev_source) { + + case PORT_SOURCE_FD: + ev = event->portev_user; + events = event->portev_events; + + nxt_log_debug(ev->log, "eventport: fd:%d ev:%04Xd u:%p rd:%d wr:%d", + event->portev_object, events, ev, + ev->read, ev->write); + + if (nxt_slow_path(events & (POLLERR | POLLHUP | POLLNVAL)) != 0) { + nxt_log_alert(ev->log, + "port_getn(%d) error fd:%d events:%04Xud", + es->port, ev->fd, events); + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_eventport_error_handler, + ev, ev->data, ev->log); + continue; + } + + if (events & POLLIN) { + ev->read_ready = 1; + + if (ev->read != NXT_EVENT_BLOCKED) { + nxt_thread_work_queue_add(thr, ev->read_work_queue, + ev->read_handler, + ev, ev->data, ev->log); + + } + + if (ev->read != NXT_EVENT_LEVEL) { + ev->read = NXT_EVENT_INACTIVE; + } + } + + if (events & POLLOUT) { + ev->write_ready = 1; + + if (ev->write != NXT_EVENT_BLOCKED) { + nxt_thread_work_queue_add(thr, ev->write_work_queue, + ev->write_handler, + ev, ev->data, ev->log); + } + + ev->write = NXT_EVENT_INACTIVE; + } + + /* + * Reactivate counterpart direction, because the + * eventport is oneshot notification facility. + */ + events = (ev->read == NXT_EVENT_INACTIVE) ? 0 : POLLIN; + events |= (ev->write == NXT_EVENT_INACTIVE) ? 0 : POLLOUT; + + if (events != 0) { + nxt_eventport_enable_event(event_set, ev, events); + } + + break; + + case PORT_SOURCE_USER: + nxt_log_debug(thr->log, "eventport: user ev:%d u:%p", + event->portev_events, event->portev_user); + + signo = event->portev_events; + + handler = (signo == 0) ? es->post_handler : es->signal_handler; + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, handler, + (void *) (uintptr_t) signo, NULL, + thr->log); + + break; + + default: + nxt_log_alert(thr->log, "unexpected port_getn(%d) event: " + "ev:%d src:%d obj:%p u:%p", + es->port, event->portev_events, + event->portev_source, event->portev_object, + event->portev_user); + } + } +} diff --git a/src/nxt_fastcgi_record_parse.c b/src/nxt_fastcgi_record_parse.c new file mode 100644 index 00000000..66b9d4c3 --- /dev/null +++ b/src/nxt_fastcgi_record_parse.c @@ -0,0 +1,307 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#define NXT_FASTCGI_DATA_MIDDLE 0 +#define NXT_FASTCGI_DATA_END_ON_BORDER 1 +#define NXT_FASTCGI_DATA_END 2 + + +static nxt_int_t nxt_fastcgi_buffer(nxt_fastcgi_parse_t *fp, nxt_buf_t ***tail, + nxt_buf_t *in); + + +void +nxt_fastcgi_record_parse(nxt_fastcgi_parse_t *fp, nxt_buf_t *in) +{ + u_char ch; + nxt_int_t ret, stream; + nxt_buf_t *b, *nb, **tail[2]; + const char *msg; + nxt_thread_t *thr; + enum { + sw_fastcgi_version = 0, + sw_fastcgi_type, + sw_fastcgi_request_id_high, + sw_fastcgi_request_id_low, + sw_fastcgi_content_length_high, + sw_fastcgi_content_length_low, + sw_fastcgi_padding_length, + sw_fastcgi_reserved, + sw_fastcgi_data, + sw_fastcgi_padding, + sw_fastcgi_end_request, + } state; + + fp->out[0] = NULL; + fp->out[1] = NULL; + + tail[0] = &fp->out[0]; + tail[1] = &fp->out[1]; + + state = fp->state; + + for (b = in; b != NULL; b = b->next) { + + if (nxt_buf_is_sync(b)) { + **tail = b; + *tail = &b->next; + continue; + } + + fp->pos = b->mem.pos; + + while (fp->pos < b->mem.free) { + /* + * The sw_fastcgi_data state is tested outside the + * switch to preserve fp->pos and to not touch memory. + */ + if (state == sw_fastcgi_data) { + + /* + * fp->type here can be only NXT_FASTCGI_STDOUT + * or NXT_FASTCGI_STDERR. NXT_FASTCGI_END_REQUEST + * is tested in sw_fastcgi_reserved. + */ + stream = fp->type - NXT_FASTCGI_STDOUT; + + ret = nxt_fastcgi_buffer(fp, &tail[stream], b); + + if (ret == NXT_FASTCGI_DATA_MIDDLE) { + goto next; + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + fp->error = 1; + goto done; + } + + if (fp->padding == 0) { + state = sw_fastcgi_version; + + } else { + state = sw_fastcgi_padding; + } + + if (ret == NXT_FASTCGI_DATA_END_ON_BORDER) { + goto next; + } + + /* ret == NXT_FASTCGI_DATA_END */ + } + + ch = *fp->pos++; + + nxt_thread_log_debug("fastcgi record byte: %02Xd", ch); + + switch (state) { + + case sw_fastcgi_version: + if (nxt_fast_path(ch == 1)) { + state = sw_fastcgi_type; + continue; + } + + msg = "unsupported FastCGI protocol version"; + goto fastcgi_error; + + case sw_fastcgi_type: + switch (ch) { + case NXT_FASTCGI_STDOUT: + case NXT_FASTCGI_STDERR: + case NXT_FASTCGI_END_REQUEST: + fp->type = ch; + state = sw_fastcgi_request_id_high; + continue; + default: + msg = "invalid FastCGI record type"; + goto fastcgi_error; + } + + case sw_fastcgi_request_id_high: + /* FastCGI multiplexing is not supported. */ + if (nxt_fast_path(ch == 0)) { + state = sw_fastcgi_request_id_low; + continue; + } + + msg = "unexpected FastCGI request ID high byte"; + goto fastcgi_error; + + case sw_fastcgi_request_id_low: + if (nxt_fast_path(ch == 1)) { + state = sw_fastcgi_content_length_high; + continue; + } + + msg = "unexpected FastCGI request ID low byte"; + goto fastcgi_error; + + case sw_fastcgi_content_length_high: + fp->length = ch << 8; + state = sw_fastcgi_content_length_low; + continue; + + case sw_fastcgi_content_length_low: + fp->length |= ch; + state = sw_fastcgi_padding_length; + continue; + + case sw_fastcgi_padding_length: + fp->padding = ch; + state = sw_fastcgi_reserved; + continue; + + case sw_fastcgi_reserved: + nxt_thread_log_debug("fastcgi record type:%d " + "length:%uz padding:%d", + fp->type, fp->length, fp->padding); + + if (nxt_fast_path(fp->type != NXT_FASTCGI_END_REQUEST)) { + state = sw_fastcgi_data; + continue; + } + + state = sw_fastcgi_end_request; + continue; + + case sw_fastcgi_data: + /* + * This state is processed before the switch. + * It added here just to suppress a warning. + */ + continue; + + case sw_fastcgi_padding: + /* + * No special fast processing of padding + * because it usually takes just 1-7 bytes. + */ + fp->padding--; + + if (fp->padding == 0) { + nxt_thread_log_debug("fastcgi record end"); + state = sw_fastcgi_version; + } + continue; + + case sw_fastcgi_end_request: + /* Just skip 8 bytes of END_REQUEST. */ + fp->length--; + + if (fp->length != 0) { + continue; + } + + fp->done = 1; + + nxt_thread_log_debug("fastcgi end request"); + + goto done; + } + } + + if (b->retain == 0) { + /* No record data was found in a buffer. */ + thr = nxt_thread(); + nxt_thread_current_work_queue_add(thr, b->completion_handler, + b, b->parent, thr->log); + } + + next: + + continue; + } + + fp->state = state; + + return; + +fastcgi_error: + + nxt_thread_log_error(NXT_LOG_ERR, "upstream sent %s: %d", msg, ch); + + fp->fastcgi_error = 1; + +done: + + nb = fp->last_buf(fp); + + if (nxt_fast_path(nb != NULL)) { + *tail[0] = nb; + + } else { + fp->error = 1; + } + + // STUB: fp->fastcgi_error = 1; + // STUB: fp->error = 1; + + return; +} + + +static nxt_int_t +nxt_fastcgi_buffer(nxt_fastcgi_parse_t *fp, nxt_buf_t ***tail, nxt_buf_t *in) +{ + u_char *p; + size_t size; + nxt_buf_t *b; + + if (fp->length == 0) { + return NXT_FASTCGI_DATA_END; + } + + p = fp->pos; + size = in->mem.free - p; + + if (fp->length >= size && in->retain == 0) { + /* + * Use original buffer if the buffer is lesser than or equal to + * FastCGI record size and this is the first record in the buffer. + */ + in->mem.pos = p; + **tail = in; + *tail = &in->next; + + } else { + b = nxt_buf_mem_alloc(fp->mem_pool, 0, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + **tail = b; + *tail = &b->next; + + b->parent = in; + in->retain++; + b->mem.pos = p; + b->mem.start = p; + + if (fp->length < size) { + p += fp->length; + fp->pos = p; + + b->mem.free = p; + b->mem.end = p; + + return NXT_FASTCGI_DATA_END; + } + + b->mem.free = in->mem.free; + b->mem.end = in->mem.free; + } + + fp->length -= size; + + if (fp->length == 0) { + return NXT_FASTCGI_DATA_END_ON_BORDER; + } + + return NXT_FASTCGI_DATA_MIDDLE; +} diff --git a/src/nxt_fastcgi_source.c b/src/nxt_fastcgi_source.c new file mode 100644 index 00000000..14c51d6f --- /dev/null +++ b/src/nxt_fastcgi_source.c @@ -0,0 +1,756 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#define NXT_FASTCGI_RESPONDER 1 +#define NXT_FASTCGI_KEEP_CONN 1 + + +typedef struct { + u_char *buf; + uint32_t len; + u_char length[4]; +} nxt_fastcgi_param_t; + + +#define \ +nxt_fastcgi_set_record_length(p, length) \ + do { \ + uint32_t len = length; \ + \ + p[1] = (u_char) len; len >>= 8; \ + p[0] = (u_char) len; \ + } while (0) + + +nxt_inline size_t +nxt_fastcgi_param_length(u_char *p, uint32_t length) +{ + if (nxt_fast_path(length < 128)) { + *p = (u_char) length; + return 1; + } + + p[3] = (u_char) length; length >>= 8; + p[2] = (u_char) length; length >>= 8; + p[1] = (u_char) length; length >>= 8; + p[0] = (u_char) (length | 0x80); + + return 4; +} + + +static nxt_buf_t *nxt_fastcgi_request_create(nxt_fastcgi_source_t *fs); +static nxt_int_t nxt_fastcgi_next_param(nxt_fastcgi_source_t *fs, + nxt_fastcgi_param_t *param); + +static void nxt_fastcgi_source_record_filter(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_fastcgi_source_record_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_fastcgi_source_header_filter(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_fastcgi_source_sync_buffer(nxt_thread_t *thr, + nxt_fastcgi_source_t *fs, nxt_buf_t *b); + +static nxt_int_t nxt_fastcgi_source_header_process(nxt_fastcgi_source_t *fs); +static nxt_int_t nxt_fastcgi_source_status(nxt_upstream_source_t *us, + nxt_name_value_t *nv); +static nxt_int_t nxt_fastcgi_source_content_length(nxt_upstream_source_t *us, + nxt_name_value_t *nv); + +static void nxt_fastcgi_source_header_ready(nxt_fastcgi_source_t *fs, + nxt_buf_t *b); +static void nxt_fastcgi_source_body_filter(nxt_thread_t *thr, void *obj, + void *data); +static nxt_buf_t *nxt_fastcgi_source_last_buf(nxt_fastcgi_parse_t *fp); +static void nxt_fastcgi_source_error(nxt_stream_source_t *stream); +static void nxt_fastcgi_source_fail(nxt_fastcgi_source_t *fs); + + +/* + * A FastCGI request: + * FCGI_BEGIN_REQUEST record; + * Several FCGI_PARAMS records, the last FCGI_PARAMS record must have + * zero content length, + * Several FCGI_STDIN records, the last FCGI_STDIN record must have + * zero content length. + */ + +static const uint8_t nxt_fastcgi_begin_request[] = { + 1, /* FastCGI version. */ + NXT_FASTCGI_BEGIN_REQUEST, /* The BEGIN_REQUEST record type. */ + 0, 1, /* Request ID. */ + 0, 8, /* Content length of the Role record. */ + 0, /* Padding length. */ + 0, /* Reserved. */ + + 0, NXT_FASTCGI_RESPONDER, /* The Responder Role. */ + 0, /* Flags. */ + 0, 0, 0, 0, 0, /* Reserved. */ +}; + + +static const uint8_t nxt_fastcgi_params_record[] = { + 1, /* FastCGI version. */ + NXT_FASTCGI_PARAMS, /* The PARAMS record type. */ + 0, 1, /* Request ID. */ + 0, 0, /* Content length. */ + 0, /* Padding length. */ + 0, /* Reserved. */ +}; + + +static const uint8_t nxt_fastcgi_stdin_record[] = { + 1, /* FastCGI version. */ + NXT_FASTCGI_STDIN, /* The STDIN record type. */ + 0, 1, /* Request ID. */ + 0, 0, /* Content length. */ + 0, /* Padding length. */ + 0, /* Reserved. */ +}; + + +void +nxt_fastcgi_source_handler(nxt_upstream_source_t *us, + nxt_fastcgi_source_request_create_t request_create) +{ + nxt_stream_source_t *stream; + nxt_fastcgi_source_t *fs; + + fs = nxt_mem_zalloc(us->buffers.mem_pool, sizeof(nxt_fastcgi_source_t)); + if (nxt_slow_path(fs == NULL)) { + goto fail; + } + + us->protocol_source = fs; + + fs->header_in.list = nxt_list_create(us->buffers.mem_pool, 8, + sizeof(nxt_name_value_t)); + if (nxt_slow_path(fs->header_in.list == NULL)) { + goto fail; + } + + fs->header_in.hash = us->header_hash; + fs->upstream = us; + fs->request_create = request_create; + + stream = us->stream; + + if (stream == NULL) { + stream = nxt_mem_zalloc(us->buffers.mem_pool, + sizeof(nxt_stream_source_t)); + if (nxt_slow_path(stream == NULL)) { + goto fail; + } + + us->stream = stream; + stream->upstream = us; + + } else { + nxt_memzero(stream, sizeof(nxt_stream_source_t)); + } + + /* + * Create the FastCGI source filter chain: + * stream source | FastCGI record filter | FastCGI HTTP header filter + */ + stream->next = &fs->query; + stream->error_handler = nxt_fastcgi_source_error; + + fs->record.next.context = fs; + fs->record.next.filter = nxt_fastcgi_source_header_filter; + + fs->record.parse.last_buf = nxt_fastcgi_source_last_buf; + fs->record.parse.data = fs; + fs->record.parse.mem_pool = us->buffers.mem_pool; + + fs->query.context = &fs->record.parse; + fs->query.filter = nxt_fastcgi_source_record_filter; + + fs->header_in.content_length = -1; + + stream->out = nxt_fastcgi_request_create(fs); + + if (nxt_fast_path(stream->out != NULL)) { + nxt_memzero(&fs->u.header, sizeof(nxt_http_split_header_parse_t)); + fs->u.header.mem_pool = fs->upstream->buffers.mem_pool; + + nxt_stream_source_connect(stream); + return; + } + +fail: + + nxt_fastcgi_source_fail(fs); +} + + +static nxt_buf_t * +nxt_fastcgi_request_create(nxt_fastcgi_source_t *fs) +{ + u_char *p, *record_length; + size_t len, size, max_record_size; + nxt_int_t ret; + nxt_buf_t *b, *req, **prev; + nxt_bool_t begin_request; + nxt_fastcgi_param_t param; + + nxt_thread_log_debug("fastcgi request"); + + begin_request = 1; + param.len = 0; + prev = &req; + +new_buffer: + + ret = nxt_buf_pool_mem_alloc(&fs->upstream->buffers, 0); + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + + b = fs->upstream->buffers.current; + fs->upstream->buffers.current = NULL; + + *prev = b; + prev = &b->next; + +new_record: + + size = b->mem.end - b->mem.free; + size = nxt_align_size(size, 8) - 8; + /* The maximal FastCGI record content size is 65535. 65528 is 64K - 8. */ + max_record_size = nxt_min(65528, size); + + p = b->mem.free; + + if (begin_request) { + /* TODO: fastcgi keep conn in flags. */ + p = nxt_cpymem(p, nxt_fastcgi_begin_request, 16); + max_record_size -= 16; + begin_request = 0; + } + + b->mem.free = nxt_cpymem(p, nxt_fastcgi_params_record, 8); + record_length = &p[4]; + size = 0; + + for ( ;; ) { + if (param.len == 0) { + ret = nxt_fastcgi_next_param(fs, ¶m); + + if (nxt_slow_path(ret != NXT_OK)) { + + if (nxt_slow_path(ret == NXT_ERROR)) { + return NULL; + } + + /* ret == NXT_DONE */ + break; + } + } + + len = max_record_size; + + if (nxt_fast_path(len >= param.len)) { + len = param.len; + param.len = 0; + + } else { + param.len -= len; + } + + nxt_thread_log_debug("fastcgi copy len:%uz", len); + + b->mem.free = nxt_cpymem(b->mem.free, param.buf, len); + + size += len; + max_record_size -= len; + + if (nxt_slow_path(param.len != 0)) { + /* The record is full. */ + + param.buf += len; + + nxt_thread_log_debug("fastcgi content size:%uz", size); + + nxt_fastcgi_set_record_length(record_length, size); + + /* The minimal size of aligned record with content is 16 bytes. */ + if (b->mem.end - b->mem.free >= 16) { + goto new_record; + } + + nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, + b->mem.pos); + goto new_buffer; + } + } + + nxt_thread_log_debug("fastcgi content size:%uz", size); + + nxt_fastcgi_set_record_length(record_length, size); + + /* A padding length. */ + size = 8 - size % 8; + record_length[2] = (u_char) size; + nxt_memzero(b->mem.free, size); + b->mem.free += size; + + nxt_thread_log_debug("fastcgi padding:%uz", size); + + if (b->mem.end - b->mem.free < 16) { + nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, b->mem.pos); + + b = nxt_buf_mem_alloc(fs->upstream->buffers.mem_pool, 16, 0); + if (nxt_slow_path(b == NULL)) { + return NULL; + } + + *prev = b; + prev = &b->next; + } + + /* The end of FastCGI params. */ + p = nxt_cpymem(b->mem.free, nxt_fastcgi_params_record, 8); + + /* The end of FastCGI stdin. */ + b->mem.free = nxt_cpymem(p, nxt_fastcgi_stdin_record, 8); + + nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, b->mem.pos); + + return req; +} + + +static nxt_int_t +nxt_fastcgi_next_param(nxt_fastcgi_source_t *fs, nxt_fastcgi_param_t *param) +{ + nxt_int_t ret; + + enum { + sw_name_length = 0, + sw_value_length, + sw_name, + sw_value, + }; + + switch (fs->state) { + + case sw_name_length: + ret = fs->request_create(fs); + + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + nxt_thread_log_debug("fastcgi param \"%V: %V\"", + &fs->u.request.name, &fs->u.request.value); + + fs->state = sw_value_length; + param->buf = param->length; + param->len = nxt_fastcgi_param_length(param->length, + fs->u.request.name.len); + break; + + case sw_value_length: + fs->state = sw_name; + param->buf = param->length; + param->len = nxt_fastcgi_param_length(param->length, + fs->u.request.value.len); + break; + + case sw_name: + fs->state = sw_value; + param->buf = fs->u.request.name.data; + param->len = fs->u.request.name.len; + break; + + case sw_value: + fs->state = sw_name_length; + param->buf = fs->u.request.value.data; + param->len = fs->u.request.value.len; + break; + } + + return NXT_OK; +} + + +static void +nxt_fastcgi_source_record_filter(nxt_thread_t *thr, void *obj, void *data) +{ + size_t size; + u_char *p; + nxt_buf_t *b, *in; + nxt_fastcgi_source_t *fs; + nxt_fastcgi_source_record_t *fsr; + + fsr = obj; + in = data; + + nxt_log_debug(thr->log, "fastcgi source record filter"); + + if (nxt_slow_path(fsr->parse.done)) { + return; + } + + nxt_fastcgi_record_parse(&fsr->parse, in); + + fs = nxt_container_of(fsr, nxt_fastcgi_source_t, record); + + if (fsr->parse.error) { + nxt_fastcgi_source_fail(fs); + return; + } + + if (fsr->parse.fastcgi_error) { + /* + * Output all parsed before a FastCGI record error and close upstream. + */ + nxt_thread_current_work_queue_add(thr, nxt_fastcgi_source_record_error, + fs, NULL, thr->log); + } + + /* Log FastCGI stderr output. */ + + for (b = fsr->parse.out[1]; b != NULL; b = b->next) { + + for (p = b->mem.free - 1; p >= b->mem.pos; p--) { + if (*p != NXT_CR && *p != NXT_LF) { + break; + } + } + + size = (p + 1) - b->mem.pos; + + if (size != 0) { + nxt_log_error(NXT_LOG_ERR, thr->log, + "upstream sent in FastCGI stderr: \"%*s\"", + size, b->mem.pos); + } + + b->completion_handler(thr, b, b->parent); + } + + /* Process FastCGI stdout output. */ + + if (fsr->parse.out[0] != NULL) { + nxt_source_filter(thr, fs->upstream->work_queue, &fsr->next, + fsr->parse.out[0]); + } +} + + +static void +nxt_fastcgi_source_record_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_fastcgi_source_t *fs; + + fs = obj; + + nxt_fastcgi_source_fail(fs); +} + + +static void +nxt_fastcgi_source_header_filter(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_int_t ret; + nxt_buf_t *b; + nxt_fastcgi_source_t *fs; + + fs = obj; + b = data; + + do { + nxt_log_debug(thr->log, "fastcgi source header filter"); + + if (nxt_slow_path(nxt_buf_is_sync(b))) { + nxt_fastcgi_source_sync_buffer(thr, fs, b); + return; + } + + for ( ;; ) { + ret = nxt_http_split_header_parse(&fs->u.header, &b->mem); + + if (nxt_slow_path(ret != NXT_OK)) { + break; + } + + ret = nxt_fastcgi_source_header_process(fs); + + if (nxt_slow_path(ret != NXT_OK)) { + break; + } + } + + if (nxt_fast_path(ret == NXT_DONE)) { + nxt_log_debug(thr->log, "fastcgi source header done"); + nxt_fastcgi_source_header_ready(fs, b); + return; + } + + if (nxt_fast_path(ret != NXT_AGAIN)) { + + if (ret != NXT_ERROR) { + /* n == NXT_DECLINED: "\r" is not followed by "\n" */ + nxt_log_error(NXT_LOG_ERR, thr->log, + "upstream sent invalid header line: \"%*s\\r...\"", + fs->u.header.parse.header_end + - fs->u.header.parse.header_name_start, + fs->u.header.parse.header_name_start); + } + + /* ret == NXT_ERROR */ + + nxt_fastcgi_source_fail(fs); + return; + } + + b = b->next; + + } while (b != NULL); +} + + +static void +nxt_fastcgi_source_sync_buffer(nxt_thread_t *thr, nxt_fastcgi_source_t *fs, + nxt_buf_t *b) +{ + if (nxt_buf_is_last(b)) { + nxt_log_error(NXT_LOG_ERR, thr->log, + "upstream closed prematurely connection"); + + } else { + nxt_log_error(NXT_LOG_ERR, thr->log, "%ui buffers %uz each are not " + "enough to process upstream response header", + fs->upstream->buffers.max, + fs->upstream->buffers.size); + } + + /* The stream source sends only the last and the nobuf sync buffer. */ + + nxt_fastcgi_source_fail(fs); +} + + +static nxt_int_t +nxt_fastcgi_source_header_process(nxt_fastcgi_source_t *fs) +{ + size_t len; + nxt_thread_t *thr; + nxt_name_value_t *nv; + nxt_lvlhsh_query_t lhq; + nxt_http_header_parse_t *hp; + nxt_upstream_name_value_t *unv; + + thr = nxt_thread(); + hp = &fs->u.header.parse; + + len = hp->header_name_end - hp->header_name_start; + + if (len > 255) { + nxt_log_error(NXT_LOG_INFO, thr->log, + "upstream sent too long header field name: \"%*s\"", + len, hp->header_name_start); + return NXT_ERROR; + } + + nv = nxt_list_add(fs->header_in.list); + if (nxt_slow_path(nv == NULL)) { + return NXT_ERROR; + } + + nv->hash = hp->header_hash; + nv->skip = 0; + nv->name_len = len; + nv->name_start = hp->header_name_start; + nv->value_len = hp->header_end - hp->header_start; + nv->value_start = hp->header_start; + + nxt_log_debug(thr->log, "http header: \"%*s: %*s\"", + nv->name_len, nv->name_start, nv->value_len, nv->value_start); + + lhq.key_hash = nv->hash; + lhq.key.len = nv->name_len; + lhq.key.data = nv->name_start; + lhq.proto = &nxt_upstream_header_hash_proto; + + if (nxt_lvlhsh_find(&fs->header_in.hash, &lhq) == NXT_OK) { + unv = lhq.value; + + if (unv->handler(fs->upstream, nv) == NXT_OK) { + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +static const nxt_upstream_name_value_t nxt_fastcgi_source_headers[] + nxt_aligned(32) = +{ + { nxt_fastcgi_source_status, + nxt_upstream_name_value("status") }, + + { nxt_fastcgi_source_content_length, + nxt_upstream_name_value("content-length") }, +}; + + +nxt_int_t +nxt_fastcgi_source_hash_create(nxt_mem_pool_t *mp, nxt_lvlhsh_t *lh) +{ + return nxt_upstream_header_hash_add(mp, lh, nxt_fastcgi_source_headers, + nxt_nitems(nxt_fastcgi_source_headers)); +} + + +static nxt_int_t +nxt_fastcgi_source_status(nxt_upstream_source_t *us, nxt_name_value_t *nv) +{ + nxt_int_t n; + nxt_str_t s; + nxt_fastcgi_source_t *fs; + + s.len = nv->value_len; + s.data = nv->value_start; + + n = nxt_str_int_parse(&s); + + if (nxt_fast_path(n > 0)) { + fs = us->protocol_source; + fs->header_in.status = n; + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_fastcgi_source_content_length(nxt_upstream_source_t *us, + nxt_name_value_t *nv) +{ + nxt_off_t length; + nxt_fastcgi_source_t *fs; + + length = nxt_off_t_parse(nv->value_start, nv->value_len); + + if (nxt_fast_path(length > 0)) { + fs = us->protocol_source; + fs->header_in.content_length = length; + return NXT_OK; + } + + return NXT_ERROR; +} + + +static void +nxt_fastcgi_source_header_ready(nxt_fastcgi_source_t *fs, nxt_buf_t *b) +{ + /* + * Change the FastCGI source filter chain: + * stream source | FastCGI record filter | FastCGI body filter + */ + fs->record.next.filter = nxt_fastcgi_source_body_filter; + + if (nxt_buf_mem_used_size(&b->mem) != 0) { + fs->rest = b; + } + + if (fs->header_in.status == 0) { + /* The "200 OK" status by default. */ + fs->header_in.status = 200; + } + + fs->upstream->state->ready_handler(fs); +} + + +/* + * The FastCGI source body filter accumulates first body buffers before the next + * filter will be established and sets completion handler for the last buffer. + */ + +static void +nxt_fastcgi_source_body_filter(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b, *in; + nxt_fastcgi_source_t *fs; + + fs = obj; + in = data; + + nxt_log_debug(thr->log, "fastcgi source body filter"); + + for (b = in; b != NULL; b = b->next) { + + if (nxt_buf_is_last(b)) { + b->data = fs->upstream->data; + b->completion_handler = fs->upstream->state->completion_handler; + } + } + + if (fs->next != NULL) { + nxt_source_filter(thr, fs->upstream->work_queue, fs->next, in); + return; + } + + nxt_buf_chain_add(&fs->rest, in); +} + + +static nxt_buf_t * +nxt_fastcgi_source_last_buf(nxt_fastcgi_parse_t *fp) +{ + nxt_buf_t *b; + nxt_fastcgi_source_t *fs; + + fs = fp->data; + + b = nxt_buf_sync_alloc(fp->mem_pool, NXT_BUF_SYNC_LAST); + + if (nxt_fast_path(b != NULL)) { + b->data = fs->upstream->data; + b->completion_handler = fs->upstream->state->completion_handler; + } + + return b; +} + + +static void +nxt_fastcgi_source_error(nxt_stream_source_t *stream) +{ + nxt_fastcgi_source_t *fs; + + nxt_thread_log_debug("fastcgi source error"); + + fs = stream->upstream->protocol_source; + + nxt_fastcgi_source_fail(fs); +} + + +static void +nxt_fastcgi_source_fail(nxt_fastcgi_source_t *fs) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + + nxt_log_debug(thr->log, "fastcgi source fail"); + + /* TODO: fail, next upstream, or bad gateway */ + + fs->upstream->state->error_handler(thr, fs, NULL); +} diff --git a/src/nxt_fastcgi_source.h b/src/nxt_fastcgi_source.h new file mode 100644 index 00000000..e02cc5f4 --- /dev/null +++ b/src/nxt_fastcgi_source.h @@ -0,0 +1,92 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_FASTCGI_SOURCE_H_INCLUDED_ +#define _NXT_FASTCGI_SOURCE_H_INCLUDED_ + + +#define NXT_FASTCGI_BEGIN_REQUEST 1 +#define NXT_FASTCGI_ABORT_REQUEST 2 +#define NXT_FASTCGI_END_REQUEST 3 +#define NXT_FASTCGI_PARAMS 4 +#define NXT_FASTCGI_STDIN 5 +#define NXT_FASTCGI_STDOUT 6 +#define NXT_FASTCGI_STDERR 7 +#define NXT_FASTCGI_DATA 8 + + +typedef struct nxt_fastcgi_parse_s nxt_fastcgi_parse_t; + +struct nxt_fastcgi_parse_s { + u_char *pos; + + uint16_t length; /* 16 bits */ + uint8_t padding; + uint8_t type; + + uint8_t state; + uint8_t fastcgi_error; /* 1 bit */ + uint8_t error; /* 1 bit */ + uint8_t done; /* 1 bit */ + + /* FastCGI stdout and stderr buffer chains. */ + nxt_buf_t *out[2]; + + nxt_buf_t *(*last_buf)(nxt_fastcgi_parse_t *fp); + void *data; + nxt_mem_pool_t *mem_pool; +}; + + +typedef struct { + nxt_fastcgi_parse_t parse; + nxt_source_hook_t next; +} nxt_fastcgi_source_record_t; + + +typedef struct { + nxt_str_t name; + nxt_str_t value; + uintptr_t data[3]; +} nxt_fastcgi_source_request_t; + + +typedef struct nxt_fastcgi_source_s nxt_fastcgi_source_t; +typedef nxt_int_t (*nxt_fastcgi_source_request_create_t)( + nxt_fastcgi_source_t *fs); + + +struct nxt_fastcgi_source_s { + nxt_source_hook_t query; + nxt_source_hook_t *next; + + nxt_upstream_source_t *upstream; + + nxt_fastcgi_source_request_create_t request_create; + + nxt_upstream_header_in_t header_in; + + nxt_buf_t *rest; + + uint32_t state; /* 2 bits */ + + nxt_fastcgi_source_record_t record; + + union { + nxt_fastcgi_source_request_t request; + nxt_http_split_header_parse_t header; + } u; +}; + + +NXT_EXPORT void nxt_fastcgi_source_handler(nxt_upstream_source_t *us, + nxt_fastcgi_source_request_create_t request_create); +NXT_EXPORT nxt_int_t nxt_fastcgi_source_hash_create(nxt_mem_pool_t *mp, + nxt_lvlhsh_t *lh); +void nxt_fastcgi_record_parse(nxt_fastcgi_parse_t *fp, nxt_buf_t *in); + + +#endif /* _NXT_FASTCGI_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_fiber.c b/src/nxt_fiber.c new file mode 100644 index 00000000..74473f85 --- /dev/null +++ b/src/nxt_fiber.c @@ -0,0 +1,446 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static char *nxt_fiber_create_stack(nxt_fiber_t *fib); +static void nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent); +static void nxt_fiber_switch_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_fiber_switch(nxt_thread_t *thr, nxt_fiber_t *fib); +static void nxt_fiber_timer_handler(nxt_thread_t *thr, void *obj, + void *data); + + +#define \ +nxt_fiber_enqueue(thr, fib) \ + nxt_thread_work_queue_add(thr, &(thr)->work_queue.main, \ + nxt_fiber_switch_handler, fib, NULL, thr->log) + + +nxt_fiber_main_t * +nxt_fiber_main_create(nxt_event_engine_t *engine) +{ + nxt_fiber_main_t *fm; + + fm = nxt_zalloc(sizeof(nxt_fiber_main_t)); + if (nxt_slow_path(fm == NULL)) { + return NULL; + } + + fm->stack_size = 512 * 1024 - nxt_pagesize; + fm->idle = NULL; + + return fm; +} + + +nxt_int_t +nxt_fiber_create(nxt_fiber_start_t start, void *data, size_t stack) +{ + int ret; + jmp_buf parent; + nxt_fid_t fid; + nxt_fiber_t *fib; + nxt_thread_t *thr; + nxt_fiber_main_t *fm; + + thr = nxt_thread(); + fm = thr->engine->fibers; + + fid = ++fm->fid; + + if (fid == 0) { + fid = ++fm->fid; + } + + fib = fm->idle; + + if (fib != NULL) { + fm->idle = fib->next; + fib->fid = fid; + fib->start = start; + fib->data = data; + fib->main = fm; + + nxt_log_debug(thr->log, "fiber create cached: %PF", fib->fid); + nxt_fiber_enqueue(thr, fib); + return NXT_OK; + } + + nxt_log_debug(thr->log, "fiber create"); + + fib = nxt_malloc(sizeof(nxt_fiber_t)); + if (nxt_slow_path(fib == NULL)) { + return NXT_ERROR; + } + + fib->fid = fid; + fib->start = start; + fib->data = data; + fib->stack_size = fm->stack_size; + fib->main = fm; + + fib->stack = nxt_fiber_create_stack(fib); + + if (nxt_fast_path(fib->stack != NULL)) { + + if (_setjmp(parent) != 0) { + nxt_log_debug(thr->log, "fiber create: %PF", fib->fid); + return NXT_OK; + } + + nxt_fiber_switch_stack(fib, &parent); + /* It does not return if the switch was successful. */ + } + + ret = munmap(fib->stack - nxt_pagesize, fib->stack_size + nxt_pagesize); + + if (nxt_slow_path(ret != 0)) { + nxt_log_alert(thr->log, "munmap() failed %E", nxt_errno); + } + + nxt_free(fib); + + return NXT_ERROR; +} + + +#if (NXT_LINUX) + +static char * +nxt_fiber_create_stack(nxt_fiber_t *fib) +{ + char *s; + size_t size; + + size = fib->stack_size + nxt_pagesize; + + s = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_GROWSDOWN, -1, 0); + + if (nxt_slow_path(s == MAP_FAILED)) { + nxt_thread_log_alert("fiber stack " + "mmap(%uz, MAP_PRIVATE|MAP_ANON|MAP_GROWSDOWN) failed %E", + size, nxt_errno); + return NULL; + } + + if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) { + nxt_thread_log_alert("fiber stack mprotect(%uz, PROT_NONE) failed %E", + size, nxt_errno); + return NULL; + } + + s += nxt_pagesize; + + nxt_thread_log_debug("fiber stack mmap: %p", s); + + return s; +} + +#else /* Generic version. */ + +static char * +nxt_fiber_create_stack(nxt_fiber_t *fib) +{ + char *s; + size_t size; + + size = fib->stack_size + nxt_pagesize; + + s = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + + if (nxt_slow_path(s == MAP_FAILED)) { + nxt_thread_log_alert("fiber stack " + "mmap(%uz, MAP_PRIVATE|MAP_ANON) failed %E", + size, nxt_errno); + return NULL; + } + + if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) { + nxt_thread_log_alert("fiber stack mprotect(%uz, PROT_NONE) failed %E", + size, nxt_errno); + return NULL; + } + + s += nxt_pagesize; + + nxt_thread_log_debug("fiber stack mmap: %p", s); + + return s; +} + +#endif + + +#if (NXT_LINUX && NXT_64BIT) + +/* + * Linux 64-bit ucontext version. 64-bit glibc makecontext() passes + * pointers as signed int's. The bug has been fixed in glibc 2.8. + */ + +static void nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph, + uint32_t pl); + + +static void +nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent) +{ + ucontext_t uc; + + nxt_thread_log_debug("fiber switch to stack: %p", fib->stack); + + if (nxt_slow_path(getcontext(&uc) != 0)) { + nxt_thread_log_alert("getcontext() failed"); + return; + } + + uc.uc_link = NULL; + uc.uc_stack.ss_sp = fib->stack; + uc.uc_stack.ss_size = fib->stack_size; + + makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 4, + (uint32_t) ((uintptr_t) fib >> 32), + (uint32_t) ((uintptr_t) fib & 0xffffffff), + (uint32_t) ((uintptr_t) parent >> 32), + (uint32_t) ((uintptr_t) parent & 0xffffffff)); + + setcontext(&uc); + + nxt_thread_log_alert("setcontext() failed"); +} + + +static void +nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph, uint32_t pl) +{ + jmp_buf *parent; + nxt_fiber_t *fib; + nxt_thread_t *thr; + + fib = (nxt_fiber_t *) (((uintptr_t) fh << 32) + fl); + parent = (jmp_buf *) (((uintptr_t) ph << 32) + pl); + + thr = nxt_thread(); + + if (_setjmp(fib->jmp) == 0) { + nxt_log_debug(thr->log, "fiber return to parent stack"); + + nxt_fiber_enqueue(thr, fib); + _longjmp(*parent, 1); + nxt_unreachable(); + } + + nxt_log_debug(thr->log, "fiber start"); + + fib->start(fib->data); + + nxt_fiber_exit(&fib->main->fiber, NULL); + nxt_unreachable(); +} + +#elif (NXT_HAVE_UCONTEXT) + +/* Generic ucontext version. */ + +static void nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent); + + +static void +nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent) +{ + ucontext_t uc; + + nxt_thread_log_debug("fiber switch to stack: %p", fib->stack); + + if (nxt_slow_path(getcontext(&uc) != 0)) { + nxt_thread_log_alert("getcontext() failed"); + return; + } + + uc.uc_link = NULL; + uc.uc_stack.ss_sp = fib->stack; + uc.uc_stack.ss_size = fib->stack_size; + + makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 2, fib, parent); + + setcontext(&uc); + +#if !(NXT_SOLARIS) + /* Solaris declares setcontext() as __NORETURN. */ + + nxt_thread_log_alert("setcontext() failed"); +#endif +} + + +static void +nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + + if (_setjmp(fib->jmp) == 0) { + nxt_log_debug(thr->log, "fiber return to parent stack"); + + nxt_fiber_enqueue(thr, fib); + _longjmp(*parent, 1); + nxt_unreachable(); + } + + nxt_log_debug(thr->log, "fiber start"); + + fib->start(fib->data); + + nxt_fiber_exit(&fib->main->fiber, NULL); + nxt_unreachable(); +} + +#else + +#error No ucontext(3) interface. + +#endif + + +static void +nxt_fiber_switch_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_fiber_t *fib; + + fib = obj; + + nxt_fiber_switch(thr, fib); + nxt_unreachable(); +} + + +static void +nxt_fiber_switch(nxt_thread_t *thr, nxt_fiber_t *fib) +{ + nxt_log_debug(thr->log, "fiber switch: %PF", fib->fid); + + thr->fiber = fib; + _longjmp(fib->jmp, 1); + nxt_unreachable(); +} + + +nxt_fiber_t * +nxt_fiber_self(nxt_thread_t *thr) +{ + return (nxt_fast_path(thr != NULL)) ? thr->fiber : NULL; +} + + +void +nxt_fiber_yield(void) +{ + nxt_fiber_t *fib; + nxt_thread_t *thr; + + thr = nxt_thread(); + fib = thr->fiber; + + if (_setjmp(fib->jmp) == 0) { + + nxt_log_debug(thr->log, "fiber yield"); + + nxt_fiber_enqueue(thr, fib); + nxt_fiber_switch(thr, &fib->main->fiber); + nxt_unreachable(); + } + + nxt_log_debug(thr->log, "fiber yield return"); +} + + +void +nxt_fiber_sleep(nxt_msec_t timeout) +{ + nxt_fiber_t *fib; + nxt_thread_t *thr; + + thr = nxt_thread(); + fib = thr->fiber; + + fib->timer.work_queue = &thr->work_queue.main; + fib->timer.handler = nxt_fiber_timer_handler; + fib->timer.log = &nxt_main_log; + + nxt_event_timer_add(thr->engine, &fib->timer, timeout); + + if (_setjmp(fib->jmp) == 0) { + + nxt_log_debug(thr->log, "fiber sleep: %T", timeout); + + nxt_fiber_switch(thr, &fib->main->fiber); + nxt_unreachable(); + } + + nxt_log_debug(thr->log, "fiber sleep return"); +} + + +static void +nxt_fiber_timer_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_fiber_t *fib; + nxt_event_timer_t *ev; + + ev = obj; + + nxt_log_debug(thr->log, "fiber timer handler"); + + fib = nxt_event_timer_data(ev, nxt_fiber_t, timer); + + nxt_fiber_switch(thr, fib); + nxt_unreachable(); +} + + +void +nxt_fiber_wait(void) +{ + nxt_fiber_t *fib; + nxt_thread_t *thr; + + thr = nxt_thread(); + fib = thr->fiber; + + if (_setjmp(fib->jmp) == 0) { + nxt_log_debug(thr->log, "fiber wait"); + + nxt_fiber_switch(thr, &fib->main->fiber); + nxt_unreachable(); + } + + nxt_log_debug(thr->log, "fiber wait return"); +} + + +void +nxt_fiber_exit(nxt_fiber_t *next, void *data) +{ + nxt_fiber_t *fib; + nxt_thread_t *thr; + + thr = nxt_thread(); + fib = thr->fiber; + + nxt_log_debug(thr->log, "fiber exit"); + + /* TODO: limit idle fibers. */ + fib->next = fib->main->idle; + fib->main->idle = fib; + + nxt_fiber_switch(thr, next); + nxt_unreachable(); +} diff --git a/src/nxt_fiber.h b/src/nxt_fiber.h new file mode 100644 index 00000000..7d66612f --- /dev/null +++ b/src/nxt_fiber.h @@ -0,0 +1,54 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_FIBER_H_INCLUDED_ +#define _NXT_FIBER_H_INCLUDED_ + + +typedef struct nxt_fiber_main_s nxt_fiber_main_t; +typedef void (*nxt_fiber_start_t)(void *data); + + +typedef uint32_t nxt_fid_t; +#define nxt_fiber_id(f) (f)->fid; + + +typedef struct nxt_fiber_s nxt_fiber_t; + +struct nxt_fiber_s { + jmp_buf jmp; + nxt_fid_t fid; + nxt_fiber_start_t start; + void *data; + char *stack; + size_t stack_size; + nxt_err_t err; + + nxt_fiber_main_t *main; + nxt_fiber_t *next; + + nxt_event_timer_t timer; +}; + + +struct nxt_fiber_main_s { + nxt_fiber_t fiber; + nxt_fiber_t *idle; + size_t stack_size; + nxt_fid_t fid; +}; + + +nxt_fiber_main_t *nxt_fiber_main_create(nxt_event_engine_t *engine); +nxt_int_t nxt_fiber_create(nxt_fiber_start_t start, void *data, size_t stack); +void nxt_fiber_yield(void); +void nxt_fiber_sleep(nxt_msec_t timeout); +void nxt_fiber_wait(void); +void nxt_fiber_exit(nxt_fiber_t *next, void *data); +NXT_EXPORT nxt_fiber_t *nxt_fiber_self(nxt_thread_t *thr); + + +#endif /* _NXT_FIBER_H_INCLUDED_ */ diff --git a/src/nxt_file.c b/src/nxt_file.c new file mode 100644 index 00000000..b2cab337 --- /dev/null +++ b/src/nxt_file.c @@ -0,0 +1,601 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_int_t +nxt_file_open(nxt_file_t *file, nxt_uint_t mode, nxt_uint_t create, + nxt_file_access_t access) +{ + nxt_thread_debug(thr); + +#ifdef __CYGWIN__ + mode |= O_BINARY; +#endif + + /* O_NONBLOCK is to prevent blocking on FIFOs, special devices, etc. */ + mode |= (O_NONBLOCK | create); + + file->fd = open((char *) file->name, mode, access); + + file->error = (file->fd == -1) ? nxt_errno : 0; + + nxt_thread_time_debug_update(thr); + + nxt_log_debug(thr->log, "open(\"%FN\", 0x%uXi, 0x%uXi): %FD err:%d", + file->name, mode, access, file->fd, file->error); + + if (file->fd != -1) { + return NXT_OK; + } + + if (file->log_level != 0) { + nxt_thread_log_error(file->log_level, "open(\"%FN\") failed %E", + file->name, file->error); + } + + return NXT_ERROR; +} + + +void +nxt_file_close(nxt_file_t *file) +{ + nxt_thread_log_debug("close(%FD)", file->fd); + + if (close(file->fd) != 0) { + nxt_thread_log_error(NXT_LOG_CRIT, "close(%FD, \"%FN\") failed %E", + file->fd, file->name, nxt_errno); + } +} + + +ssize_t +nxt_file_write(nxt_file_t *file, const u_char *buf, size_t size, + nxt_off_t offset) +{ + ssize_t n; + + nxt_thread_debug(thr); + + n = pwrite(file->fd, buf, size, offset); + + file->error = (n < 0) ? nxt_errno : 0; + + nxt_thread_time_debug_update(thr); + + nxt_log_debug(thr->log, "pwrite(%FD, %p, %uz, %O): %z", + file->fd, buf, size, offset, n); + + if (nxt_fast_path(n >= 0)) { + return n; + } + + nxt_thread_log_error(NXT_LOG_CRIT, + "pwrite(%FD, \"%FN\", %p, %uz, %O) failed %E", + file->fd, file->name, buf, size, + offset, file->error); + + return NXT_ERROR; +} + + +ssize_t +nxt_file_read(nxt_file_t *file, u_char *buf, size_t size, nxt_off_t offset) +{ + ssize_t n; + + nxt_thread_debug(thr); + + n = pread(file->fd, buf, size, offset); + + file->error = (n <= 0) ? nxt_errno : 0; + + nxt_thread_time_debug_update(thr); + + nxt_log_debug(thr->log, "pread(%FD, %p, %uz, %O): %z", + file->fd, buf, size, offset, n); + + if (nxt_fast_path(n >= 0)) { + return n; + } + + nxt_thread_log_error(NXT_LOG_CRIT, + "pread(%FD, \"%FN\", %p, %uz, %O) failed %E", + file->fd, file->name, buf, size, + offset, file->error); + + return NXT_ERROR; +} + + +#if (NXT_HAVE_READAHEAD) + +/* FreeBSD 8.0 fcntl(F_READAHEAD, size) enables read ahead up to the size. */ + +void +nxt_file_read_ahead(nxt_file_t *file, nxt_off_t offset, size_t size) +{ + int ret; + u_char buf; + + ret = fcntl(file->fd, F_READAHEAD, (int) size); + + nxt_thread_log_debug("fcntl(%FD, F_READAHEAD, %uz): %d", + file->fd, size, ret); + + if (nxt_fast_path(ret != -1)) { + (void) nxt_file_read(file, &buf, 1, offset); + return; + } + + nxt_thread_log_error(NXT_LOG_CRIT, + "fcntl(%FD, \"%FN\", F_READAHEAD, %uz) failed %E", + file->fd, file->name, size, nxt_errno); +} + +#elif (NXT_HAVE_POSIX_FADVISE) + +/* + * POSIX_FADV_SEQUENTIAL + * Linux doubles the default readahead window size of a backing device + * which is usually 128K. + * + * FreeBSD does nothing. + * + * POSIX_FADV_WILLNEED + * Linux preloads synchronously up to 2M of specified file region in + * the kernel page cache. Linux-specific readahead(2) syscall does + * the same. Both operations are blocking despite posix_fadvise(2) + * claims the opposite. + * + * FreeBSD does nothing. + */ + +void +nxt_file_read_ahead(nxt_file_t *file, nxt_off_t offset, size_t size) +{ + nxt_err_t err; + + err = posix_fadvise(file->fd, offset, size, POSIX_FADV_WILLNEED); + + nxt_thread_log_debug("posix_fadvise(%FD, \"%FN\", %O, %uz, %d): %d", + file->fd, file->name, offset, size, + POSIX_FADV_WILLNEED, err); + + if (nxt_fast_path(err == 0)) { + return; + } + + nxt_thread_log_error(NXT_LOG_CRIT, + "posix_fadvise(%FD, \"%FN\", %O, %uz, %d) failed %E", + file->fd, file->name, offset, size, + POSIX_FADV_WILLNEED, err); +} + +#elif (NXT_HAVE_RDAHEAD) + +/* MacOSX fcntl(F_RDAHEAD). */ + +void +nxt_file_read_ahead(nxt_file_t *file, nxt_off_t offset, size_t size) +{ + int ret; + u_char buf; + + ret = fcntl(file->fd, F_RDAHEAD, 1); + + nxt_thread_log_debug("fcntl(%FD, F_RDAHEAD, 1): %d", file->fd, ret); + + if (nxt_fast_path(ret != -1)) { + (void) nxt_file_read(file, &buf, 1, offset); + return; + } + + nxt_thread_log_error(NXT_LOG_CRIT, + "fcntl(%FD, \"%FN\", F_RDAHEAD, 1) failed %E", + file->fd, file->name, nxt_errno); +} + +#else + +void +nxt_file_read_ahead(nxt_file_t *file, nxt_off_t offset, size_t size) +{ + u_char buf; + + (void) nxt_file_read(file, &buf, 1, offset); +} + +#endif + + +nxt_int_t +nxt_file_info(nxt_file_t *file, nxt_file_info_t *fi) +{ + int n; + + if (file->fd == NXT_FILE_INVALID) { + n = stat((char *) file->name, fi); + + file->error = (n != 0) ? nxt_errno : 0; + + nxt_thread_log_debug("stat(\"%FN)\": %d", file->name, n); + + if (n == 0) { + return NXT_OK; + } + + if (file->log_level != 0) { + nxt_thread_log_error(file->log_level, "stat(\"%FN\") failed %E", + file->name, file->error); + } + + return NXT_ERROR; + + } else { + n = fstat(file->fd, fi); + + file->error = (n != 0) ? nxt_errno : 0; + + nxt_thread_log_debug("fstat(%FD): %d", file->fd, n); + + if (n == 0) { + return NXT_OK; + } + + /* Use NXT_LOG_CRIT because fstat() error on open file is strange. */ + + nxt_thread_log_error(NXT_LOG_CRIT, "fstat(%FD, \"%FN\") failed %E", + file->fd, file->name, file->error); + + return NXT_ERROR; + } +} + + +nxt_int_t +nxt_file_delete(nxt_file_name_t *name) +{ + nxt_thread_log_debug("unlink(\"%FN\")", name); + + if (nxt_fast_path(unlink((char *) name) == 0)) { + return NXT_OK; + } + + nxt_thread_log_alert("unlink(\"%FN\") failed %E", name, nxt_errno); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_file_set_access(nxt_file_name_t *name, nxt_file_access_t access) +{ + if (nxt_fast_path(chmod((char *) name, access) == 0)) { + return NXT_OK; + } + + nxt_thread_log_alert("chmod(\"%FN\") failed %E", name, nxt_errno); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_file_rename(nxt_file_name_t *old_name, nxt_file_name_t *new_name) +{ + int ret; + + nxt_thread_log_debug("rename(\"%FN\", \"%FN\")", old_name, new_name); + + ret = rename((char *) old_name, (char *) new_name); + if (nxt_fast_path(ret == 0)) { + return NXT_OK; + } + + nxt_thread_log_alert("rename(\"%FN\", \"%FN\") failed %E", + old_name, new_name, nxt_errno); + + return NXT_ERROR; +} + + +/* + * ioctl(FIONBIO) sets a non-blocking mode using one syscall, + * thereas fcntl(F_SETFL, O_NONBLOCK) needs to learn the current state + * using fcntl(F_GETFL). + * + * ioctl() and fcntl() are syscalls at least in Linux 2.2, FreeBSD 2.x, + * and Solaris 7. + * + * Linux 2.4 uses BKL for ioctl() and fcntl(F_SETFL). + * Linux 2.6 does not use BKL. + */ + +#if (NXT_HAVE_FIONBIO) + +nxt_int_t +nxt_fd_nonblocking(nxt_fd_t fd) +{ + int nb; + + nb = 1; + + if (nxt_fast_path(ioctl(fd, FIONBIO, &nb) != -1)) { + return NXT_OK; + } + + nxt_thread_log_alert("ioctl(%d, FIONBIO) failed %E", fd, nxt_errno); + + return NXT_ERROR; + +} + + +nxt_int_t +nxt_fd_blocking(nxt_fd_t fd) +{ + int nb; + + nb = 0; + + if (nxt_fast_path(ioctl(fd, FIONBIO, &nb) != -1)) { + return NXT_OK; + } + + nxt_thread_log_alert("ioctl(%d, !FIONBIO) failed %E", fd, nxt_errno); + + return NXT_ERROR; +} + +#else /* !(NXT_HAVE_FIONBIO) */ + +nxt_int_t +nxt_fd_nonblocking(nxt_fd_t fd) +{ + int flags; + + flags = fcntl(fd, F_GETFL); + + if (nxt_slow_path(flags == -1)) { + nxt_thread_log_alert("fcntl(%d, F_GETFL) failed %E", fd, nxt_errno); + return NXT_ERROR; + } + + flags |= O_NONBLOCK; + + if (nxt_slow_path(fcntl(fd, F_SETFL, flags) == -1)) { + nxt_thread_log_alert("fcntl(%d, F_SETFL, O_NONBLOCK) failed %E", + fd, nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + + +nxt_int_t +nxt_fd_blocking(nxt_fd_t fd) +{ + int flags; + + flags = fcntl(fd, F_GETFL); + + if (nxt_slow_path(flags == -1)) { + nxt_thread_log_alert("fcntl(%d, F_GETFL) failed %E", + fd, nxt_errno); + return NXT_ERROR; + } + + flags &= O_NONBLOCK; + + if (nxt_slow_path(fcntl(fd, F_SETFL, flags) == -1)) { + nxt_thread_log_alert("fcntl(%d, F_SETFL, !O_NONBLOCK) failed %E", + fd, nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif /* NXT_HAVE_FIONBIO */ + + +ssize_t +nxt_fd_write(nxt_fd_t fd, u_char *buf, size_t size) +{ + ssize_t n; + nxt_err_t err; + + n = write(fd, buf, size); + + err = (n == -1) ? nxt_errno : 0; + + nxt_thread_log_debug("write(%FD, %p, %uz): %z", fd, buf, size, n); + + if (nxt_slow_path(n <= 0)) { + nxt_thread_log_alert("write(%FD) failed %E", fd, err); + } + + return n; +} + + +ssize_t +nxt_fd_read(nxt_fd_t fd, u_char *buf, size_t size) +{ + ssize_t n; + nxt_err_t err; + + n = read(fd, buf, size); + + err = (n == -1) ? nxt_errno : 0; + + nxt_thread_log_debug("read(%FD, %p, %uz): %z", fd, buf, size, n); + + if (nxt_slow_path(n <= 0)) { + + if (err == NXT_EAGAIN) { + return 0; + } + + nxt_thread_log_alert("read(%FD) failed %E", fd, err); + } + + return n; +} + + +void +nxt_fd_close(nxt_fd_t fd) +{ + nxt_thread_log_debug("close(%FD)", fd); + + if (nxt_slow_path(close(fd) != 0)) { + nxt_thread_log_error(NXT_LOG_CRIT, "close(%FD) failed %E", + fd, nxt_errno); + } +} + + +/* + * nxt_file_redirect() redirects the file to the fd descriptor. + * Then the fd descriptor is closed. + */ + +nxt_int_t +nxt_file_redirect(nxt_file_t *file, nxt_fd_t fd) +{ + nxt_thread_log_debug("dup2(%FD, %FD, \"%FN\")", fd, file->fd, file->name); + + if (dup2(fd, file->fd) == -1) { + nxt_thread_log_error(NXT_LOG_CRIT, "dup2(%FD, %FD, \"%FN\") failed %E", + fd, file->fd, file->name, nxt_errno); + return NXT_ERROR; + } + + if (close(fd) != 0) { + nxt_thread_log_error(NXT_LOG_CRIT, "close(%FD, \"%FN\") failed %E", + fd, file->name, nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* nxt_file_stderr() redirects the stderr descriptor to the file. */ + +nxt_int_t +nxt_file_stderr(nxt_file_t *file) +{ + nxt_thread_log_debug("dup2(%FD, %FD, \"%FN\")", + file->fd, STDERR_FILENO, file->name); + + if (dup2(file->fd, STDERR_FILENO) != -1) { + return NXT_OK; + } + + nxt_thread_log_error(NXT_LOG_CRIT, "dup2(%FD, %FD, \"%FN\") failed %E", + file->fd, STDERR_FILENO, file->name); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_stderr_start(void) +{ + int flags, fd; + + flags = fcntl(nxt_stderr, F_GETFL); + + if (flags != -1) { + /* + * If the stderr output of a multithreaded application is + * redirected to a file: + * Linux, Solaris and MacOSX do not write atomically to the output; + * MacOSX besides adds zeroes to the output. + * O_APPEND fixes this. + */ + (void) fcntl(nxt_stderr, F_SETFL, flags | O_APPEND); + + } else { + /* + * The stderr descriptor is closed before application start. + * Reserve the stderr descriptor for future use. Errors are + * ignored because anyway they could be written nowhere. + */ + fd = open("/dev/null", O_WRONLY | O_APPEND); + + if (fd != -1) { + (void) dup2(fd, nxt_stderr); + + if (fd != nxt_stderr) { + (void) close(fd); + } + } + } + + return flags; +} + + +nxt_int_t +nxt_pipe_create(nxt_fd_t *pp, nxt_bool_t nbread, nxt_bool_t nbwrite) +{ + if (pipe(pp) != 0) { + nxt_thread_log_alert("pipe() failed %E", nxt_errno); + + return NXT_ERROR; + } + + nxt_thread_log_debug("pipe(): %FD:%FD", pp[0], pp[1]); + + if (nbread) { + if (nxt_fd_nonblocking(pp[0]) != NXT_OK) { + return NXT_ERROR; + } + } + + if (nbwrite) { + if (nxt_fd_nonblocking(pp[1]) != NXT_OK) { + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +void +nxt_pipe_close(nxt_fd_t *pp) +{ + nxt_thread_log_debug("pipe close(%FD:%FD)", pp[0], pp[1]); + + if (close(pp[0]) != 0) { + nxt_thread_log_alert("pipe close (%FD) failed %E", pp[0], nxt_errno); + } + + if (close(pp[1]) != 0) { + nxt_thread_log_alert("pipe close(%FD) failed %E", pp[1], nxt_errno); + } +} + + +size_t +nxt_dir_current(char *buf, size_t len) +{ + if (nxt_fast_path(getcwd(buf, len) != NULL)) { + return nxt_strlen(buf); + } + + nxt_thread_log_alert("getcwd(%uz) failed %E", len, nxt_errno); + + return 0; +} diff --git a/src/nxt_file.h b/src/nxt_file.h new file mode 100644 index 00000000..3ef5fbfb --- /dev/null +++ b/src/nxt_file.h @@ -0,0 +1,195 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_FILE_H_INCLUDED_ +#define _NXT_UNIX_FILE_H_INCLUDED_ + + +typedef int nxt_fd_t; + +#define NXT_FILE_INVALID -1 + +typedef nxt_uint_t nxt_file_access_t; +typedef struct stat nxt_file_info_t; + + +#define NXT_FILE_SYSTEM_NAME_UTF8 1 + +typedef u_char nxt_file_name_t; + + +typedef struct { + size_t len; + nxt_file_name_t *start; +} nxt_file_name_str_t; + + +#define \ +nxt_file_name_str_set(file_name, mem_pool, name) \ + ((file_name) = (nxt_file_name_t *) (name), NXT_OK) + + +#define \ +nxt_file_name_alloc(mem_pool, len) \ + nxt_mem_nalloc(mem_pool, len) + + +#define \ +nxt_file_name_copy(dst, src, len) \ + nxt_cpymem(dst, src, len) + + +#define \ +nxt_file_name_add(dst, src, len) \ + nxt_cpymem(dst, src, len) + + +#if (NXT_HAVE_CASELESS_FILESYSTEM) + +/* MacOSX, Cygwin. */ + +#define \ +nxt_file_name_eq(fn1, fn2) \ + (nxt_strcasecmp(fn1, fn2) == 0) + +#else + +#define \ +nxt_file_name_eq(fn1, fn2) \ + (nxt_strcmp(fn1, fn2) == 0) + +#endif + + +#define \ +nxt_file_name_is_absolute(name) \ + (name[0] == '/') + + +#define NXT_MAX_PATH_LEN MAXPATHLEN + + +typedef enum { + NXT_FILE_UNKNOWN = 0, + NXT_FILE_REGULAR, + NXT_FILE_DIRECTORY, +} nxt_file_type_t; + + +typedef struct { + nxt_file_name_t *name; + + /* Both are int's. */ + nxt_fd_t fd; + nxt_err_t error; + +#define NXT_FILE_ACCESSED_LONG_AGO 0xffff + /* + * Number of seconds ago the file content was last + * read. The maximum value is about 18 hours. + */ + uint16_t accessed; + + uint8_t type; /* nxt_file_type_t */ + + /* + * Log open() file error with given log level if it is non zero. + * Note that zero log level is NXT_LOG_EMERG. + */ + uint8_t log_level; + + nxt_time_t mtime; + nxt_off_t size; +} nxt_file_t; + + +NXT_EXPORT nxt_int_t nxt_file_open(nxt_file_t *file, nxt_uint_t mode, + nxt_uint_t create, nxt_file_access_t access); + +#define nxt_file_open_n "open" + + +/* The file open access modes. */ +#define NXT_FILE_RDONLY O_RDONLY +#define NXT_FILE_WRONLY O_WRONLY +#define NXT_FILE_RDWR O_RDWR +#define NXT_FILE_APPEND (O_WRONLY | O_APPEND) + +/* The file creation modes. */ +#define NXT_FILE_CREATE_OR_OPEN O_CREAT +#define NXT_FILE_OPEN 0 +#define NXT_FILE_TRUNCATE (O_CREAT | O_TRUNC) + +/* The file access rights. */ +#define NXT_FILE_DEFAULT_ACCESS 0644 +#define NXT_FILE_OWNER_ACCESS 0600 + + +NXT_EXPORT void nxt_file_close(nxt_file_t *file); +NXT_EXPORT ssize_t nxt_file_write(nxt_file_t *file, const u_char *buf, + size_t size, nxt_off_t offset); +NXT_EXPORT ssize_t nxt_file_read(nxt_file_t *file, u_char *buf, size_t size, + nxt_off_t offset); +NXT_EXPORT void nxt_file_read_ahead(nxt_file_t *file, nxt_off_t offset, + size_t size); +NXT_EXPORT nxt_int_t nxt_file_info(nxt_file_t *file, nxt_file_info_t *fi); + + +#define \ +nxt_is_dir(fi) \ + (S_ISDIR((fi)->st_mode)) + +#define \ +nxt_is_file(fi) \ + (S_ISREG((fi)->st_mode)) + +#define \ +nxt_file_size(fi) \ + (fi)->st_size + +#define \ +nxt_file_mtime(fi) \ + (fi)->st_mtime + + +NXT_EXPORT nxt_int_t nxt_file_delete(nxt_file_name_t *name); +NXT_EXPORT nxt_int_t nxt_file_set_access(nxt_file_name_t *name, + nxt_file_access_t access); +NXT_EXPORT nxt_int_t nxt_file_rename(nxt_file_name_t *old_name, + nxt_file_name_t *new_name); + +NXT_EXPORT nxt_int_t nxt_fd_nonblocking(nxt_fd_t fd); +NXT_EXPORT nxt_int_t nxt_fd_blocking(nxt_fd_t fd); +NXT_EXPORT ssize_t nxt_fd_write(nxt_fd_t fd, u_char *buf, size_t size); +NXT_EXPORT ssize_t nxt_fd_read(nxt_fd_t fd, u_char *buf, size_t size); +NXT_EXPORT void nxt_fd_close(nxt_fd_t fd); + +NXT_EXPORT nxt_int_t nxt_file_redirect(nxt_file_t *file, nxt_fd_t fd); +NXT_EXPORT nxt_int_t nxt_file_stderr(nxt_file_t *file); +NXT_EXPORT nxt_int_t nxt_stderr_start(void); + + +#define nxt_stdout STDOUT_FILENO +#define nxt_stderr STDERR_FILENO + + +#define \ +nxt_write_console(fd, buf, size) \ + write(fd, buf, size) + +#define \ +nxt_write_syslog(priority, message) \ + syslog(priority, "%s", message) + + +NXT_EXPORT nxt_int_t nxt_pipe_create(nxt_fd_t *pp, nxt_bool_t nbread, + nxt_bool_t nbwrite); +NXT_EXPORT void nxt_pipe_close(nxt_fd_t *pp); + +NXT_EXPORT size_t nxt_dir_current(char *buf, size_t len); + + +#endif /* _NXT_UNIX_FILE_H_INCLUDED_ */ diff --git a/src/nxt_file_cache.c b/src/nxt_file_cache.c new file mode 100644 index 00000000..3af3c0c5 --- /dev/null +++ b/src/nxt_file_cache.c @@ -0,0 +1,508 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_file_cache_lvlhsh_test(nxt_lvlhsh_key_t *hkey, void *data); +static nxt_work_handler_t nxt_file_cache_query_locked(nxt_file_cache_t *cache, + nxt_file_cache_query_t *q, nxt_lvlhsh_key_t *hkey); +static nxt_work_handler_t nxt_file_cache_node_hold(nxt_file_cache_t *cache, + nxt_file_cache_query_t *q, nxt_lvlhsh_key_t *hkey); +static nxt_work_handler_t nxt_file_cache_node_test(nxt_file_cache_t *cache, + nxt_file_cache_query_t *q); + +static void nxt_file_cache_wait_handler(void *data); +static void nxt_file_cache_timeout_handler(nxt_event_timer_t *ev); +static void nxt_file_cache_wake_handler(void *data); + +static nxt_file_cache_node_t *nxt_file_cache_node_alloc(nxt_cache_t *cache); +static void nxt_file_cache_node_free(nxt_file_cache_t *cache, + nxt_file_cache_node_t *node, nxt_bool_t fast); +static nxt_file_cache_query_wait_t *nxt_file_cache_query_wait_alloc( + nxt_file_cache_t *cache, nxt_bool_t *fast); +static void nxt_file_cache_query_wait_free(nxt_file_cache_t *cache, + nxt_file_cache_query_wait_t *qw); +static void nxt_file_cache_lock(nxt_file_cache_t *cache); +static void nxt_file_cache_unlock(nxt_file_cache_t *cache); + + +void +nxt_file_cache_init(nxt_cache_t *cache) +{ + static const nxt_lvlhsh_ctx_t ctx = { + nxt_file_cache_lvlhsh_test, + nxt_lvlhsh_alloc, + nxt_lvlhsh_free, + 0, + }; + + /* lvlhsh with large first level. */ + cache->lvlhsh.shift[1] = 10; + + cache->lvlhsh.ctx = &ctx; + + cache->start_time = nxt_thread_time(); +} + + +static nxt_int_t +nxt_file_cache_lvlhsh_test(nxt_lvlhsh_key_t *hkey, void *data) +{ + nxt_file_cache_node_t *node; + + node = data; + + if (nxt_strmem_eq(&hkey->key, node->key_data, node->key_len)) { + return NXT_OK; + } + + return NXT_DECLINED; +} + + +void +nxt_file_cache_query(nxt_file_cache_t *cache, nxt_file_cache_query_t *q) +{ + nxt_lvlhsh_key_t hkey; + nxt_work_handler_t handler; + + if (cache != NULL) { + hkey.key.len = q->key_len; + hkey.key.data = q->key_data; + hkey.key_hash = nxt_murmur_hash2(q->key_data, q->key_len); + hkey.replace = 0; + + nxt_file_cache_lock(cache); + + handler = nxt_file_cache_query_locked(cache, q, &hkey); + + nxt_file_cache_unlock(cache); + + } else { + handler = q->state->nocache_handler; + } + + handler(q); +} + + +static nxt_work_handler_t +nxt_file_cache_query_locked(nxt_file_cache_t *cache, nxt_file_cache_query_t *q, + nxt_lvlhsh_key_t *hkey) +{ + nxt_int_t ret; + nxt_bool_t fast; + nxt_work_handler_t handler; + nxt_file_cache_node_t *node, *sentinel; + nxt_file_cache_query_wait_t *qw; + nxt_file_cache_query_state_t *state; + + state = q->state; + sentinel = nxt_file_cache_node_alloc(cache); + + if (nxt_slow_path(sentinel == NULL)) { + return state->error_handler; + } + + sentinel->key_data = q->key_data; + sentinel->key_len = q->key_len; + hkey->value = sentinel; + + /* + * Try to insert an empty sentinel node to hold updating + * process if there is no existent cache node in cache. + */ + + ret = nxt_lvlhsh_insert(&cache->lvlhsh, hkey); + + if (ret == NXT_OK) { + /* The sentinel node was successully added. */ + + q->node = sentinel; + sentinel->updating = 1; + return state->update_handler; + } + + nxt_cache_node_free(cache, sentinel, 1); + + if (ret == NXT_ERROR) { + return state->error_handler; + } + + /* NXT_DECLINED: a cache node exists. */ + + node = hkey->value; + node->count++; + q->node = node; + + handler = nxt_cache_node_test(cache, q); + + if (handler == NULL) { + /* Add the node to a wait queue. */ + + qw = nxt_cache_query_wait_alloc(cache, &fast); + if (nxt_slow_path(qw == NULL)) { + return state->error_handler; + } + + if (!fast) { + /* The node state may be changed during slow allocation. */ + handler = nxt_cache_node_test(cache, q); + + if (handler != NULL) { + nxt_cache_query_wait_free(cache, qw); + return handler; + } + } + + qw->query = q; + qw->next = node->waiting; + qw->busy = 0; + qw->deleted = 0; + qw->pid = nxt_pid; + qw->engine = nxt_thread_event_engine(); + qw->handler = nxt_cache_wake_handler; + qw->cache = cache; + + node->waiting = qw; + + return nxt_cache_wait_handler; + } + + return handler; +} + + +static nxt_work_handler_t +nxt_cache_node_test(nxt_cache_t *cache, nxt_cache_query_t *q) +{ + nxt_time_t expiry; + nxt_cache_node_t *node; + nxt_cache_query_state_t *state; + + q->stale = 0; + state = q->state; + node = q->node; + + expiry = cache->start_time + node->expiry; + + if (nxt_thread_time() < expiry) { + return state->ready_handler; + } + + /* + * A valid stale or empty sentinel cache node. + * The sentinel node can be only in updating state. + */ + + if (node->updating) { + + if (node->expiry != 0) { + /* A valid stale cache node. */ + + q->stale = 1; + + if (q->use_stale) { + return state->stale_handler; + } + } + + return NULL; + } + + /* A valid stale cache node is not being updated now. */ + + q->stale = 1; + + if (q->use_stale) { + + if (q->update_stale) { + node->updating = 1; + return state->update_stale_handler; + } + + return state->stale_handler; + } + + node->updating = 1; + return state->update_handler; +} + + +static void +nxt_cache_wait_handler(void *data) +{ + nxt_thread_t *thr; + nxt_event_timer_t *ev; + nxt_cache_query_t *q; + + q = data; + + if (&q->timeout == 0) { + return; + } + + ev = &q->timer; + + if (!nxt_event_timer_is_set(ev)) { + thr = nxt_thread(); + ev->log = thr->log; + ev->handler = nxt_cache_timeout_handler; + ev->data = q; + nxt_event_timer_ident(ev, -1); + + nxt_event_timer_add(thr->engine, ev, q->timeout); + } +} + + +static void +nxt_cache_timeout_handler(nxt_event_timer_t *ev) +{ + nxt_cache_query_t *q; + + q = ev->data; + + q->state->timeout_handler(q); +} + + +static void +nxt_cache_wake_handler(void *data) +{ + nxt_cache_t *cache; + nxt_work_handler_t handler; + nxt_cache_query_t *q; + nxt_cache_query_wait_t *qw; + + qw = data; + q = qw->query; + cache = qw->cache; + + nxt_cache_lock(cache); + + handler = nxt_cache_node_test(cache, q); + + if (handler == NULL) { + /* Wait again. */ + qw->next = q->node->waiting; + q->node->waiting = qw; + } + + nxt_cache_unlock(cache); + + if (handler != NULL) { + nxt_cache_query_wait_free(cache, qw); + } + + handler(q); +} + + +static nxt_cache_node_t * +nxt_cache_node_alloc(nxt_cache_t *cache) +{ + nxt_queue_node_t *qn; + nxt_cache_node_t *node; + + qn = nxt_queue_first(&cache->free_nodes); + + if (nxt_fast_path(qn != nxt_queue_tail(&cache->free_nodes))) { + cache->nfree_nodes--; + nxt_queue_remove(qn); + + node = nxt_queue_node_data(qn, nxt_cache_node_t, queue); + nxt_memzero(node, sizeof(nxt_cache_node_t)); + + return node; + } + + nxt_cache_unlock(cache); + + node = cache->alloc(cache->data, sizeof(nxt_cache_node_t)); + + nxt_cache_lock(cache); + + return node; +} + + +static void +nxt_cache_node_free(nxt_cache_t *cache, nxt_cache_node_t *node, nxt_bool_t fast) +{ + if (fast || cache->nfree_nodes < 32) { + nxt_queue_insert_head(&cache->free_nodes, &node->queue); + cache->nfree_nodes++; + return; + } + + nxt_cache_unlock(cache); + + cache->free(cache->data, node); + + nxt_cache_lock(cache); +} + + +static nxt_cache_query_wait_t * +nxt_cache_query_wait_alloc(nxt_cache_t *cache, nxt_bool_t *fast) +{ + nxt_cache_query_wait_t *qw; + + qw = cache->free_query_wait; + + if (nxt_fast_path(qw != NULL)) { + cache->free_query_wait = qw->next; + cache->nfree_query_wait--; + + *fast = 1; + return qw; + } + + nxt_cache_unlock(cache); + + qw = cache->alloc(cache->data, sizeof(nxt_cache_query_wait_t)); + *fast = 0; + + nxt_cache_lock(cache); + + return qw; +} + + +static void +nxt_cache_query_wait_free(nxt_cache_t *cache, nxt_cache_query_wait_t *qw) +{ + if (cache->nfree_query_wait < 32) { + qw->next = cache->free_query_wait; + cache->free_query_wait = qw; + cache->nfree_query_wait++; + return; + } + + nxt_cache_unlock(cache); + + cache->free(cache->data, qw); + + nxt_cache_lock(cache); +} + + +#if 0 + +nxt_int_t +nxt_cache_update(nxt_cache_t *cache, nxt_cache_node_t *node) +{ + nxt_lvlhsh_key_t hkey; + + if (node->expiry == 0) { + /* An empty sentinel node. */ + nxt_cache_release(cache, node); + return; + } + + hkey.key.len = node->key_len; + hkey.key.data = node->key_data; + hkey.key_hash = nxt_murmur_hash2(node->key_data, node->key_len); + hkey.replace = 1; + hkey.value = node; + + node->count = 1; + + if (nxt_lvlhsh_insert(&cache->lvlhsh, &hkey) != NXT_OK) { + return NXT_ERROR; + } + + node = hkey.value; + + if (node != NULL) { + if (node->count != 0) { + node->delete = 1; + + } else { + // delete cache node + } + } + + return NXT_OK; +} + +#endif + + +void +nxt_cache_node_release(nxt_cache_t *cache, nxt_cache_node_t *node) +{ + nxt_bool_t delete; + + nxt_cache_lock(cache); + + delete = nxt_cache_node_release_locked(cache, node); + + nxt_cache_unlock(cache); + + if (delete) { + nxt_thread_work_queue_add(cache->delete_handler, node); + } +} + + +nxt_bool_t +nxt_cache_node_release_locked(nxt_cache_t *cache, nxt_cache_node_t *node) +{ +#if 0 + nxt_lvlhsh_key_t hkey; +#endif + + node->count--; + + if (node->count != 0) { + return 0; + } + + if (!node->deleted) { + /* + * A cache node is locked whilst its count is non zero. + * To minimize number of operations the node's place in expiry + * queue can be updated only if the node is not currently used. + */ + node->accessed = nxt_thread_time() - cache->start_time; + + nxt_queue_remove(&node->queue); + nxt_queue_insert_head(&cache->expiry_queue, &node->queue); + + return 0; + } + +#if 0 + hkey.key.len = node->key_len; + hkey.key.data = node->key_data; + hkey.key_hash = nxt_murmur_hash2(node->key_data, node->key_len); + + nxt_lvlhsh_delete(&cache->lvlhsh, &hkey); +#endif + + return 1; +} + + +static void +nxt_file_cache_lock(nxt_file_cache_t *cache) +{ + if (cache->shared) { + nxt_thread_spin_lock(&cache->lock); + } +} + + +static void +nxt_file_cache_unlock(nxt_file_cache_t *cache) +{ + if (cache->shared) { + nxt_thread_spin_unlock(&cache->lock); + } +} diff --git a/src/nxt_file_name.c b/src/nxt_file_name.c new file mode 100644 index 00000000..a59cd924 --- /dev/null +++ b/src/nxt_file_name.c @@ -0,0 +1,201 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * Supported formats: + * %s null-terminated string + * %*s length and string + * %FN nxt_file_name_t * + * %V nxt_str_t * + * %Z '\0', this null is not counted in file name lenght. + */ + +nxt_int_t +nxt_file_name_create(nxt_mem_pool_t *mp, nxt_file_name_str_t *file_name, + const char *format, ...) +{ + u_char ch, *p; + size_t len; + va_list args; + nxt_str_t *v; + nxt_bool_t zero; + const char *fmt; + nxt_file_name_t *dst, *fn; + + va_start(args, format); + fmt = format; + zero = 0; + len = 0; + + for ( ;; ) { + ch = *fmt++; + + if (ch != '%') { + + if (ch != '\0') { + len++; + continue; + } + + break; + } + + ch = *fmt++; + + switch (ch) { + + case 'V': + v = va_arg(args, nxt_str_t *); + + if (nxt_fast_path(v != NULL)) { + len += v->len; + } + + continue; + + case 's': + p = va_arg(args, u_char *); + + if (nxt_fast_path(p != NULL)) { + while (*p != '\0') { + p++; + len++; + } + } + + continue; + + case '*': + len += va_arg(args, u_int); + fmt++; + + continue; + + case 'F': + ch = *fmt++; + + if (nxt_fast_path(ch == 'N')) { + fn = va_arg(args, nxt_file_name_t *); + + if (nxt_fast_path(fn != NULL)) { + while (*fn != '\0') { + fn++; + len += sizeof(nxt_file_name_t); + } + } + } + + continue; + + case 'Z': + zero = 1; + len++; + continue; + + default: + continue; + } + } + + va_end(args); + + if (len == 0) { + return NXT_ERROR; + } + + file_name->len = len - zero; + + fn = nxt_file_name_alloc(mp, len); + if (nxt_slow_path(fn == NULL)) { + return NXT_ERROR; + } + + file_name->start = fn; + dst = fn; + + va_start(args, format); + fmt = format; + + for ( ;; ) { + ch = *fmt++; + + if (ch != '%') { + + if (ch != '\0') { + *dst++ = (nxt_file_name_t) ch; + continue; + } + + break; + } + + ch = *fmt++; + + switch (ch) { + + case 'V': + v = va_arg(args, nxt_str_t *); + + if (nxt_fast_path(v != NULL)) { + dst = nxt_file_name_add(dst, v->data, v->len); + } + + continue; + + case 's': + p = va_arg(args, u_char *); + + if (nxt_fast_path(p != NULL)) { + while (*p != '\0') { + *dst++ = (nxt_file_name_t) (*p++); + } + } + + continue; + + case '*': + len += va_arg(args, u_int); + + ch = *fmt++; + + if (nxt_fast_path(ch == 's')) { + p = va_arg(args, u_char *); + dst = nxt_file_name_add(dst, p, len); + } + + continue; + + case 'F': + ch = *fmt++; + + if (nxt_fast_path(ch == 'N')) { + fn = va_arg(args, nxt_file_name_t *); + + if (nxt_fast_path(fn != NULL)) { + while (*fn != '\0') { + *dst++ = *fn++; + } + } + } + + continue; + + case 'Z': + *dst++ = '\0'; + continue; + + default: + continue; + } + } + + va_end(args); + + return NXT_OK; +} diff --git a/src/nxt_file_name.h b/src/nxt_file_name.h new file mode 100644 index 00000000..a0c00406 --- /dev/null +++ b/src/nxt_file_name.h @@ -0,0 +1,15 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_FILE_NAME_H_INCLUDED_ +#define _NXT_FILE_NAME_H_INCLUDED_ + + +NXT_EXPORT nxt_int_t nxt_file_name_create(nxt_mem_pool_t *mp, + nxt_file_name_str_t *fn, const char *format, ...); + + +#endif /* _NXT_FILE_NAME_H_INCLUDED_ */ diff --git a/src/nxt_freebsd_sendfile.c b/src/nxt_freebsd_sendfile.c new file mode 100644 index 00000000..9cc05c0f --- /dev/null +++ b/src/nxt_freebsd_sendfile.c @@ -0,0 +1,145 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * sendfile() has been introduced in FreeBSD 3.1, + * however, early implementation had various bugs. + * This code supports FreeBSD 5.0 implementation. + */ + +#ifdef NXT_TEST_BUILD_FREEBSD_SENDFILE + +ssize_t nxt_freebsd_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + +static int nxt_sys_sendfile(int fd, int s, off_t offset, size_t nbytes, + struct sf_hdtr *hdtr, off_t *sbytes, int flags) +{ + return -1; +} + +#else +#define nxt_sys_sendfile sendfile +#endif + + +ssize_t +nxt_freebsd_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit) +{ + size_t file_size; + ssize_t n; + nxt_buf_t *fb; + nxt_err_t err; + nxt_off_t sent; + nxt_uint_t nhd, ntr; + struct iovec hd[NXT_IOBUF_MAX], tr[NXT_IOBUF_MAX]; + struct sf_hdtr hdtr, *ht; + nxt_sendbuf_coalesce_t sb; + + sb.buf = b; + sb.iobuf = hd; + sb.nmax = NXT_IOBUF_MAX; + sb.sync = 0; + sb.size = 0; + sb.limit = limit; + + nhd = nxt_sendbuf_mem_coalesce(&sb); + + if (nhd == 0 && sb.sync) { + return 0; + } + + if (sb.buf == NULL || !nxt_buf_is_file(sb.buf)) { + return nxt_event_conn_io_writev(c, hd, nhd); + } + + fb = sb.buf; + + file_size = nxt_sendbuf_file_coalesce(&sb); + + if (file_size == 0) { + return nxt_event_conn_io_writev(c, hd, nhd); + } + + sb.iobuf = tr; + + ntr = nxt_sendbuf_mem_coalesce(&sb); + + /* + * Disposal of surplus kernel operations + * if there are no headers or trailers. + */ + + ht = NULL; + nxt_memzero(&hdtr, sizeof(struct sf_hdtr)); + + if (nhd != 0) { + ht = &hdtr; + hdtr.headers = hd; + hdtr.hdr_cnt = nhd; + } + + if (ntr != 0) { + ht = &hdtr; + hdtr.trailers = tr; + hdtr.trl_cnt = ntr; + } + + nxt_log_debug(c->socket.log, "sendfile(%FD, %d, @%O, %uz) hd:%ui tr:%ui", + fb->file->fd, c->socket.fd, fb->file_pos, file_size, + nhd, ntr); + + sent = 0; + n = nxt_sys_sendfile(fb->file->fd, c->socket.fd, fb->file_pos, + file_size, ht, &sent, 0); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(c->socket.log, "sendfile(): %d sent:%O", n, sent); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "sendfile(%FD, %d, %O, %uz) failed " + "%E \"%FN\" hd:%ui tr:%ui", fb->file->fd, + c->socket.fd, fb->file_pos, file_size, + err, fb->file->name, nhd, ntr); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "sendfile() %E", err); + + return sent; + + } else if (sent == 0) { + nxt_log_error(NXT_LOG_ERR, c->socket.log, + "file \"%FN\" was truncated while sendfile()", + fb->file->name); + + return NXT_ERROR; + } + + if (sent < (nxt_off_t) sb.size) { + c->socket.write_ready = 0; + } + + return sent; +} diff --git a/src/nxt_gmtime.c b/src/nxt_gmtime.c new file mode 100644 index 00000000..9c3fa190 --- /dev/null +++ b/src/nxt_gmtime.c @@ -0,0 +1,79 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* The function is valid for positive nxt_time_t only. */ + +void +nxt_gmtime(nxt_time_t s, struct tm *tm) +{ + nxt_int_t yday; + nxt_uint_t daytime, mday, mon, year, days, leap; + + days = (nxt_uint_t) (s / 86400); + daytime = (nxt_uint_t) (s % 86400); + + /* January 1, 1970 was Thursday. */ + tm->tm_wday = (4 + days) % 7; + + /* The algorithm based on Gauss' formula. */ + + /* Days since March 1, 1 BCE. */ + days = days - (31 + 28) + 719527; + + /* + * The "days" should be adjusted by 1 only, however some March 1st's + * go to previous year, so "days" are adjusted by 2. This also shifts + * the last February days to the next year, but this is catched by + * negative "yday". + */ + year = (days + 2) * 400 / (365 * 400 + 100 - 4 + 1); + + yday = days - (365 * year + year / 4 - year / 100 + year / 400); + + leap = (year % 4 == 0) && (year % 100 || (year % 400 == 0)); + + if (yday < 0) { + yday = 365 + leap + yday; + year--; + } + + /* + * An empirical formula that maps "yday" to month. + * There are at least 10 variants, some of them are: + * mon = (yday + 31) * 15 / 459 + * mon = (yday + 31) * 17 / 520 + * mon = (yday + 31) * 20 / 612 + */ + + mon = (yday + 31) * 10 / 306; + + /* The Gauss' formula that evaluates days before month. */ + + mday = yday - (367 * mon / 12 - 30) + 1; + + if (yday >= 306) { + year++; + mon -= 11; + yday -= 306; + + } else { + mon++; + yday += 31 + 28 + leap; + } + + tm->tm_mday = mday; + tm->tm_mon = mon; + tm->tm_year = year - 1900; + tm->tm_yday = yday; + + tm->tm_hour = daytime / 3600; + daytime %= 3600; + tm->tm_min = daytime / 60; + tm->tm_sec = daytime % 60; +} diff --git a/src/nxt_gnutls.c b/src/nxt_gnutls.c new file mode 100644 index 00000000..15db7fc8 --- /dev/null +++ b/src/nxt_gnutls.c @@ -0,0 +1,742 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <gnutls/gnutls.h> + + +typedef struct { + gnutls_session_t session; + + uint8_t times; /* 2 bits */ + uint8_t no_shutdown; /* 1 bit */ + + nxt_buf_mem_t buffer; +} nxt_gnutls_conn_t; + + +typedef struct { + gnutls_priority_t ciphers; + gnutls_certificate_credentials_t certificate; +} nxt_gnutls_ctx_t; + + + +#if (NXT_HAVE_GNUTLS_SET_TIME) +time_t nxt_gnutls_time(time_t *tp); +#endif +static nxt_int_t nxt_gnutls_server_init(nxt_ssltls_conf_t *conf); +static nxt_int_t nxt_gnutls_set_ciphers(nxt_ssltls_conf_t *conf); + +static void nxt_gnutls_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c); +static void nxt_gnutls_session_cleanup(void *data); +static ssize_t nxt_gnutls_pull(gnutls_transport_ptr_t data, void *buf, + size_t size); +static ssize_t nxt_gnutls_push(gnutls_transport_ptr_t data, const void *buf, + size_t size); +#if (NXT_HAVE_GNUTLS_VEC_PUSH) +static ssize_t nxt_gnutls_vec_push(gnutls_transport_ptr_t data, + const giovec_t *iov, int iovcnt); +#endif +static void nxt_gnutls_conn_handshake(nxt_thread_t *thr, void *obj, void *data); +static void nxt_gnutls_conn_io_read(nxt_thread_t *thr, void *obj, void *data); +static ssize_t nxt_gnutls_conn_io_write_chunk(nxt_thread_t *thr, + nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); +static ssize_t nxt_gnutls_conn_io_send(nxt_event_conn_t *c, void *buf, + size_t size); +static void nxt_gnutls_conn_io_shutdown(nxt_thread_t *thr, void *obj, + void *data); +static nxt_int_t nxt_gnutls_conn_test_error(nxt_thread_t *thr, + nxt_event_conn_t *c, ssize_t err, nxt_work_handler_t handler); +static void nxt_cdecl nxt_gnutls_conn_log_error(nxt_event_conn_t *c, + ssize_t err, const char *fmt, ...); +static nxt_uint_t nxt_gnutls_log_error_level(nxt_event_conn_t *c, ssize_t err); +static void nxt_cdecl nxt_gnutls_log_error(nxt_uint_t level, nxt_log_t *log, + int err, const char *fmt, ...); + + +const nxt_ssltls_lib_t nxt_gnutls_lib = { + nxt_gnutls_server_init, + NULL, +}; + + +static nxt_event_conn_io_t nxt_gnutls_event_conn_io = { + NULL, + NULL, + + nxt_gnutls_conn_io_read, + NULL, + NULL, + + nxt_event_conn_io_write, + nxt_gnutls_conn_io_write_chunk, + NULL, + NULL, + nxt_gnutls_conn_io_send, + + nxt_gnutls_conn_io_shutdown, +}; + + +static nxt_int_t +nxt_gnutls_start(void) +{ + int ret; + static nxt_bool_t started; + + if (nxt_fast_path(started)) { + return NXT_OK; + } + + started = 1; + + /* TODO: gnutls_global_deinit */ + + ret = gnutls_global_init(); + if (ret != GNUTLS_E_SUCCESS) { + nxt_gnutls_log_error(NXT_LOG_CRIT, nxt_thread_log(), ret, + "gnutls_global_init() failed"); + return NXT_ERROR; + } + + nxt_thread_log_error(NXT_LOG_INFO, "GnuTLS version: %s", + gnutls_check_version(NULL)); + +#if (NXT_HAVE_GNUTLS_SET_TIME) + gnutls_global_set_time_function(nxt_gnutls_time); +#endif + + return NXT_OK; +} + + +#if (NXT_HAVE_GNUTLS_SET_TIME) + +/* GnuTLS 2.12.0 */ + +time_t +nxt_gnutls_time(time_t *tp) +{ + time_t t; + nxt_thread_t *thr; + + thr = nxt_thread(); + nxt_log_debug(thr->log, "gnutls time"); + + t = (time_t) nxt_thread_time(thr); + + if (tp != NULL) { + *tp = t; + } + + return t; +} + +#endif + + +static nxt_int_t +nxt_gnutls_server_init(nxt_ssltls_conf_t *conf) +{ + int ret; + char *certificate, *key, *ca_certificate; + nxt_thread_t *thr; + nxt_gnutls_ctx_t *ctx; + + if (nxt_slow_path(nxt_gnutls_start() != NXT_OK)) { + return NXT_ERROR; + } + + /* TODO: mem_pool, cleanup: gnutls_certificate_free_credentials, + gnutls_priority_deinit */ + + ctx = nxt_zalloc(sizeof(nxt_gnutls_ctx_t)); + if (ctx == NULL) { + return NXT_ERROR; + } + + conf->ctx = ctx; + conf->conn_init = nxt_gnutls_conn_init; + + thr = nxt_thread(); + + ret = gnutls_certificate_allocate_credentials(&ctx->certificate); + if (ret != GNUTLS_E_SUCCESS) { + nxt_gnutls_log_error(NXT_LOG_CRIT, thr->log, ret, + "gnutls_certificate_allocate_credentials() failed"); + return NXT_ERROR; + } + + certificate = conf->certificate; + key = conf->certificate_key; + + ret = gnutls_certificate_set_x509_key_file(ctx->certificate, certificate, + key, GNUTLS_X509_FMT_PEM); + if (ret != GNUTLS_E_SUCCESS) { + nxt_gnutls_log_error(NXT_LOG_CRIT, thr->log, ret, + "gnutls_certificate_set_x509_key_file(\"%s\", \"%s\") failed", + certificate, key); + goto certificate_fail; + } + + if (nxt_gnutls_set_ciphers(conf) != NXT_OK) { + goto ciphers_fail; + } + + if (conf->ca_certificate != NULL) { + ca_certificate = conf->ca_certificate; + + ret = gnutls_certificate_set_x509_trust_file(ctx->certificate, + ca_certificate, + GNUTLS_X509_FMT_PEM); + if (ret < 0) { + nxt_gnutls_log_error(NXT_LOG_CRIT, thr->log, ret, + "gnutls_certificate_set_x509_trust_file(\"%s\") failed", + ca_certificate); + goto ca_certificate_fail; + } + } + + return NXT_OK; + +ca_certificate_fail: + + gnutls_priority_deinit(ctx->ciphers); + +ciphers_fail: + +certificate_fail: + + gnutls_certificate_free_credentials(ctx->certificate); + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_gnutls_set_ciphers(nxt_ssltls_conf_t *conf) +{ + int ret; + const char *ciphers; + const char *err; + nxt_gnutls_ctx_t *ctx; + + ciphers = (conf->ciphers != NULL) ? conf->ciphers : "NORMAL:!COMP-DEFLATE"; + ctx = conf->ctx; + + ret = gnutls_priority_init(&ctx->ciphers, ciphers, &err); + + switch (ret) { + + case GNUTLS_E_SUCCESS: + return NXT_OK; + + case GNUTLS_E_INVALID_REQUEST: + nxt_gnutls_log_error(NXT_LOG_CRIT, nxt_thread_log(), ret, + "gnutls_priority_init(\"%s\") failed at \"%s\"", + ciphers, err); + return NXT_ERROR; + + default: + nxt_gnutls_log_error(NXT_LOG_CRIT, nxt_thread_log(), ret, + "gnutls_priority_init() failed"); + return NXT_ERROR; + } +} + + +static void +nxt_gnutls_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c) +{ + int ret; + gnutls_session_t sess; + nxt_gnutls_ctx_t *ctx; + nxt_gnutls_conn_t *ssltls; + nxt_mem_pool_cleanup_t *mpcl; + + nxt_log_debug(c->socket.log, "gnutls conn init"); + + ssltls = nxt_mem_zalloc(c->mem_pool, sizeof(nxt_gnutls_conn_t)); + if (ssltls == NULL) { + goto fail; + } + + c->u.ssltls = ssltls; + nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size); + + mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0); + if (mpcl == NULL) { + goto fail; + } + + ret = gnutls_init(&ssltls->session, GNUTLS_SERVER); + if (ret != GNUTLS_E_SUCCESS) { + nxt_gnutls_log_error(NXT_LOG_CRIT, c->socket.log, ret, + "gnutls_init() failed"); + goto fail; + } + + sess = ssltls->session; + mpcl->handler = nxt_gnutls_session_cleanup; + mpcl->data = ssltls; + + ctx = conf->ctx; + + ret = gnutls_priority_set(sess, ctx->ciphers); + if (ret != GNUTLS_E_SUCCESS) { + nxt_gnutls_log_error(NXT_LOG_CRIT, c->socket.log, ret, + "gnutls_priority_set() failed"); + goto fail; + } + + /* + * Disable TLS random padding of records in CBC ciphers, + * which may be up to 255 bytes. + */ + gnutls_record_disable_padding(sess); + + ret = gnutls_credentials_set(sess, GNUTLS_CRD_CERTIFICATE, + ctx->certificate); + if (ret != GNUTLS_E_SUCCESS) { + nxt_gnutls_log_error(NXT_LOG_CRIT, c->socket.log, ret, + "gnutls_credentials_set() failed"); + goto fail; + } + + if (conf->ca_certificate != NULL) { + gnutls_certificate_server_set_request(sess, GNUTLS_CERT_REQUEST); + } + + gnutls_transport_set_ptr(sess, (gnutls_transport_ptr_t) c); + gnutls_transport_set_pull_function(sess, nxt_gnutls_pull); + gnutls_transport_set_push_function(sess, nxt_gnutls_push); +#if (NXT_HAVE_GNUTLS_VEC_PUSH) + gnutls_transport_set_vec_push_function(sess, nxt_gnutls_vec_push); +#endif + + c->io = &nxt_gnutls_event_conn_io; + c->sendfile = NXT_CONN_SENDFILE_OFF; + + nxt_gnutls_conn_handshake(thr, c, c->socket.data); + return; + +fail: + + nxt_event_conn_io_handle(thr, c->read_work_queue, + c->read_state->error_handler, c, c->socket.data); +} + + +static void +nxt_gnutls_session_cleanup(void *data) +{ + nxt_gnutls_conn_t *ssltls; + + ssltls = data; + + nxt_thread_log_debug("gnutls session cleanup"); + + nxt_free(ssltls->buffer.start); + + gnutls_deinit(ssltls->session); +} + + +static ssize_t +nxt_gnutls_pull(gnutls_transport_ptr_t data, void *buf, size_t size) +{ + ssize_t n; + nxt_thread_t *thr; + nxt_event_conn_t *c; + + c = data; + thr = nxt_thread(); + + n = thr->engine->event->io->recv(c, buf, size, 0); + + if (n == NXT_AGAIN) { + nxt_set_errno(NXT_EAGAIN); + return -1; + } + + return n; +} + + +static ssize_t +nxt_gnutls_push(gnutls_transport_ptr_t data, const void *buf, size_t size) +{ + ssize_t n; + nxt_thread_t *thr; + nxt_event_conn_t *c; + + c = data; + thr = nxt_thread(); + + n = thr->engine->event->io->send(c, (u_char *) buf, size); + + if (n == NXT_AGAIN) { + nxt_set_errno(NXT_EAGAIN); + return -1; + } + + return n; +} + + +#if (NXT_HAVE_GNUTLS_VEC_PUSH) + +/* GnuTLS 2.12.0 */ + +static ssize_t +nxt_gnutls_vec_push(gnutls_transport_ptr_t data, const giovec_t *iov, + int iovcnt) +{ + ssize_t n; + nxt_thread_t *thr; + nxt_event_conn_t *c; + + c = data; + thr = nxt_thread(); + + /* + * This code assumes that giovec_t is the same as "struct iovec" + * and nxt_iobuf_t. It is not true for Windows. + */ + n = thr->engine->event->io->writev(c, (nxt_iobuf_t *) iov, iovcnt); + + if (n == NXT_AGAIN) { + nxt_set_errno(NXT_EAGAIN); + return -1; + } + + return n; +} + +#endif + + +static void +nxt_gnutls_conn_handshake(nxt_thread_t *thr, void *obj, void *data) +{ + int err; + nxt_int_t ret; + nxt_event_conn_t *c; + nxt_gnutls_conn_t *ssltls; + + c = obj; + ssltls = c->u.ssltls; + + nxt_log_debug(thr->log, "gnutls conn handshake: %d", ssltls->times); + + /* "ssltls->times == 1" is suitable to run gnutls_handshake() in job. */ + + err = gnutls_handshake(ssltls->session); + + nxt_thread_time_debug_update(thr); + + nxt_log_debug(thr->log, "gnutls_handshake(): %d", err); + + if (err == GNUTLS_E_SUCCESS) { + nxt_gnutls_conn_io_read(thr, c, data); + return; + } + + ret = nxt_gnutls_conn_test_error(thr, c, err, nxt_gnutls_conn_handshake); + + if (ret == NXT_ERROR) { + nxt_gnutls_conn_log_error(c, err, "gnutls_handshake() failed"); + + nxt_event_conn_io_handle(thr, c->read_work_queue, + c->read_state->error_handler, c, data); + + } else if (err == GNUTLS_E_AGAIN + && ssltls->times < 2 + && gnutls_record_get_direction(ssltls->session) == 0) + { + ssltls->times++; + } +} + + +static void +nxt_gnutls_conn_io_read(nxt_thread_t *thr, void *obj, void *data) +{ + ssize_t n; + nxt_buf_t *b; + nxt_int_t ret; + nxt_event_conn_t *c; + nxt_gnutls_conn_t *ssltls; + nxt_work_handler_t handler; + + c = obj; + + nxt_log_debug(thr->log, "gnutls conn read"); + + handler = c->read_state->ready_handler; + b = c->read; + + /* b == NULL is used to test descriptor readiness. */ + + if (b != NULL) { + ssltls = c->u.ssltls; + + n = gnutls_record_recv(ssltls->session, b->mem.free, + b->mem.end - b->mem.free); + + nxt_log_debug(thr->log, "gnutls_record_recv(%d, %p, %uz): %z", + c->socket.fd, b->mem.free, b->mem.end - b->mem.free, n); + + if (n > 0) { + /* c->socket.read_ready is kept. */ + b->mem.free += n; + handler = c->read_state->ready_handler; + + } else if (n == 0) { + handler = c->read_state->close_handler; + + } else { + ret = nxt_gnutls_conn_test_error(thr, c, n, + nxt_gnutls_conn_io_read); + + if (nxt_fast_path(ret != NXT_ERROR)) { + return; + } + + nxt_gnutls_conn_log_error(c, n, + "gnutls_record_recv(%d, %p, %uz): failed", + c->socket.fd, b->mem.free, + b->mem.end - b->mem.free); + + handler = c->read_state->error_handler; + } + } + + nxt_event_conn_io_handle(thr, c->read_work_queue, handler, c, data); +} + + +static ssize_t +nxt_gnutls_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, + nxt_buf_t *b, size_t limit) +{ + nxt_gnutls_conn_t *ssltls; + + nxt_log_debug(thr->log, "gnutls conn write chunk"); + + ssltls = c->u.ssltls; + + return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit); +} + + +static ssize_t +nxt_gnutls_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) +{ + ssize_t n; + nxt_int_t ret; + nxt_gnutls_conn_t *ssltls; + + ssltls = c->u.ssltls; + + n = gnutls_record_send(ssltls->session, buf, size); + + nxt_log_debug(c->socket.log, "gnutls_record_send(%d, %p, %uz): %z", + c->socket.fd, buf, size, n); + + if (n > 0) { + return n; + } + + ret = nxt_gnutls_conn_test_error(nxt_thread(), c, n, + nxt_event_conn_io_write); + + if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_gnutls_conn_log_error(c, n, + "gnutls_record_send(%d, %p, %uz): failed", + c->socket.fd, buf, size); + } + + return ret; +} + + +static void +nxt_gnutls_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) +{ + int err; + nxt_int_t ret; + nxt_event_conn_t *c; + nxt_gnutls_conn_t *ssltls; + nxt_work_handler_t handler; + gnutls_close_request_t how; + + c = obj; + ssltls = c->u.ssltls; + + if (ssltls->session == NULL || ssltls->no_shutdown) { + handler = c->write_state->close_handler; + goto done; + } + + nxt_log_debug(c->socket.log, "gnutls conn shutdown"); + + if (c->socket.timedout || c->socket.error != 0) { + how = GNUTLS_SHUT_WR; + + } else if (c->socket.closed) { + how = GNUTLS_SHUT_RDWR; + + } else { + how = GNUTLS_SHUT_RDWR; + } + + err = gnutls_bye(ssltls->session, how); + + nxt_log_debug(c->socket.log, "gnutls_bye(%d, %d): %d", + c->socket.fd, how, err); + + if (err == GNUTLS_E_SUCCESS) { + handler = c->write_state->close_handler; + + } else { + ret = nxt_gnutls_conn_test_error(thr, c, err, + nxt_gnutls_conn_io_shutdown); + + if (ret != NXT_ERROR) { /* ret == NXT_AGAIN */ + c->socket.error_handler = c->read_state->error_handler; + nxt_event_timer_add(thr->engine, &c->read_timer, 5000); + return; + } + + nxt_gnutls_conn_log_error(c, err, "gnutls_bye(%d) failed", + c->socket.fd); + + handler = c->write_state->error_handler; + } + +done: + + nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); +} + + +static nxt_int_t +nxt_gnutls_conn_test_error(nxt_thread_t *thr, nxt_event_conn_t *c, ssize_t err, + nxt_work_handler_t handler) +{ + int ret; + nxt_gnutls_conn_t *ssltls; + + switch (err) { + + case GNUTLS_E_REHANDSHAKE: + case GNUTLS_E_AGAIN: + ssltls = c->u.ssltls; + ret = gnutls_record_get_direction(ssltls->session); + + nxt_log_debug(thr->log, "gnutls_record_get_direction(): %d", ret); + + if (ret == 0) { + /* A read direction. */ + + nxt_event_fd_block_write(thr->engine, &c->socket); + + c->socket.read_ready = 0; + c->socket.read_handler = handler; + + if (nxt_event_fd_is_disabled(c->socket.read)) { + nxt_event_fd_enable_read(thr->engine, &c->socket); + } + + } else { + /* A write direction. */ + + nxt_event_fd_block_read(thr->engine, &c->socket); + + c->socket.write_ready = 0; + c->socket.write_handler = handler; + + if (nxt_event_fd_is_disabled(c->socket.write)) { + nxt_event_fd_enable_write(thr->engine, &c->socket); + } + } + + return NXT_AGAIN; + + default: + c->socket.error = 1000; /* Nonexistent errno code. */ + return NXT_ERROR; + } +} + + +static void +nxt_gnutls_conn_log_error(nxt_event_conn_t *c, ssize_t err, + const char *fmt, ...) +{ + va_list args; + nxt_uint_t level; + u_char *p, msg[NXT_MAX_ERROR_STR]; + + level = nxt_gnutls_log_error_level(c, err); + + if (nxt_log_level_enough(c->socket.log, level)) { + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); + va_end(args); + + nxt_log_error(level, c->socket.log, "%*s (%d: %s)", + p - msg, msg, err, gnutls_strerror(err)); + } +} + + +static nxt_uint_t +nxt_gnutls_log_error_level(nxt_event_conn_t *c, ssize_t err) +{ + nxt_gnutls_conn_t *ssltls; + + switch (err) { + + case GNUTLS_E_UNKNOWN_CIPHER_SUITE: /* -21 */ + + /* Disable gnutls_bye(), because it returns GNUTLS_E_INTERNAL_ERROR. */ + ssltls = c->u.ssltls; + ssltls->no_shutdown = 1; + + /* Fall through. */ + + case GNUTLS_E_UNEXPECTED_PACKET_LENGTH: /* -9 */ + c->socket.error = 1000; /* Nonexistent errno code. */ + break; + + default: + return NXT_LOG_CRIT; + } + + return NXT_LOG_INFO; +} + + +static void +nxt_gnutls_log_error(nxt_uint_t level, nxt_log_t *log, int err, + const char *fmt, ...) +{ + va_list args; + u_char *p, msg[NXT_MAX_ERROR_STR]; + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); + va_end(args); + + nxt_log_error(level, log, "%*s (%d: %s)", + p - msg, msg, err, gnutls_strerror(err)); +} diff --git a/src/nxt_hash.h b/src/nxt_hash.h new file mode 100644 index 00000000..01c727d2 --- /dev/null +++ b/src/nxt_hash.h @@ -0,0 +1,47 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_HASH_H_INCLUDED_ +#define _NXT_HASH_H_INCLUDED_ + + +typedef struct { + nxt_lvlhsh_t lvlhsh; + const nxt_lvlhsh_proto_t *proto; + void *pool; +} nxt_hash_t; + + +nxt_inline nxt_int_t +nxt_hash_find(nxt_hash_t *h, nxt_lvlhsh_query_t *lhq) +{ + lhq->proto = h->proto; + + return nxt_lvlhsh_find(&h->lvlhsh, lhq); +} + + +nxt_inline nxt_int_t +nxt_hash_insert(nxt_hash_t *h, nxt_lvlhsh_query_t *lhq) +{ + lhq->proto = h->proto; + lhq->pool = h->pool; + + return nxt_lvlhsh_insert(&h->lvlhsh, lhq); +} + + +nxt_inline nxt_int_t +nxt_hash_delete(nxt_hash_t *h, nxt_lvlhsh_query_t *lhq) +{ + lhq->proto = h->proto; + lhq->pool = h->pool; + + return nxt_lvlhsh_delete(&h->lvlhsh, lhq); +} + + +#endif /* _NXT_HASH_H_INCLUDED_ */ diff --git a/src/nxt_hpux_sendfile.c b/src/nxt_hpux_sendfile.c new file mode 100644 index 00000000..a105b684 --- /dev/null +++ b/src/nxt_hpux_sendfile.c @@ -0,0 +1,138 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#ifdef NXT_TEST_BUILD_HPUX_SENDFILE + +ssize_t nxt_hpux_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + +static ssize_t nxt_sys_sendfile(int s, int fd, off_t offset, size_t nbytes, + const struct iovec *hdtrl, int flags) +{ + return -1; +} + +#else + +/* sendfile() is not declared if _XOPEN_SOURCE_EXTENDED is defined. */ + +sbsize_t sendfile(int s, int fd, off_t offset, bsize_t nbytes, + const struct iovec *hdtrl, int flags); + +#define nxt_sys_sendfile sendfile + +#endif + + +ssize_t +nxt_hpux_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, size_t limit) +{ + size_t file_size; + ssize_t n; + nxt_buf_t *fb; + nxt_err_t err; + nxt_uint_t nhd, ntr; + struct iovec iov[NXT_IOBUF_MAX], *hdtrl; + nxt_sendbuf_coalesce_t sb; + + sb.buf = b; + sb.iobuf = iov; + sb.nmax = NXT_IOBUF_MAX; + sb.sync = 0; + sb.size = 0; + sb.limit = limit; + + nhd = nxt_sendbuf_mem_coalesce(&sb); + + if (nhd == 0 && sb.sync) { + return 0; + } + + if (nhd > 1 || sb.buf == NULL || !nxt_buf_is_file(sb.buf)) { + return nxt_event_conn_io_writev(c, iov, nhd); + } + + fb = sb.buf; + + file_size = nxt_sendbuf_file_coalesce(&sb); + + if (file_size == 0) { + return nxt_event_conn_io_writev(c, iov, nhd); + } + + sb.iobuf = &iov[1]; + sb.nmax = 1; + + ntr = nxt_sendbuf_mem_coalesce(&sb); + + /* + * Disposal of surplus kernel operations + * if there are no headers and trailers. + */ + + if (nhd == 0) { + hdtrl = NULL; + iov[0].iov_base = NULL; + iov[0].iov_len = 0; + + } else { + hdtrl = iov; + } + + if (ntr == 0) { + iov[1].iov_base = NULL; + iov[1].iov_len = 0; + + } else { + hdtrl = iov; + } + + nxt_log_debug(c->socket.log, "sendfile(%d, %FD, @%O, %uz) hd:%ui tr:%ui", + c->socket.fd, fb->file->fd, fb->file_pos, file_size, + nhd, ntr); + + n = nxt_sys_sendfile(c->socket.fd, fb->file->fd, fb->file_pos, + file_size, hdtrl, 0); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(c->socket.log, "sendfile(): %uz", n); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "sendfile(%d, %FD, @%O, %uz) failed " + "%E \"%FN\" hd:%ui tr:%ui", c->socket.fd, + fb->file->fd, fb->file_pos, file_size, + err, &fb->file->name, nhd, ntr); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "sendfile() %E", err); + + return 0; + } + + if (n < (ssize_t) sb.size) { + c->socket.write_ready = 0; + } + + return n; +} diff --git a/src/nxt_http_chunk_parse.c b/src/nxt_http_chunk_parse.c new file mode 100644 index 00000000..c0402605 --- /dev/null +++ b/src/nxt_http_chunk_parse.c @@ -0,0 +1,263 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#define NXT_HTTP_CHUNK_MIDDLE 0 +#define NXT_HTTP_CHUNK_END_ON_BORDER 1 +#define NXT_HTTP_CHUNK_END 2 + + +#define \ +nxt_size_is_sufficient(cs) \ + (cs < ((__typeof__(cs)) 1 << (sizeof(cs) * 8 - 4))) + + +static nxt_int_t nxt_http_chunk_buffer(nxt_http_chunk_parse_t *hcp, + nxt_buf_t ***tail, nxt_buf_t *in); + + +nxt_buf_t * +nxt_http_chunk_parse(nxt_http_chunk_parse_t *hcp, nxt_buf_t *in) +{ + u_char c, ch; + nxt_int_t ret; + nxt_buf_t *b, *out, *nb, **tail; + nxt_thread_t *thr; + enum { + sw_start = 0, + sw_chunk_size, + sw_chunk_size_linefeed, + sw_chunk_end_newline, + sw_chunk_end_linefeed, + sw_chunk, + } state; + + out = NULL; + tail = &out; + + state = hcp->state; + + for (b = in; b != NULL; b = b->next) { + + hcp->pos = b->mem.pos; + + while (hcp->pos < b->mem.free) { + /* + * The sw_chunk state is tested outside the switch + * to preserve hcp->pos and to not touch memory. + */ + if (state == sw_chunk) { + ret = nxt_http_chunk_buffer(hcp, &tail, b); + + if (ret == NXT_HTTP_CHUNK_MIDDLE) { + goto next; + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + hcp->error = 1; + goto done; + } + + state = sw_chunk_end_newline; + + if (ret == NXT_HTTP_CHUNK_END_ON_BORDER) { + goto next; + } + + /* ret == NXT_HTTP_CHUNK_END_ON_BORDER */ + } + + ch = *hcp->pos++; + + switch (state) { + + case sw_start: + state = sw_chunk_size; + + c = ch - '0'; + + if (c <= 9) { + hcp->chunk_size = c; + continue; + } + + c = (ch | 0x20) - 'a'; + + if (c <= 5) { + hcp->chunk_size = 0x0a + c; + continue; + } + + goto chunk_error; + + case sw_chunk_size: + + c = ch - '0'; + + if (c > 9) { + c = (ch | 0x20) - 'a'; + + if (nxt_fast_path(c <= 5)) { + c += 0x0a; + + } else if (nxt_fast_path(ch == NXT_CR)) { + state = sw_chunk_size_linefeed; + continue; + + } else { + goto chunk_error; + } + } + + if (nxt_fast_path(nxt_size_is_sufficient(hcp->chunk_size))) { + hcp->chunk_size = (hcp->chunk_size << 4) + c; + continue; + } + + goto chunk_error; + + case sw_chunk_size_linefeed: + if (nxt_fast_path(ch == NXT_LF)) { + + if (hcp->chunk_size != 0) { + state = sw_chunk; + continue; + } + + hcp->last = 1; + state = sw_chunk_end_newline; + continue; + } + + goto chunk_error; + + case sw_chunk_end_newline: + if (nxt_fast_path(ch == NXT_CR)) { + state = sw_chunk_end_linefeed; + continue; + } + + goto chunk_error; + + case sw_chunk_end_linefeed: + if (nxt_fast_path(ch == NXT_LF)) { + + if (!hcp->last) { + state = sw_start; + continue; + } + + goto done; + } + + goto chunk_error; + + case sw_chunk: + /* + * This state is processed before the switch. + * It added here just to suppress a warning. + */ + continue; + } + } + + if (b->retain == 0) { + /* No chunk data was found in a buffer. */ + thr = nxt_thread(); + nxt_thread_current_work_queue_add(thr, b->completion_handler, + b, b->parent, thr->log); + + } + + next: + + continue; + } + + hcp->state = state; + + return out; + +chunk_error: + + hcp->chunk_error = 1; + +done: + + nb = nxt_buf_sync_alloc(hcp->mem_pool, NXT_BUF_SYNC_LAST); + + if (nxt_fast_path(nb != NULL)) { + *tail = nb; + + } else { + hcp->error = 1; + } + + // STUB: hcp->chunk_error = 1; + // STUB: hcp->error = 1; + + return out; +} + + +static nxt_int_t +nxt_http_chunk_buffer(nxt_http_chunk_parse_t *hcp, nxt_buf_t ***tail, + nxt_buf_t *in) +{ + u_char *p; + size_t size; + nxt_buf_t *b; + + p = hcp->pos; + size = in->mem.free - p; + + if (hcp->chunk_size >= size && in->retain == 0) { + /* + * Use original buffer if the buffer is lesser than or equal + * to a chunk size and this is the first chunk in the buffer. + */ + in->mem.pos = p; + **tail = in; + *tail = &in->next; + + } else { + b = nxt_buf_mem_alloc(hcp->mem_pool, 0, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + **tail = b; + *tail = &b->next; + + b->parent = in; + in->retain++; + b->mem.pos = p; + b->mem.start = p; + + if (hcp->chunk_size < size) { + p += hcp->chunk_size; + hcp->pos = p; + + b->mem.free = p; + b->mem.end = p; + + return NXT_HTTP_CHUNK_END; + } + + b->mem.free = in->mem.free; + b->mem.end = in->mem.free; + } + + hcp->chunk_size -= size; + + if (hcp->chunk_size == 0) { + return NXT_HTTP_CHUNK_END_ON_BORDER; + } + + return NXT_HTTP_CHUNK_MIDDLE; +} diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c new file mode 100644 index 00000000..fbc2f73f --- /dev/null +++ b/src/nxt_http_parse.c @@ -0,0 +1,595 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_http_split_header_part(nxt_http_split_header_parse_t *shp, + u_char *start, u_char *end); +static nxt_int_t nxt_http_split_header_join(nxt_http_split_header_parse_t *shp); + + +nxt_int_t +nxt_http_status_parse(nxt_http_status_parse_t *sp, nxt_buf_mem_t *b) +{ + u_char ch, *p; + enum { + sw_start = 0, + sw_H, + sw_HT, + sw_HTT, + sw_HTTP, + sw_major_digit, + sw_dot, + sw_minor_digit, + sw_space_after_version, + sw_status_start, + sw_status_code, + sw_status_text, + sw_end, + } state; + + state = sp->state; + + for (p = b->pos; p < b->free; p++) { + + ch = *p; + + switch (state) { + + /* "HTTP/" */ + case sw_start: + if (nxt_fast_path(ch == 'H')) { + state = sw_H; + continue; + } + + return NXT_ERROR; + + case sw_H: + if (nxt_fast_path(ch == 'T')) { + state = sw_HT; + continue; + } + + return NXT_ERROR; + + case sw_HT: + if (nxt_fast_path(ch == 'T')) { + state = sw_HTT; + continue; + } + + return NXT_ERROR; + + case sw_HTT: + if (nxt_fast_path(ch == 'P')) { + state = sw_HTTP; + continue; + } + + return NXT_ERROR; + + case sw_HTTP: + if (nxt_fast_path(ch == '/')) { + state = sw_major_digit; + continue; + } + + return NXT_ERROR; + + /* + * Only HTTP/x.x format is tested because it + * is unlikely that other formats will appear. + */ + case sw_major_digit: + if (nxt_fast_path(ch >= '1' && ch <= '9')) { + sp->http_version = 10 * (ch - '0'); + state = sw_dot; + continue; + } + + return NXT_ERROR; + + case sw_dot: + if (nxt_fast_path(ch == '.')) { + state = sw_minor_digit; + continue; + } + + return NXT_ERROR; + + case sw_minor_digit: + if (nxt_fast_path(ch >= '0' && ch <= '9')) { + sp->http_version += ch - '0'; + state = sw_space_after_version; + continue; + } + + return NXT_ERROR; + + case sw_space_after_version: + if (nxt_fast_path(ch == ' ')) { + state = sw_status_start; + continue; + } + + return NXT_ERROR; + + case sw_status_start: + if (nxt_slow_path(ch == ' ')) { + continue; + } + + sp->start = p; + state = sw_status_code; + + /* Fall through. */ + + /* HTTP status code. */ + case sw_status_code: + if (nxt_fast_path(ch >= '0' && ch <= '9')) { + sp->code = sp->code * 10 + (ch - '0'); + continue; + } + + switch (ch) { + case ' ': + state = sw_status_text; + continue; + case '.': /* IIS may send 403.1, 403.2, etc. */ + state = sw_status_text; + continue; + case NXT_CR: + sp->end = p; + state = sw_end; + continue; + case NXT_LF: + sp->end = p; + goto done; + default: + return NXT_ERROR; + } + + /* Any text until end of line. */ + case sw_status_text: + switch (ch) { + case NXT_CR: + sp->end = p; + state = sw_end; + continue; + case NXT_LF: + sp->end = p; + goto done; + } + continue; + + /* End of status line. */ + case sw_end: + if (nxt_fast_path(ch == NXT_LF)) { + goto done; + } + + return NXT_ERROR; + } + } + + b->pos = p; + sp->state = state; + + return NXT_AGAIN; + +done: + + b->pos = p + 1; + + return NXT_OK; +} + + +nxt_int_t +nxt_http_header_parse(nxt_http_header_parse_t *hp, nxt_buf_mem_t *b) +{ + u_char c, ch, *p; + uint32_t hash; + enum { + sw_start = 0, + sw_name, + sw_space_before_value, + sw_value, + sw_space_after_value, + sw_ignore_line, + sw_almost_done, + sw_header_almost_done, + } state; + + static const u_char normal[256] nxt_aligned(64) = + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0-\0\0" "0123456789\0\0\0\0\0\0" + + /* These 64 bytes should reside in one cache line */ + "\0abcdefghijklmnopqrstuvwxyz\0\0\0\0\0" + "\0abcdefghijklmnopqrstuvwxyz\0\0\0\0\0" + + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + + nxt_prefetch(&normal[0]); + nxt_prefetch(&normal[64]); + + state = hp->state; + hash = hp->header_hash; + + for (p = b->pos; p < b->free; p++) { + ch = *p; + + switch (state) { + + /* first char */ + case sw_start: + hp->header_name_start = p; + hp->invalid_header = 0; + + switch (ch) { + case NXT_CR: + hp->header_end = p; + state = sw_header_almost_done; + break; + case NXT_LF: + hp->header_end = p; + goto header_done; + default: + state = sw_name; + + c = normal[ch]; + + if (c) { + hash = nxt_djb_hash_add(NXT_DJB_HASH_INIT, c); + break; + } + + if (ch == '_') { + hash = nxt_djb_hash_add(NXT_DJB_HASH_INIT, ch); + hp->underscore = 1; + break; + } + + hp->invalid_header = 1; + break; + } + break; + + /* header name */ + case sw_name: + c = normal[ch]; + + if (c) { + hash = nxt_djb_hash_add(hash, c); + break; + } + + if (ch == ':') { + hp->header_name_end = p; + state = sw_space_before_value; + break; + } + + if (ch == NXT_CR) { + hp->header_name_end = p; + hp->header_start = p; + hp->header_end = p; + state = sw_almost_done; + break; + } + + if (ch == NXT_LF) { + hp->header_name_end = p; + hp->header_start = p; + hp->header_end = p; + goto done; + } + + if (ch == '_') { + hash = nxt_djb_hash_add(hash, ch); + hp->underscore = 1; + break; + } + + /* IIS may send the duplicate "HTTP/1.1 ..." lines */ + if (ch == '/' + && hp->upstream + && p - hp->header_name_start == 4 + && nxt_memcmp(hp->header_name_start, "HTTP", 4) == 0) + { + state = sw_ignore_line; + break; + } + + hp->invalid_header = 1; + break; + + /* space* before header value */ + case sw_space_before_value: + switch (ch) { + case ' ': + break; + case NXT_CR: + hp->header_start = p; + hp->header_end = p; + state = sw_almost_done; + break; + case NXT_LF: + hp->header_start = p; + hp->header_end = p; + goto done; + case '\0': + hp->invalid_header = 1; + /* Fall through. */ + default: + hp->header_start = p; + state = sw_value; + break; + } + break; + + /* header value */ + case sw_value: + switch (ch) { + case ' ': + hp->header_end = p; + state = sw_space_after_value; + break; + case NXT_CR: + hp->header_end = p; + state = sw_almost_done; + break; + case NXT_LF: + hp->header_end = p; + goto done; + case '\0': + hp->invalid_header = 1; + break; + } + break; + + /* space* before end of header line */ + case sw_space_after_value: + switch (ch) { + case ' ': + break; + case NXT_CR: + state = sw_almost_done; + break; + case NXT_LF: + goto done; + case '\0': + hp->invalid_header = 1; + /* Fall through. */ + default: + state = sw_value; + break; + } + break; + + /* ignore header line */ + case sw_ignore_line: + switch (ch) { + case NXT_LF: + state = sw_start; + break; + default: + break; + } + break; + + /* end of header line */ + case sw_almost_done: + switch (ch) { + case NXT_LF: + goto done; + case NXT_CR: + break; + default: + return NXT_DECLINED; + } + break; + + /* end of header */ + case sw_header_almost_done: + switch (ch) { + case NXT_LF: + goto header_done; + default: + return NXT_DECLINED; + } + } + } + + b->pos = p; + hp->state = state; + hp->header_hash = hash; + + return NXT_AGAIN; + +done: + + b->pos = p + 1; + hp->state = sw_start; + hp->header_hash = hash; + + return NXT_OK; + +header_done: + + b->pos = p + 1; + hp->state = sw_start; + + return NXT_DONE; +} + + +nxt_int_t +nxt_http_split_header_parse(nxt_http_split_header_parse_t *shp, + nxt_buf_mem_t *b) +{ + u_char *end; + nxt_int_t ret; + + if (shp->parts == NULL || nxt_array_is_empty(shp->parts)) { + + ret = nxt_http_header_parse(&shp->parse, b); + + if (nxt_fast_path(ret == NXT_OK)) { + return ret; + } + + if (nxt_fast_path(ret == NXT_AGAIN)) { + /* A buffer is over. */ + + if (shp->parse.state == 0) { + /* + * A previous parsed header line is + * over right on the end of the buffer. + */ + return ret; + } + /* + * Add the first header line part and return NXT_AGAIN on success. + */ + return nxt_http_split_header_part(shp, shp->parse.header_name_start, + b->pos); + } + + return ret; + } + + /* A header line is split in buffers. */ + + end = nxt_memchr(b->pos, NXT_LF, b->free - b->pos); + + if (end != NULL) { + + /* The last header line part found. */ + end++; + + ret = nxt_http_split_header_part(shp, b->pos, end); + + if (nxt_fast_path(ret != NXT_ERROR)) { + /* ret == NXT_AGAIN: success, mark the part if it were parsed. */ + b->pos = end; + + return nxt_http_split_header_join(shp); + } + + return ret; + } + + /* Add another header line part and return NXT_AGAIN on success. */ + + return nxt_http_split_header_part(shp, b->pos, b->free); +} + + +static nxt_int_t +nxt_http_split_header_part(nxt_http_split_header_parse_t *shp, u_char *start, + u_char *end) +{ + nxt_http_header_part_t *part; + + nxt_thread_log_debug("http source header part: \"%*s\"", + end - start, start); + + if (shp->parts == NULL) { + shp->parts = nxt_array_create(shp->mem_pool, 2, + sizeof(nxt_http_header_part_t)); + if (nxt_slow_path(shp->parts == NULL)) { + return NXT_ERROR; + } + } + + if (!nxt_array_is_empty(shp->parts)) { + + part = nxt_array_last(shp->parts); + + if (part->end == end) { + part->end = end; + return NXT_AGAIN; + } + } + + part = nxt_array_add(shp->parts); + + if (nxt_fast_path(part != NULL)) { + part->start = start; + part->end = end; + return NXT_AGAIN; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_http_split_header_join(nxt_http_split_header_parse_t *shp) +{ + u_char *p; + size_t size; + nxt_uint_t n; + nxt_buf_mem_t b; + nxt_http_header_part_t *part; + + part = shp->parts->elts; + n = shp->parts->nelts; + + if (n == 1) { + /* + * A header line was read by parts, but resides continuously in a + * stream source buffer, so use disposition in the original buffer. + */ + b.pos = part->start; + b.free = part->end; + + } else { + /* Join header line parts to store the header line and ot parse it. */ + + size = 0; + + do { + size += part->end - part->start; + part++; + n--; + } while (n != 0); + + p = nxt_mem_alloc(shp->mem_pool, size); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + b.pos = p; + + part = shp->parts->elts; + n = shp->parts->nelts; + + do { + p = nxt_cpymem(p, part->start, part->end - part->start); + part++; + n--; + } while (n != 0); + + b.free = p; + } + + /* b.start and b.end are not required for parsing. */ + + nxt_array_reset(shp->parts); + + /* Reset a header parse state to the sw_start. */ + shp->parse.state = 0; + + return nxt_http_header_parse(&shp->parse, &b); +} diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h new file mode 100644 index 00000000..e61aa16c --- /dev/null +++ b/src/nxt_http_parse.h @@ -0,0 +1,79 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_HTTP_PARSE_H_INCLUDED_ +#define _NXT_HTTP_PARSE_H_INCLUDED_ + + +typedef struct { + uint8_t state; + uint8_t http_version; + + uint32_t code; + + u_char *start; + u_char *end; +} nxt_http_status_parse_t; + + +nxt_int_t nxt_http_status_parse(nxt_http_status_parse_t *sp, nxt_buf_mem_t *b); + + +typedef struct { + uint32_t header_hash; + + uint8_t state; + uint8_t underscore; /* 1 bit */ + uint8_t invalid_header; /* 1 bit */ + uint8_t upstream; /* 1 bit */ + + u_char *header_start; + u_char *header_end; + u_char *header_name_start; + u_char *header_name_end; +} nxt_http_header_parse_t; + + +NXT_EXPORT nxt_int_t nxt_http_header_parse(nxt_http_header_parse_t *hp, + nxt_buf_mem_t *b); + + +typedef struct { + u_char *start; + u_char *end; +} nxt_http_header_part_t; + + +typedef struct { + nxt_array_t *parts; /* of nxt_http_header_part_t */ + nxt_mem_pool_t *mem_pool; + + nxt_http_header_parse_t parse; +} nxt_http_split_header_parse_t; + + +nxt_int_t nxt_http_split_header_parse(nxt_http_split_header_parse_t *shp, + nxt_buf_mem_t *b); + + +typedef struct { + u_char *pos; + nxt_mem_pool_t *mem_pool; + + uint64_t chunk_size; + + uint8_t state; + uint8_t last; /* 1 bit */ + uint8_t chunk_error; /* 1 bit */ + uint8_t error; /* 1 bit */ +} nxt_http_chunk_parse_t; + + +NXT_EXPORT nxt_buf_t *nxt_http_chunk_parse(nxt_http_chunk_parse_t *hcp, + nxt_buf_t *in); + + +#endif /* _NXT_HTTP_PARSE_H_INCLUDED_ */ diff --git a/src/nxt_http_source.c b/src/nxt_http_source.c new file mode 100644 index 00000000..045d585b --- /dev/null +++ b/src/nxt_http_source.c @@ -0,0 +1,630 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +typedef struct { + nxt_http_chunk_parse_t parse; + nxt_source_hook_t next; +} nxt_http_source_chunk_t; + + +static nxt_buf_t *nxt_http_source_request_create(nxt_http_source_t *hs); + +static void nxt_http_source_status_filter(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_http_source_header_filter(nxt_thread_t *thr, void *obj, + void *data); + +static nxt_int_t nxt_http_source_header_line_process(nxt_http_source_t *hs); +static nxt_int_t nxt_http_source_content_length(nxt_upstream_source_t *us, + nxt_name_value_t *nv); +static nxt_int_t nxt_http_source_transfer_encoding(nxt_upstream_source_t *us, + nxt_name_value_t *nv); + +static void nxt_http_source_header_ready(nxt_http_source_t *hs, + nxt_buf_t *rest); +static void nxt_http_source_chunk_filter(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_http_source_chunk_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_http_source_body_filter(nxt_thread_t *thr, void *obj, + void *data); + +static void nxt_http_source_sync_buffer(nxt_thread_t *thr, + nxt_http_source_t *hs, nxt_buf_t *b); +static void nxt_http_source_error(nxt_stream_source_t *stream); +static void nxt_http_source_fail(nxt_http_source_t *hs); +static void nxt_http_source_message(const char *msg, size_t len, u_char *p); + + +void +nxt_http_source_handler(nxt_upstream_source_t *us, + nxt_http_source_request_create_t request_create) +{ + nxt_http_source_t *hs; + nxt_stream_source_t *stream; + + hs = nxt_mem_zalloc(us->buffers.mem_pool, sizeof(nxt_http_source_t)); + if (nxt_slow_path(hs == NULL)) { + goto fail; + } + + us->protocol_source = hs; + + hs->header_in.list = nxt_list_create(us->buffers.mem_pool, 8, + sizeof(nxt_name_value_t)); + if (nxt_slow_path(hs->header_in.list == NULL)) { + goto fail; + } + + hs->header_in.hash = us->header_hash; + hs->upstream = us; + hs->request_create = request_create; + + stream = us->stream; + + if (stream == NULL) { + stream = nxt_mem_zalloc(us->buffers.mem_pool, + sizeof(nxt_stream_source_t)); + if (nxt_slow_path(stream == NULL)) { + goto fail; + } + + us->stream = stream; + stream->upstream = us; + + } else { + nxt_memzero(stream, sizeof(nxt_stream_source_t)); + } + + /* + * Create the HTTP source filter chain: + * stream source | HTTP status line filter + */ + stream->next = &hs->query; + stream->error_handler = nxt_http_source_error; + + hs->query.context = hs; + hs->query.filter = nxt_http_source_status_filter; + + hs->header_in.content_length = -1; + + stream->out = nxt_http_source_request_create(hs); + + if (nxt_fast_path(stream->out != NULL)) { + nxt_memzero(&hs->u.status_parse, sizeof(nxt_http_status_parse_t)); + + nxt_stream_source_connect(stream); + return; + } + +fail: + + nxt_http_source_fail(hs); +} + + +nxt_inline u_char * +nxt_http_source_copy(u_char *p, nxt_str_t *src, size_t len) +{ + u_char *s; + + if (nxt_fast_path(len >= src->len)) { + len = src->len; + src->len = 0; + + } else { + src->len -= len; + } + + s = src->data; + src->data += len; + + return nxt_cpymem(p, s, len); +} + + +static nxt_buf_t * +nxt_http_source_request_create(nxt_http_source_t *hs) +{ + nxt_int_t ret; + nxt_buf_t *b, *req, **prev; + + nxt_thread_log_debug("http source create request"); + + prev = &req; + +new_buffer: + + ret = nxt_buf_pool_mem_alloc(&hs->upstream->buffers, 0); + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + + b = hs->upstream->buffers.current; + hs->upstream->buffers.current = NULL; + + *prev = b; + prev = &b->next; + + for ( ;; ) { + ret = hs->request_create(hs); + + if (nxt_fast_path(ret == NXT_OK)) { + b->mem.free = nxt_http_source_copy(b->mem.free, &hs->u.request.copy, + b->mem.end - b->mem.free); + + if (nxt_fast_path(hs->u.request.copy.len == 0)) { + continue; + } + + nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, + b->mem.pos); + + goto new_buffer; + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + return NULL; + } + + /* ret == NXT_DONE */ + break; + } + + nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, b->mem.pos); + + return req; +} + + +static void +nxt_http_source_status_filter(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_int_t ret; + nxt_buf_t *b; + nxt_http_source_t *hs; + + hs = obj; + b = data; + + /* + * No cycle over buffer chain is required since at + * start the stream source passes buffers one at a time. + */ + + nxt_log_debug(thr->log, "http source status filter"); + + if (nxt_slow_path(nxt_buf_is_sync(b))) { + nxt_http_source_sync_buffer(thr, hs, b); + return; + } + + ret = nxt_http_status_parse(&hs->u.status_parse, &b->mem); + + if (nxt_fast_path(ret == NXT_OK)) { + /* + * Change the HTTP source filter chain: + * stream source | HTTP header filter + */ + hs->query.filter = nxt_http_source_header_filter; + + nxt_log_debug(thr->log, "upstream status: \"%*s\"", + hs->u.status_parse.end - b->mem.start, b->mem.start); + + hs->header_in.status = hs->u.status_parse.code; + + nxt_log_debug(thr->log, "upstream version:%d status:%uD \"%*s\"", + hs->u.status_parse.http_version, + hs->u.status_parse.code, + hs->u.status_parse.end - hs->u.status_parse.start, + hs->u.status_parse.start); + + nxt_memzero(&hs->u.header, sizeof(nxt_http_split_header_parse_t)); + hs->u.header.mem_pool = hs->upstream->buffers.mem_pool; + + nxt_http_source_header_filter(thr, hs, b); + return; + } + + if (nxt_slow_path(ret == NXT_ERROR)) { + /* HTTP/0.9 response. */ + hs->header_in.status = 200; + nxt_http_source_header_ready(hs, b); + return; + } + + /* ret == NXT_AGAIN */ + + /* + * b->mem.pos is always equal to b->mem.end because b is a buffer + * which points to a response part read by the stream source. + * However, since the stream source is an immediate source of the + * status filter, b->parent is a buffer the stream source reads in. + */ + if (b->parent->mem.pos == b->parent->mem.end) { + nxt_http_source_message("upstream sent too long status line: \"%*s\"", + b->mem.pos - b->mem.start, b->mem.start); + + nxt_http_source_fail(hs); + } +} + + +static void +nxt_http_source_header_filter(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_int_t ret; + nxt_buf_t *b; + nxt_http_source_t *hs; + + hs = obj; + b = data; + + /* + * No cycle over buffer chain is required since at + * start the stream source passes buffers one at a time. + */ + + nxt_log_debug(thr->log, "http source header filter"); + + if (nxt_slow_path(nxt_buf_is_sync(b))) { + nxt_http_source_sync_buffer(thr, hs, b); + return; + } + + for ( ;; ) { + ret = nxt_http_split_header_parse(&hs->u.header, &b->mem); + + if (nxt_slow_path(ret != NXT_OK)) { + break; + } + + ret = nxt_http_source_header_line_process(hs); + + if (nxt_slow_path(ret != NXT_OK)) { + break; + } + } + + if (nxt_fast_path(ret == NXT_DONE)) { + nxt_log_debug(thr->log, "http source header done"); + nxt_http_source_header_ready(hs, b); + return; + } + + if (nxt_fast_path(ret == NXT_AGAIN)) { + return; + } + + if (ret != NXT_ERROR) { + /* ret == NXT_DECLINED: "\r" is not followed by "\n" */ + nxt_log_error(NXT_LOG_ERR, thr->log, + "upstream sent invalid header line: \"%*s\\r...\"", + hs->u.header.parse.header_end + - hs->u.header.parse.header_name_start, + hs->u.header.parse.header_name_start); + } + + /* ret == NXT_ERROR */ + + nxt_http_source_fail(hs); +} + + +static nxt_int_t +nxt_http_source_header_line_process(nxt_http_source_t *hs) +{ + size_t name_len; + nxt_name_value_t *nv; + nxt_lvlhsh_query_t lhq; + nxt_http_header_parse_t *hp; + nxt_upstream_name_value_t *unv; + + hp = &hs->u.header.parse; + + name_len = hp->header_name_end - hp->header_name_start; + + if (name_len > 255) { + nxt_http_source_message("upstream sent too long header field name: " + "\"%*s\"", name_len, hp->header_name_start); + return NXT_ERROR; + } + + nv = nxt_list_add(hs->header_in.list); + if (nxt_slow_path(nv == NULL)) { + return NXT_ERROR; + } + + nv->hash = hp->header_hash; + nv->skip = 0; + nv->name_len = name_len; + nv->name_start = hp->header_name_start; + nv->value_len = hp->header_end - hp->header_start; + nv->value_start = hp->header_start; + + nxt_thread_log_debug("upstream header: \"%*s: %*s\"", + nv->name_len, nv->name_start, + nv->value_len, nv->value_start); + + lhq.key_hash = nv->hash; + lhq.key.len = nv->name_len; + lhq.key.data = nv->name_start; + lhq.proto = &nxt_upstream_header_hash_proto; + + if (nxt_lvlhsh_find(&hs->header_in.hash, &lhq) == NXT_OK) { + unv = lhq.value; + + if (unv->handler(hs->upstream, nv) != NXT_OK) { + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +static const nxt_upstream_name_value_t nxt_http_source_headers[] + nxt_aligned(32) = +{ + { nxt_http_source_content_length, + nxt_upstream_name_value("content-length") }, + + { nxt_http_source_transfer_encoding, + nxt_upstream_name_value("transfer-encoding") }, +}; + + +nxt_int_t +nxt_http_source_hash_create(nxt_mem_pool_t *mp, nxt_lvlhsh_t *lh) +{ + return nxt_upstream_header_hash_add(mp, lh, nxt_http_source_headers, + nxt_nitems(nxt_http_source_headers)); +} + + +static nxt_int_t +nxt_http_source_content_length(nxt_upstream_source_t *us, nxt_name_value_t *nv) +{ + nxt_off_t length; + nxt_http_source_t *hs; + + length = nxt_off_t_parse(nv->value_start, nv->value_len); + + if (nxt_fast_path(length > 0)) { + hs = us->protocol_source; + hs->header_in.content_length = length; + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_http_source_transfer_encoding(nxt_upstream_source_t *us, + nxt_name_value_t *nv) +{ + u_char *end; + nxt_http_source_t *hs; + + end = nv->value_start + nv->value_len; + + if (nxt_memcasestrn(nv->value_start, end, "chunked", 7) != NULL) { + hs = us->protocol_source; + hs->chunked = 1; + } + + return NXT_OK; +} + + +static void +nxt_http_source_header_ready(nxt_http_source_t *hs, nxt_buf_t *rest) +{ + nxt_buf_t *b; + nxt_upstream_source_t *us; + nxt_http_source_chunk_t *hsc; + + us = hs->upstream; + + /* Free buffers used for request header. */ + + for (b = us->stream->out; b != NULL; b = b->next) { + nxt_buf_pool_free(&us->buffers, b); + } + + if (nxt_fast_path(nxt_buf_pool_available(&us->buffers))) { + + if (hs->chunked) { + hsc = nxt_mem_zalloc(hs->upstream->buffers.mem_pool, + sizeof(nxt_http_source_chunk_t)); + if (nxt_slow_path(hsc == NULL)) { + goto fail; + } + + /* + * Change the HTTP source filter chain: + * stream source | chunk filter | HTTP body filter + */ + hs->query.context = hsc; + hs->query.filter = nxt_http_source_chunk_filter; + + hsc->next.context = hs; + hsc->next.filter = nxt_http_source_body_filter; + + hsc->parse.mem_pool = hs->upstream->buffers.mem_pool; + + if (nxt_buf_mem_used_size(&rest->mem) != 0) { + hs->rest = nxt_http_chunk_parse(&hsc->parse, rest); + + if (nxt_slow_path(hs->rest == NULL)) { + goto fail; + } + } + + } else { + /* + * Change the HTTP source filter chain: + * stream source | HTTP body filter + */ + hs->query.filter = nxt_http_source_body_filter; + + if (nxt_buf_mem_used_size(&rest->mem) != 0) { + hs->rest = rest; + } + } + + hs->upstream->state->ready_handler(hs); + return; + } + + nxt_thread_log_error(NXT_LOG_ERR, "%d buffers %uDK each " + "are not enough to read upstream response", + us->buffers.max, us->buffers.size / 1024); +fail: + + nxt_http_source_fail(hs); +} + + +static void +nxt_http_source_chunk_filter(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_http_source_t *hs; + nxt_http_source_chunk_t *hsc; + + hsc = obj; + b = data; + + nxt_log_debug(thr->log, "http source chunk filter"); + + b = nxt_http_chunk_parse(&hsc->parse, b); + + hs = hsc->next.context; + + if (hsc->parse.error) { + nxt_http_source_fail(hs); + return; + } + + if (hsc->parse.chunk_error) { + /* Output all parsed before a chunk error and close upstream. */ + nxt_thread_current_work_queue_add(thr, nxt_http_source_chunk_error, + hs, NULL, thr->log); + } + + if (b != NULL) { + nxt_source_filter(thr, hs->upstream->work_queue, &hsc->next, b); + } +} + + +static void +nxt_http_source_chunk_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_http_source_t *hs; + + hs = obj; + + nxt_http_source_fail(hs); +} + + +/* + * The HTTP source body filter accumulates first body buffers before the next + * filter will be established and sets completion handler for the last buffer. + */ + +static void +nxt_http_source_body_filter(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b, *in; + nxt_http_source_t *hs; + + hs = obj; + in = data; + + nxt_log_debug(thr->log, "http source body filter"); + + for (b = in; b != NULL; b = b->next) { + + if (nxt_buf_is_last(b)) { + b->data = hs->upstream->data; + b->completion_handler = hs->upstream->state->completion_handler; + } + } + + if (hs->next != NULL) { + nxt_source_filter(thr, hs->upstream->work_queue, hs->next, in); + return; + } + + nxt_buf_chain_add(&hs->rest, in); +} + + +static void +nxt_http_source_sync_buffer(nxt_thread_t *thr, nxt_http_source_t *hs, + nxt_buf_t *b) +{ + if (nxt_buf_is_last(b)) { + nxt_log_error(NXT_LOG_ERR, thr->log, + "upstream closed prematurely connection"); + + } else { + nxt_log_error(NXT_LOG_ERR, thr->log, "%ui buffers %uz each are not " + "enough to process upstream response header", + hs->upstream->buffers.max, + hs->upstream->buffers.size); + } + + /* The stream source sends only the last and the nobuf sync buffer. */ + + nxt_http_source_fail(hs); +} + + +static void +nxt_http_source_error(nxt_stream_source_t *stream) +{ + nxt_http_source_t *hs; + + nxt_thread_log_debug("http source error"); + + hs = stream->next->context; + nxt_http_source_fail(hs); +} + + +static void +nxt_http_source_fail(nxt_http_source_t *hs) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + + nxt_log_debug(thr->log, "http source fail"); + + /* TODO: fail, next upstream, or bad gateway */ + + hs->upstream->state->error_handler(thr, hs, NULL); +} + + +static void +nxt_http_source_message(const char *msg, size_t len, u_char *p) +{ + if (len > NXT_MAX_ERROR_STR - 300) { + len = NXT_MAX_ERROR_STR - 300; + p[len++] = '.'; p[len++] = '.'; p[len++] = '.'; + } + + nxt_thread_log_error(NXT_LOG_ERR, msg, len, p); +} diff --git a/src/nxt_http_source.h b/src/nxt_http_source.h new file mode 100644 index 00000000..53924a4f --- /dev/null +++ b/src/nxt_http_source.h @@ -0,0 +1,49 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_HTTP_SOURCE_H_INCLUDED_ +#define _NXT_HTTP_SOURCE_H_INCLUDED_ + + +typedef struct { + nxt_str_t copy; + uintptr_t data[3]; +} nxt_http_source_request_t; + + +typedef struct nxt_http_source_s nxt_http_source_t; +typedef nxt_int_t (*nxt_http_source_request_create_t)(nxt_http_source_t *hs); + + +struct nxt_http_source_s { + nxt_source_hook_t query; + nxt_source_hook_t *next; + + nxt_upstream_source_t *upstream; + + nxt_http_source_request_create_t request_create; + + nxt_upstream_header_in_t header_in; + + nxt_buf_t *rest; + + uint32_t chunked; /* 1 bit */ + + union { + nxt_http_source_request_t request; + nxt_http_status_parse_t status_parse; + nxt_http_split_header_parse_t header; + } u; +}; + + +NXT_EXPORT void nxt_http_source_handler(nxt_upstream_source_t *us, + nxt_http_source_request_create_t request_create); +NXT_EXPORT nxt_int_t nxt_http_source_hash_create(nxt_mem_pool_t *mp, + nxt_lvlhsh_t *lh); + + +#endif /* _NXT_HTTP_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_job.c b/src/nxt_job.c new file mode 100644 index 00000000..2b7d8818 --- /dev/null +++ b/src/nxt_job.c @@ -0,0 +1,202 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#if (NXT_THREADS) +static void nxt_job_thread_trampoline(nxt_thread_t *thr, void *obj, void *data); +static void nxt_job_thread_return_handler(nxt_thread_t *thr, void *obj, + void *data); +#endif + + +void * +nxt_job_create(nxt_mem_pool_t *mp, size_t size) +{ + size_t cache_size; + nxt_job_t *job; + + if (mp == NULL) { + mp = nxt_mem_pool_create(256); + + if (nxt_slow_path(mp == NULL)) { + return NULL; + } + + job = nxt_mem_zalloc(mp, size); + cache_size = 0; + + } else { + job = nxt_mem_cache_zalloc0(mp, size); + cache_size = size; + } + + if (nxt_fast_path(job != NULL)) { + job->cache_size = (uint16_t) cache_size; + job->mem_pool = mp; + nxt_job_set_name(job, "job"); + } + + /* Allow safe nxt_queue_remove() in nxt_job_destroy(). */ + nxt_queue_self(&job->link); + + return job; +} + + +void +nxt_job_init(nxt_job_t *job, size_t size) +{ + nxt_memzero(job, size); + + nxt_job_set_name(job, "job"); + + nxt_queue_self(&job->link); +} + + +void +nxt_job_destroy(void *data) +{ + nxt_job_t *job; + + job = data; + + nxt_queue_remove(&job->link); + + if (job->cache_size == 0) { + + if (job->mem_pool != NULL) { + nxt_mem_pool_destroy(job->mem_pool); + } + + } else { + nxt_mem_cache_free0(job->mem_pool, job, job->cache_size); + } +} + + +nxt_int_t +nxt_job_cleanup_add(nxt_mem_pool_t *mp, nxt_job_t *job) +{ + nxt_mem_pool_cleanup_t *mpcl; + + mpcl = nxt_mem_pool_cleanup(mp, 0); + + if (nxt_fast_path(mpcl != NULL)) { + mpcl->handler = nxt_job_destroy; + mpcl->data = job; + return NXT_OK; + } + + return NXT_ERROR; +} + + +/* + * The (void *) casts in nxt_thread_pool_post() and nxt_event_engine_post() + * calls and to the "nxt_work_handler_t" are required by Sun C. + */ + +void +nxt_job_start(nxt_thread_t *thr, nxt_job_t *job, nxt_work_handler_t handler) +{ + nxt_log_debug(thr->log, "%s start", job->name); + +#if (NXT_THREADS) + + if (job->thread_pool != NULL) { + nxt_int_t ret; + + job->engine = thr->engine; + ret = nxt_thread_pool_post(job->thread_pool, nxt_job_thread_trampoline, + job, (void *) handler, job->log); + if (ret == NXT_OK) { + return; + } + + handler = job->abort_handler; + } + +#endif + + handler(thr, job, job->data); +} + + +#if (NXT_THREADS) + +/* A trampoline function is called by a thread pool thread. */ + +static void +nxt_job_thread_trampoline(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_job_t *job; + nxt_work_handler_t handler; + + job = obj; + handler = (nxt_work_handler_t) data; + + nxt_log_debug(thr->log, "%s thread", job->name); + + if (nxt_slow_path(job->cancel)) { + nxt_job_return(thr, job, job->abort_handler); + + } else { + handler(thr, job, job->data); + } +} + +#endif + + +void +nxt_job_return(nxt_thread_t *thr, nxt_job_t *job, nxt_work_handler_t handler) +{ + nxt_log_debug(thr->log, "%s return", job->name); + +#if (NXT_THREADS) + + if (job->engine != NULL) { + /* A return function is called in thread pool thread context. */ + nxt_event_engine_post(job->engine, nxt_job_thread_return_handler, + job, (void *) handler, job->log); + return; + } + +#endif + + if (nxt_slow_path(job->cancel)) { + nxt_log_debug(thr->log, "%s cancellation", job->name); + handler = job->abort_handler; + } + + nxt_thread_work_queue_push(thr, &thr->work_queue.main, + handler, job, job->data, thr->log); +} + + +#if (NXT_THREADS) + +static void +nxt_job_thread_return_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_job_t *job; + nxt_work_handler_t handler; + + job = obj; + handler = (nxt_work_handler_t) data; + + if (nxt_slow_path(job->cancel)) { + nxt_log_debug(thr->log, "%s cancellation", job->name); + handler = job->abort_handler; + } + + handler(thr, job, job->data); +} + +#endif diff --git a/src/nxt_job.h b/src/nxt_job.h new file mode 100644 index 00000000..320716ca --- /dev/null +++ b/src/nxt_job.h @@ -0,0 +1,87 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_JOB_H_INCLUDED_ +#define _NXT_JOB_H_INCLUDED_ + + +/* + * A job may run by separate thread, so each job should have its + * its own mem_pool. A job itself is allocated from this mem_pool. + * On job completion a job initiator can destroy the job at once + * with nxt_job_destroy() or can postpone the destruction with + * nxt_job_cleanup_add(), if the initiator uses data from the job's + * mem_pool. + * + * Several child jobs may run in context of another job in the same + * thread. In this case the child job may use a mem_pool of the + * parent job and the child job is allocated using the mem_pool's cache. + * nxt_job_destroy() just returns the job to the cache. All job + * allocations however still remain in the parent mem_pool. + * + * The first thread in job thread pool is created on demand. If this + * operation fails the job abort handler is called. It also is called + * if the job is canceled. To avoid race condition the abort handler + * always runs in context of a thread initiated the job. The abort + * handler may be as simple as nxt_job_destroy(). + */ + + +typedef struct { + void *data; + + nxt_work_handler_t abort_handler; + + uint16_t cache_size; + uint8_t cancel; /* 1 bit */ + + nxt_mem_pool_t *mem_pool; + nxt_queue_link_t link; + +#if (NXT_THREADS) + nxt_thread_pool_t *thread_pool; + nxt_event_engine_t *engine; + nxt_log_t *log; +#endif + +#if (NXT_DEBUG) + const char *name; +#endif + +} nxt_job_t; + + +NXT_EXPORT void *nxt_job_create(nxt_mem_pool_t *mp, size_t size); +NXT_EXPORT void nxt_job_init(nxt_job_t *job, size_t size); +NXT_EXPORT void nxt_job_destroy(void *data); +NXT_EXPORT nxt_int_t nxt_job_cleanup_add(nxt_mem_pool_t *mp, nxt_job_t *job); + +NXT_EXPORT void nxt_job_start(nxt_thread_t *thr, nxt_job_t *job, + nxt_work_handler_t handler); +NXT_EXPORT void nxt_job_return(nxt_thread_t *thr, nxt_job_t *job, + nxt_work_handler_t handler); + + +#define \ +nxt_job_cancel(job) \ + (job)->cancel = 1 + + +#if (NXT_DEBUG) + +#define \ +nxt_job_set_name(job, text) \ + (job)->name = text + +#else + +#define \ +nxt_job_set_name(job, text) + +#endif + + +#endif /* _NXT_JOB_H_INCLUDED_ */ diff --git a/src/nxt_job_cache_file.c b/src/nxt_job_cache_file.c new file mode 100644 index 00000000..786691f6 --- /dev/null +++ b/src/nxt_job_cache_file.c @@ -0,0 +1,24 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#include <nxt_main.h> + + +nxt_job_cache_file_t * +nxt_job_cache_file_create(nxt_mem_pool_t *mp) +{ + nxt_job_cache_file_t *jbc; + + jbc = nxt_job_create(mp, sizeof(nxt_job_cache_file_t)); + + if (nxt_fast_path(jbc != NULL)) { + jbc->file.fd = NXT_FILE_INVALID; + jbc->read_required = nxt_job_file_read_required; + } + + return jbc; +} diff --git a/src/nxt_job_file.c b/src/nxt_job_file.c new file mode 100644 index 00000000..bbebdee6 --- /dev/null +++ b/src/nxt_job_file.c @@ -0,0 +1,303 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#include <nxt_main.h> + + +static void nxt_job_file_open_and_read(nxt_thread_t *thr, void *obj, + void *data); +static nxt_int_t nxt_job_file_open(nxt_job_file_t *jbf); +static nxt_int_t nxt_job_file_info(nxt_job_file_t *jbf); +static nxt_int_t nxt_job_file_mmap(nxt_job_file_t *jbf, size_t size); +static nxt_int_t nxt_job_file_read_data(nxt_job_file_t *jbf, size_t size); +static nxt_int_t nxt_job_file_read_required(nxt_job_file_t *jbf); + + +nxt_job_file_t * +nxt_job_file_create(nxt_mem_pool_t *mp) +{ + nxt_job_file_t *jbf; + + jbf = nxt_job_create(mp, sizeof(nxt_job_file_t)); + + if (nxt_fast_path(jbf != NULL)) { + jbf->file.fd = NXT_FILE_INVALID; + jbf->file.accessed = NXT_FILE_ACCESSED_LONG_AGO; + jbf->read_required = nxt_job_file_read_required; + } + + return jbf; +} + + +void +nxt_job_file_init(nxt_job_file_t *jbf) +{ + nxt_job_init(&jbf->job, sizeof(nxt_job_file_t)); + + jbf->file.fd = NXT_FILE_INVALID; + jbf->file.accessed = NXT_FILE_ACCESSED_LONG_AGO; + jbf->read_required = nxt_job_file_read_required; +} + + +/* + * Must be a function but not a macro, because + * it can be used as function pointer. + */ + +void +nxt_job_file_read(nxt_thread_t *thr, nxt_job_t *job) +{ + nxt_job_start(thr, job, nxt_job_file_open_and_read); +} + + +static void +nxt_job_file_open_and_read(nxt_thread_t *thr, void *obj, void *data) +{ + size_t size; + nxt_int_t n; + nxt_bool_t read_ahead; + nxt_file_t *file; + nxt_job_file_t *jbf; + nxt_work_handler_t handler; + + jbf = obj; + file = &jbf->file; + + nxt_log_debug(thr->log, "file job read: \"%FN\"", file->name); + + if (file->fd != NXT_FILE_INVALID && jbf->close_before_open) { + nxt_file_close(file); + file->fd = NXT_FILE_INVALID; + } + + if (file->fd == NXT_FILE_INVALID) { + + switch (nxt_job_file_open(jbf)) { + + case NXT_OK: + break; + + case NXT_DECLINED: + handler = jbf->ready_handler; + goto done; + + default: /* NXT_ERROR */ + handler = jbf->error_handler; + goto done; + } + } + + if (file->size > 0) { + + if (jbf->buffer != NULL) { + size = nxt_buf_mem_size(&jbf->buffer->mem); + size = nxt_min(file->size, (nxt_off_t) size); + read_ahead = nxt_buf_is_mmap(jbf->buffer); + + } else { + size = nxt_min(file->size, 1024 * 1024); + read_ahead = jbf->read_ahead; + } + + if (read_ahead) { + nxt_file_read_ahead(&jbf->file, jbf->offset, size); + } + + if (jbf->buffer != NULL) { + + if (nxt_buf_is_mmap(jbf->buffer)) { + n = nxt_job_file_mmap(jbf, size); + + } else { + n = nxt_job_file_read_data(jbf, size); + } + + if (nxt_slow_path(n != NXT_OK)) { + handler = jbf->error_handler; + goto done; + } + } + } + + if (jbf->offset == file->size) { + jbf->complete = 1; + + if (jbf->close) { + nxt_file_close(file); + file->fd = NXT_FILE_INVALID; + } + } + + nxt_job_return(thr, &jbf->job, jbf->ready_handler); + return; + +done: + + if (file->fd != NXT_FILE_INVALID) { + nxt_file_close(file); + file->fd = NXT_FILE_INVALID; + } + + nxt_job_return(thr, &jbf->job, handler); +} + + +static nxt_int_t +nxt_job_file_open(nxt_job_file_t *jbf) +{ + nxt_int_t n; + + if (jbf->test_before_open) { + n = nxt_job_file_info(jbf); + + if (n != NXT_OK) { + goto test_directory; + } + + if (jbf->file.type == NXT_FILE_DIRECTORY) { + return NXT_DECLINED; + } + + if (jbf->read_required(jbf) != NXT_OK) { + return NXT_DECLINED; + } + } + + n = nxt_file_open(&jbf->file, NXT_FILE_RDONLY, NXT_FILE_OPEN, 0); + + if (n == NXT_OK) { + n = nxt_job_file_info(jbf); + + if (nxt_fast_path(n == NXT_OK)) { + + if (jbf->file.type == NXT_FILE_DIRECTORY) { + return NXT_DECLINED; + } + + return jbf->read_required(jbf); + } + + return n; + } + +test_directory: + + if (jbf->directory_end != 0 + && jbf->file.error != NXT_ENOTDIR + && jbf->file.error != NXT_ENAMETOOLONG + && jbf->file.error != NXT_EACCES) + { + jbf->file.name[jbf->directory_end] = '\0'; + + return nxt_job_file_info(jbf); + } + + return n; +} + + +static nxt_int_t +nxt_job_file_info(nxt_job_file_t *jbf) +{ + nxt_int_t n; + nxt_file_t *file; + nxt_file_info_t fi; + + file = &jbf->file; + + n = nxt_file_info(file, &fi); + + if (n != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_is_file(&fi)) { + file->type = NXT_FILE_REGULAR; + file->size = nxt_file_size(&fi); + file->mtime = nxt_file_mtime(&fi); + + } else if (nxt_is_dir(&fi)) { + file->type = NXT_FILE_DIRECTORY; + file->size = nxt_file_size(&fi); + file->mtime = nxt_file_mtime(&fi); + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_job_file_mmap(nxt_job_file_t *jbf, size_t size) +{ + u_char *p, *end; + static nxt_uint_t n; + + p = nxt_mem_map(NULL, &jbf->buffer->mmap, size, NXT_MEM_MAP_READ, + (NXT_MEM_MAP_FILE | NXT_MEM_MAP_PREFAULT), + jbf->file.fd, jbf->offset); + + if (nxt_fast_path(p != NXT_MEM_MAP_FAILED)) { + + end = p + size; + + jbf->buffer->mem.pos = p; + jbf->buffer->mem.free = end; + jbf->buffer->mem.start = p; + jbf->buffer->mem.end = end; + jbf->buffer->file_end += size; + jbf->offset += size; + + /* + * The mapped pages should be already preloaded in the kernel page + * cache by nxt_file_read_ahead(). Touching them should wire the pages + * in user land memory if mmap() did not do this. Adding to the static + * variable "n" disables the loop elimination during optimization. + */ + n += *p; + + for (p = nxt_align_ptr(p, nxt_pagesize); p < end; p += nxt_pagesize) { + n += *p; + } + + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_job_file_read_data(nxt_job_file_t *jbf, size_t size) +{ + ssize_t n; + + n = nxt_file_read(&jbf->file, jbf->buffer->mem.pos, size, jbf->offset); + + if (nxt_fast_path(n > 0)) { + + jbf->buffer->mem.free += n; + jbf->offset += n; + + if (nxt_buf_is_file(jbf->buffer)) { + jbf->buffer->file_end += n; + } + + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_job_file_read_required(nxt_job_file_t *jbf) +{ + return NXT_OK; +} diff --git a/src/nxt_job_file.h b/src/nxt_job_file.h new file mode 100644 index 00000000..f61d517d --- /dev/null +++ b/src/nxt_job_file.h @@ -0,0 +1,74 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_JOB_FILE_H_INCLUDED_ +#define _NXT_JOB_FILE_H_INCLUDED_ + + +/* + * nxt_job_file_read() allows to open a file, to get its type, size, and + * modification time, to read or map file content to memory, and to close + * the file. It can be done as one operation for small file or as several + * operations for large file. On each operation completion ready_handler + * or error_handler completion handlers are called. Since they are job + * operations, they can be run by a thread pool. + * + * If a file is not opened then it is opened and its type, size, and + * modification time are got. Then file content starting from given offset + * is read or mapped in memory if there is a buffer supplied. The offset + * field is correspondingly updated. + * + * If there is no buffer but the read_ahead flag is set then the first + * byte is read to initiate read ahead operation. + * + * If the close flag is set then file descriptor is closed when the file + * is completely read. + * + * The complete flag is set by nxt_job_file_read() when the file is + * completely read. + * + * The test_before_open flag allows to save syscalls in some case, for + * example, not to open and then not to close a directory. It calls + * nxt_file_info() to get file type, size, and modification time before + * opening the file. A custom read_required() callback combined with this + * flag can also omit opening and reading on some conditions. However, + * if the callback forces opening then additional nxt_file_info() is + * called after opening. The default read_required() callback always + * forces opening and reading. + */ + + +typedef struct nxt_job_file_s nxt_job_file_t; + +struct nxt_job_file_s { + nxt_job_t job; + + nxt_file_t file; + + nxt_off_t offset; + nxt_buf_t *buffer; + + nxt_work_handler_t ready_handler; + nxt_work_handler_t error_handler; + + nxt_int_t (*read_required)(nxt_job_file_t *jbf); + + uint16_t directory_end; + + uint16_t close_before_open:1; + uint16_t test_before_open:1; + uint16_t read_ahead:1; + uint16_t close:1; + uint16_t complete:1; +}; + + +NXT_EXPORT nxt_job_file_t *nxt_job_file_create(nxt_mem_pool_t *mp); +NXT_EXPORT void nxt_job_file_init(nxt_job_file_t *jbf); +NXT_EXPORT void nxt_job_file_read(nxt_thread_t *thr, nxt_job_t *job); + + +#endif /* _NXT_JOB_FILE_H_INCLUDED_ */ diff --git a/src/nxt_job_file_cache.c b/src/nxt_job_file_cache.c new file mode 100644 index 00000000..47468dc2 --- /dev/null +++ b/src/nxt_job_file_cache.c @@ -0,0 +1,47 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#include <nxt_main.h> + + +typedef struct { + nxt_cache_node_t node; + nxt_file_t file; +} nxt_file_cache_t; + + +void +nxt_job_file_cache_read(nxt_cache_t *cache, nxt_job_file_t *jbf) +{ + nxt_file_cache_node_t *node; + + node = nxt_cache_find(cache); + + if (node != NULL) { + + if (node->fd != -1) { + nxt_job_return(&jbf->job, jbf->ready_handler); + return; + } + + if (node->error != 0) { + nxt_job_return(&jbf->job, jbf->error_handler); + return; + } + +#if (NXT_THREADS) + + if (node->accessed + 60 > nxt_thread_time()) { + jbf->job.thread_pool = NULL; + } + +#endif + + } + + nxt_job_file_read(jbf); +} diff --git a/src/nxt_job_resolve.c b/src/nxt_job_resolve.c new file mode 100644 index 00000000..600f1aae --- /dev/null +++ b/src/nxt_job_resolve.c @@ -0,0 +1,125 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +void +nxt_job_resolve(nxt_job_resolve_t *jbr) +{ + int err; + u_char *host; + size_t len; + nxt_uint_t n; + nxt_mem_pool_t *mp; + nxt_sockaddr_t *sa; + struct addrinfo hint, *res, *r; + nxt_work_handler_t handler; + + #define NXT_BUFSIZE 64 + u_char buf[NXT_BUFSIZE]; + + handler = jbr->error_handler; + res = NULL; + + len = jbr->name.len + 1; + + if (nxt_fast_path(len <= NXT_BUFSIZE)) { + host = buf; + + } else { + host = nxt_mem_alloc(jbr->job.mem_pool, len); + if (nxt_slow_path(host == NULL)) { + goto fail; + } + } + + nxt_cpystrn(host, jbr->name.data, len); + + nxt_memzero(&hint, sizeof(struct addrinfo)); + hint.ai_socktype = SOCK_STREAM; + + err = getaddrinfo((char *) host, NULL, &hint, &res); + + if (err != 0) { + nxt_thread_log_error(jbr->log_level, + "getaddrinfo(\"%s\") failed (%d: %s)", + host, err, gai_strerror(err)); + goto fail; + } + + n = 0; + for (r = res; r != NULL; r = r->ai_next) { + + switch (r->ai_addr->sa_family) { +#if (NXT_INET6) + case AF_INET6: +#endif + case AF_INET: + n++; + break; + + default: + break; + } + } + + jbr->count = n; + mp = jbr->job.mem_pool; + + jbr->sockaddrs = nxt_mem_alloc(mp, n * sizeof(nxt_sockaddr_t *)); + if (nxt_slow_path(jbr->sockaddrs == NULL)) { + goto fail; + } + + n = 0; + for (r = res; r != NULL; r = r->ai_next) { + + switch (r->ai_addr->sa_family) { +#if (NXT_INET6) + case AF_INET6: +#endif + case AF_INET: + break; + + default: + continue; + } + + sa = nxt_sockaddr_create(mp, r->ai_addr, r->ai_addrlen); + if (nxt_slow_path(sa == NULL)) { + goto fail; + } + + jbr->sockaddrs[n++] = sa; + + if (jbr->port != 0) { + + switch (sa->u.sockaddr.sa_family) { + case AF_INET: + sa->u.sockaddr_in.sin_port = jbr->port; + break; +#if (NXT_INET6) + case AF_INET6: + sa->u.sockaddr_in6.sin6_port = jbr->port; + break; +#endif + default: + break; + } + } + } + + handler = jbr->ready_handler; + +fail: + + if (nxt_fast_path(res != NULL)) { + freeaddrinfo(res); + } + + nxt_job_return(nxt_thread(), &jbr->job, handler); +} diff --git a/src/nxt_job_resolve.h b/src/nxt_job_resolve.h new file mode 100644 index 00000000..fef55b99 --- /dev/null +++ b/src/nxt_job_resolve.h @@ -0,0 +1,29 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_JOB_RESOLVE_H_INCLUDED_ +#define _NXT_JOB_RESOLVE_H_INCLUDED_ + + +typedef struct { + nxt_job_t job; + nxt_str_t name; + + uint32_t log_level; /* 4 bits */ + in_port_t port; + uint16_t count; + + nxt_sockaddr_t **sockaddrs; + + nxt_work_handler_t ready_handler; + nxt_work_handler_t error_handler; +} nxt_job_resolve_t; + + +void nxt_job_resolve(nxt_job_resolve_t *jbr); + + +#endif /* _NXT_JOB_RESOLVE_H_INCLUDED_ */ diff --git a/src/nxt_kqueue.c b/src/nxt_kqueue.c new file mode 100644 index 00000000..25d9eefe --- /dev/null +++ b/src/nxt_kqueue.c @@ -0,0 +1,1063 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * kqueue() has been introduced in FreeBSD 4.1 and then was ported + * to OpenBSD 2.9, MacOSX 10.3 (Panther), and NetBSD 2.0. + * DragonFlyBSD inherited it with FreeBSD 4 code base. + * + * NOTE_REVOKE has been introduced in FreeBSD 4.3 and then was ported + * to OpenBSD 2.9, MacOSX 10.3 (Panther), and NetBSD 2.0. + * DragonFlyBSD inherited it with FreeBSD 4 code base. + * + * EVFILT_TIMER has been introduced in FreeBSD 4.4-STABLE and then was + * ported to NetBSD 2.0, MacOSX 10.4 (Tiger), and OpenBSD 4.2. + * DragonFlyBSD inherited it with FreeBSD 4 code base. + * + * EVFILT_USER and EV_DISPATCH have been introduced in MacOSX 10.6 (Snow + * Leopard) as part of the Grand Central Dispatch framework + * and then were ported to FreeBSD 8.0-STABLE as part of the + * libdispatch support. + */ + + +/* + * EV_DISPATCH is better because it just disables an event on delivery + * whilst EV_ONESHOT deletes the event. This eliminates in-kernel memory + * deallocation and probable subsequent allocation with a lock acquiring. + */ +#ifdef EV_DISPATCH +#define NXT_KEVENT_ONESHOT EV_DISPATCH +#else +#define NXT_KEVENT_ONESHOT EV_ONESHOT +#endif + + +#if (NXT_NETBSD) +/* NetBSD defines the kevent.udata field as intptr_t. */ + +#define nxt_kevent_set_udata(udata) (intptr_t) (udata) +#define nxt_kevent_get_udata(udata) (void *) (udata) + +#else +#define nxt_kevent_set_udata(udata) (void *) (udata) +#define nxt_kevent_get_udata(udata) (udata) +#endif + + +static nxt_event_set_t *nxt_kqueue_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +static void nxt_kqueue_free(nxt_event_set_t *event_set); +static void nxt_kqueue_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_kqueue_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_kqueue_delete(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_kqueue_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_kqueue_drop_changes(nxt_event_set_t *event_set, + uintptr_t ident); +static void nxt_kqueue_enable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_enable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_disable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_disable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_block_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_block_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_oneshot_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_oneshot_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_enable_accept(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_kqueue_enable_file(nxt_event_set_t *event_set, + nxt_event_file_t *ev); +static void nxt_kqueue_close_file(nxt_event_set_t *event_set, + nxt_event_file_t *ev); +static void nxt_kqueue_fd_set(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_int_t filter, nxt_uint_t flags); +static struct kevent *nxt_kqueue_get_kevent(nxt_kqueue_event_set_t *ks); +static void nxt_kqueue_commit_changes(nxt_kqueue_event_set_t *ks); +static void nxt_kqueue_error(nxt_kqueue_event_set_t *ks); +static void nxt_kqueue_fd_error_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_kqueue_file_error_handler(nxt_thread_t *thr, void *obj, + void *data); +static nxt_int_t nxt_kqueue_add_signal(nxt_kqueue_event_set_t *kq, + const nxt_event_sig_t *sigev); +#if (NXT_HAVE_EVFILT_USER) +static nxt_int_t nxt_kqueue_enable_post(nxt_event_set_t *event_set, + nxt_work_handler_t handler); +static void nxt_kqueue_signal(nxt_event_set_t *event_set, nxt_uint_t signo); +#endif +static void nxt_kqueue_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout); + +static void nxt_kqueue_event_conn_io_connect(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_kqueue_event_conn_connected(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_kqueue_listen_handler(nxt_thread_t *thr, void *obj, void *data); +static void nxt_kqueue_event_conn_io_accept(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_kqueue_event_conn_io_read(nxt_thread_t *thr, void *obj, + void *data); +static ssize_t nxt_kqueue_event_conn_io_recvbuf(nxt_event_conn_t *c, + nxt_buf_t *b); + + +static nxt_event_conn_io_t nxt_kqueue_event_conn_io = { + nxt_kqueue_event_conn_io_connect, + nxt_kqueue_event_conn_io_accept, + + nxt_kqueue_event_conn_io_read, + nxt_kqueue_event_conn_io_recvbuf, + nxt_event_conn_io_recv, + + nxt_event_conn_io_write, + nxt_event_conn_io_write_chunk, + +#if (NXT_HAVE_FREEBSD_SENDFILE) + nxt_freebsd_event_conn_io_sendfile, +#elif (NXT_HAVE_MACOSX_SENDFILE) + nxt_macosx_event_conn_io_sendfile, +#else + nxt_event_conn_io_sendbuf, +#endif + + nxt_event_conn_io_writev, + nxt_event_conn_io_send, + + nxt_event_conn_io_shutdown, +}; + + +const nxt_event_set_ops_t nxt_kqueue_event_set = { + "kqueue", + nxt_kqueue_create, + nxt_kqueue_free, + nxt_kqueue_enable, + nxt_kqueue_disable, + nxt_kqueue_delete, + nxt_kqueue_close, + nxt_kqueue_enable_read, + nxt_kqueue_enable_write, + nxt_kqueue_disable_read, + nxt_kqueue_disable_write, + nxt_kqueue_block_read, + nxt_kqueue_block_write, + nxt_kqueue_oneshot_read, + nxt_kqueue_oneshot_write, + nxt_kqueue_enable_accept, + nxt_kqueue_enable_file, + nxt_kqueue_close_file, +#if (NXT_HAVE_EVFILT_USER) + nxt_kqueue_enable_post, + nxt_kqueue_signal, +#else + NULL, + NULL, +#endif + nxt_kqueue_poll, + + &nxt_kqueue_event_conn_io, + + NXT_FILE_EVENTS, + NXT_SIGNAL_EVENTS, +}; + + +static nxt_event_set_t * +nxt_kqueue_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + nxt_event_set_t *event_set; + const nxt_event_sig_t *sigev; + nxt_kqueue_event_set_t *ks; + + event_set = nxt_zalloc(sizeof(nxt_kqueue_event_set_t)); + if (event_set == NULL) { + return NULL; + } + + ks = &event_set->kqueue; + + ks->kqueue = -1; + ks->mchanges = mchanges; + ks->mevents = mevents; + ks->pid = nxt_pid; + + ks->changes = nxt_malloc(sizeof(struct kevent) * mchanges); + if (ks->changes == NULL) { + goto fail; + } + + ks->events = nxt_malloc(sizeof(struct kevent) * mevents); + if (ks->events == NULL) { + goto fail; + } + + ks->kqueue = kqueue(); + if (ks->kqueue == -1) { + nxt_main_log_emerg("kqueue() failed %E", nxt_errno); + goto fail; + } + + nxt_main_log_debug("kqueue(): %d", ks->kqueue); + + if (signals != NULL) { + for (sigev = signals->sigev; sigev->signo != 0; sigev++) { + if (nxt_kqueue_add_signal(ks, sigev) != NXT_OK) { + goto fail; + } + } + } + + return event_set; + +fail: + + nxt_kqueue_free(event_set); + + return NULL; +} + + +static void +nxt_kqueue_free(nxt_event_set_t *event_set) +{ + nxt_kqueue_event_set_t *ks; + + ks = &event_set->kqueue; + + nxt_main_log_debug("kqueue %d free", ks->kqueue); + + if (ks->kqueue != -1 && ks->pid == nxt_pid) { + /* kqueue is not inherited by fork() */ + + if (close(ks->kqueue) != 0) { + nxt_main_log_emerg("kqueue close(%d) failed %E", + ks->kqueue, nxt_errno); + } + } + + nxt_free(ks->events); + nxt_free(ks->changes); + nxt_free(ks); +} + + +static void +nxt_kqueue_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_kqueue_enable_read(event_set, ev); + nxt_kqueue_enable_write(event_set, ev); +} + + +/* + * EV_DISABLE is better because it eliminates in-kernel memory + * deallocation and probable subsequent allocation with a lock acquiring. + */ + +static void +nxt_kqueue_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_INACTIVE; + nxt_kqueue_fd_set(event_set, ev, EVFILT_READ, EV_DISABLE); + } + + if (ev->write != NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_INACTIVE; + nxt_kqueue_fd_set(event_set, ev, EVFILT_WRITE, EV_DISABLE); + } +} + + +static void +nxt_kqueue_delete(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_INACTIVE; + nxt_kqueue_fd_set(event_set, ev, EVFILT_READ, EV_DELETE); + } + + if (ev->write != NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_INACTIVE; + nxt_kqueue_fd_set(event_set, ev, EVFILT_WRITE, EV_DELETE); + } +} + + +/* + * kqueue(2): + * + * Calling close() on a file descriptor will remove any kevents that + * reference the descriptor. + */ + +static void +nxt_kqueue_close(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_kqueue_drop_changes(event_set, ev->fd); +} + + +static void +nxt_kqueue_drop_changes(nxt_event_set_t *event_set, uintptr_t ident) +{ + struct kevent *dst, *src, *end; + nxt_kqueue_event_set_t *ks; + + ks = &event_set->kqueue; + + dst = ks->changes; + end = dst + ks->nchanges; + + for (src = dst; src < end; src++) { + + if (src->ident == ident) { + + switch (src->filter) { + + case EVFILT_READ: + case EVFILT_WRITE: + case EVFILT_VNODE: + continue; + } + } + + if (dst != src) { + *dst = *src; + } + + dst++; + } + + ks->nchanges -= end - dst; +} + + +/* + * The kqueue event set uses only three states: inactive, blocked, and + * default. An active oneshot event is marked as it is in the default + * state. The event will eventually be converted to the default EV_CLEAR + * mode after it will become inactive after delivery. + */ + +static void +nxt_kqueue_enable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read == NXT_EVENT_INACTIVE) { + nxt_kqueue_fd_set(event_set, ev, EVFILT_READ, + EV_ADD | EV_ENABLE | EV_CLEAR); + } + + ev->read = NXT_EVENT_DEFAULT; +} + + +static void +nxt_kqueue_enable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write == NXT_EVENT_INACTIVE) { + nxt_kqueue_fd_set(event_set, ev, EVFILT_WRITE, + EV_ADD | EV_ENABLE | EV_CLEAR); + } + + ev->write = NXT_EVENT_DEFAULT; +} + + +static void +nxt_kqueue_disable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + + nxt_kqueue_fd_set(event_set, ev, EVFILT_READ, EV_DISABLE); +} + + +static void +nxt_kqueue_disable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->write = NXT_EVENT_INACTIVE; + + nxt_kqueue_fd_set(event_set, ev, EVFILT_WRITE, EV_DISABLE); +} + + +static void +nxt_kqueue_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_kqueue_block_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write != NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_kqueue_oneshot_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->write = NXT_EVENT_DEFAULT; + + nxt_kqueue_fd_set(event_set, ev, EVFILT_WRITE, + EV_ADD | EV_ENABLE | NXT_KEVENT_ONESHOT); +} + + +static void +nxt_kqueue_oneshot_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->write = NXT_EVENT_DEFAULT; + + nxt_kqueue_fd_set(event_set, ev, EVFILT_WRITE, + EV_ADD | EV_ENABLE | NXT_KEVENT_ONESHOT); +} + + +static void +nxt_kqueue_enable_accept(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_DEFAULT; + ev->read_handler = nxt_kqueue_listen_handler; + + nxt_kqueue_fd_set(event_set, ev, EVFILT_READ, EV_ADD | EV_ENABLE); +} + + +static void +nxt_kqueue_enable_file(nxt_event_set_t *event_set, nxt_event_file_t *ev) +{ + struct kevent *kev; + nxt_kqueue_event_set_t *ks; + + ks = &event_set->kqueue; + + kev = nxt_kqueue_get_kevent(ks); + + kev->ident = ev->file->fd; + kev->filter = EVFILT_VNODE; + kev->flags = EV_ADD | EV_ENABLE | EV_ONESHOT; + kev->fflags = NOTE_DELETE | NOTE_WRITE | NOTE_EXTEND + | NOTE_ATTRIB | NOTE_RENAME | NOTE_REVOKE; + kev->data = 0; + kev->udata = nxt_kevent_set_udata(ev); + + nxt_thread_log_debug("kevent(%d) set: id:%d ft:%i fl:%04Xd, ff:%04XuD", + ks->kqueue, ev->file->fd, EVFILT_VNODE, + kev->flags, kev->fflags); +} + + +static void +nxt_kqueue_close_file(nxt_event_set_t *event_set, nxt_event_file_t *ev) +{ + nxt_kqueue_drop_changes(event_set, ev->file->fd); +} + + +static void +nxt_kqueue_fd_set(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_int_t filter, nxt_uint_t flags) +{ + struct kevent *kev; + nxt_kqueue_event_set_t *ks; + + ks = &event_set->kqueue; + + nxt_log_debug(ev->log, "kevent(%d) set event: id:%d ft:%i fl:%04Xui", + ks->kqueue, ev->fd, filter, flags); + + kev = nxt_kqueue_get_kevent(ks); + + kev->ident = ev->fd; + kev->filter = filter; + kev->flags = flags; + kev->fflags = 0; + kev->data = 0; + kev->udata = nxt_kevent_set_udata(ev); +} + + +static struct kevent * +nxt_kqueue_get_kevent(nxt_kqueue_event_set_t *ks) +{ + if (nxt_slow_path(ks->nchanges >= ks->mchanges)) { + nxt_kqueue_commit_changes(ks); + } + + return &ks->changes[ks->nchanges++]; +} + + +static void +nxt_kqueue_commit_changes(nxt_kqueue_event_set_t *ks) +{ + nxt_thread_log_debug("kevent(%d) changes:%d", ks->kqueue, ks->nchanges); + + if (kevent(ks->kqueue, ks->changes, ks->nchanges, NULL, 0, NULL) != 0) { + nxt_thread_log_alert("kevent(%d) failed %E", ks->kqueue, nxt_errno); + + nxt_kqueue_error(ks); + } + + ks->nchanges = 0; +} + + +static void +nxt_kqueue_error(nxt_kqueue_event_set_t *ks) +{ + struct kevent *kev, *end; + nxt_thread_t *thr; + nxt_event_fd_t *ev; + nxt_event_file_t *fev; + + thr = nxt_thread(); + end = &ks->changes[ks->nchanges]; + + for (kev = ks->changes; kev < end; kev++) { + + switch (kev->filter) { + + case EVFILT_READ: + case EVFILT_WRITE: + ev = nxt_kevent_get_udata(kev->udata); + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_kqueue_fd_error_handler, + ev, ev->data, ev->log); + break; + + case EVFILT_VNODE: + fev = nxt_kevent_get_udata(kev->udata); + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_kqueue_file_error_handler, + fev, fev->data, thr->log); + break; + } + } +} + + +static void +nxt_kqueue_fd_error_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_fd_t *ev; + + ev = obj; + + if (ev->kq_eof && ev->kq_errno != 0) { + ev->error = ev->kq_errno; + nxt_log_error(nxt_socket_error_level(ev->kq_errno, ev->log_error), + thr->log, "kevent() reported error on descriptor %d %E", + ev->fd, ev->kq_errno); + } + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + ev->error = ev->kq_errno; + + ev->error_handler(thr, ev, data); +} + + +static void +nxt_kqueue_file_error_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_file_t *ev; + + ev = obj; + + ev->handler(thr, ev, data); +} + + +static nxt_int_t +nxt_kqueue_add_signal(nxt_kqueue_event_set_t *ks, const nxt_event_sig_t *sigev) +{ + int signo; + struct kevent kev; + struct sigaction sa; + + signo = sigev->signo; + + nxt_memzero(&sa, sizeof(struct sigaction)); + sigemptyset(&sa.sa_mask); + + /* + * SIGCHLD must not be set to SIG_IGN, since kqueue cannot catch + * this signal. It should be set to SIG_DFL instead. And although + * SIGCHLD default action is also ignoring, nevertheless SIG_DFL + * allows kqueue to catch the signal. + */ + sa.sa_handler = (signo == SIGCHLD) ? SIG_DFL : SIG_IGN; + + if (sigaction(signo, &sa, NULL) != 0) { + nxt_main_log_alert("sigaction(%d) failed %E", signo, nxt_errno); + return NXT_ERROR; + } + + nxt_main_log_debug("kevent(%d) signo:%d (%s)", + ks->kqueue, signo, sigev->name); + + kev.ident = signo; + kev.filter = EVFILT_SIGNAL; + kev.flags = EV_ADD; + kev.fflags = 0; + kev.data = 0; + kev.udata = nxt_kevent_set_udata(sigev); + + if (kevent(ks->kqueue, &kev, 1, NULL, 0, NULL) == 0) { + return NXT_OK; + } + + nxt_main_log_alert("kevent(%d) failed %E", ks->kqueue, nxt_errno); + return NXT_ERROR; +} + + +#if (NXT_HAVE_EVFILT_USER) + +static nxt_int_t +nxt_kqueue_enable_post(nxt_event_set_t *event_set, nxt_work_handler_t handler) +{ + struct kevent kev; + nxt_kqueue_event_set_t *ks; + + /* EVFILT_USER must be added to a kqueue before it can be triggered. */ + + kev.ident = 0; + kev.filter = EVFILT_USER; + kev.flags = EV_ADD | EV_CLEAR; + kev.fflags = 0; + kev.data = 0; + kev.udata = NULL; + + ks = &event_set->kqueue; + ks->post_handler = handler; + + if (kevent(ks->kqueue, &kev, 1, NULL, 0, NULL) == 0) { + return NXT_OK; + } + + nxt_main_log_alert("kevent(%d) failed %E", ks->kqueue, nxt_errno); + return NXT_ERROR; +} + + +static void +nxt_kqueue_signal(nxt_event_set_t *event_set, nxt_uint_t signo) +{ + struct kevent kev; + nxt_kqueue_event_set_t *ks; + + /* + * kqueue has a builtin signal processing support, so the function + * is used only to post events and the signo argument is ignored. + */ + + kev.ident = 0; + kev.filter = EVFILT_USER; + kev.flags = 0; + kev.fflags = NOTE_TRIGGER; + kev.data = 0; + kev.udata = NULL; + + ks = &event_set->kqueue; + + if (kevent(ks->kqueue, &kev, 1, NULL, 0, NULL) != 0) { + nxt_thread_log_alert("kevent(%d) failed %E", ks->kqueue, nxt_errno); + } +} + +#endif + + +static void +nxt_kqueue_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout) +{ + int nevents; + void *obj, *data; + nxt_int_t i; + nxt_err_t err; + nxt_log_t *log; + nxt_uint_t level; + nxt_bool_t error, eof; + struct kevent *kev; + nxt_event_fd_t *ev; + nxt_event_sig_t *sigev; + struct timespec ts, *tp; + nxt_event_file_t *fev; + nxt_work_queue_t *wq; + nxt_work_handler_t handler; + nxt_kqueue_event_set_t *ks; + + if (timeout == NXT_INFINITE_MSEC) { + tp = NULL; + + } else { + ts.tv_sec = timeout / 1000; + ts.tv_nsec = (timeout % 1000) * 1000000; + tp = &ts; + } + + ks = &event_set->kqueue; + + nxt_log_debug(thr->log, "kevent(%d) changes:%d timeout:%M", + ks->kqueue, ks->nchanges, timeout); + + nevents = kevent(ks->kqueue, ks->changes, ks->nchanges, + ks->events, ks->mevents, tp); + + err = (nevents == -1) ? nxt_errno : 0; + + nxt_thread_time_update(thr); + + nxt_log_debug(thr->log, "kevent(%d): %d", ks->kqueue, nevents); + + if (nevents == -1) { + level = (err == NXT_EINTR) ? NXT_LOG_INFO : NXT_LOG_ALERT; + nxt_log_error(level, thr->log, "kevent(%d) failed %E", ks->kqueue, err); + + nxt_kqueue_error(ks); + return; + } + + ks->nchanges = 0; + + for (i = 0; i < nevents; i++) { + + kev = &ks->events[i]; + + nxt_log_debug(thr->log, + (kev->ident > 0x8000000 && kev->ident != (uintptr_t) -1) ? + "kevent: id:%p ft:%d fl:%04Xd ff:%d d:%d ud:%p": + "kevent: id:%d ft:%d fl:%04Xd ff:%d d:%d ud:%p", + kev->ident, kev->filter, kev->flags, kev->fflags, + kev->data, kev->udata); + + error = (kev->flags & EV_ERROR); + + if (nxt_slow_path(error)) { + nxt_log_alert(thr->log, "kevent(%d) error %E on ident:%d filter:%d", + ks->kqueue, kev->data, kev->ident, kev->filter); + } + + wq = &thr->work_queue.main; + handler = nxt_kqueue_fd_error_handler; + obj = nxt_kevent_get_udata(kev->udata); + log = thr->log; + + switch (kev->filter) { + + case EVFILT_READ: + ev = obj; + ev->read_ready = 1; + ev->kq_available = (int32_t) kev->data; + err = kev->fflags; + eof = (kev->flags & EV_EOF) != 0; + ev->kq_errno = err; + ev->kq_eof = eof; + + if (ev->read == NXT_EVENT_BLOCKED) { + nxt_log_debug(ev->log, "blocked read event fd:%d", ev->fd); + continue; + } + + if ((kev->flags & NXT_KEVENT_ONESHOT) != 0) { + ev->read = NXT_EVENT_INACTIVE; + } + + if (nxt_slow_path(ev->kq_available == 0 && eof && err != 0)) { + error = 1; + } + + if (nxt_fast_path(!error)) { + handler = ev->read_handler; + wq = ev->read_work_queue; + } + + data = ev->data; + log = ev->log; + + break; + + case EVFILT_WRITE: + ev = obj; + ev->write_ready = 1; + err = kev->fflags; + eof = (kev->flags & EV_EOF) != 0; + ev->kq_errno = err; + ev->kq_eof = eof; + + if (ev->write == NXT_EVENT_BLOCKED) { + nxt_log_debug(ev->log, "blocked write event fd:%d", ev->fd); + continue; + } + + if ((kev->flags & NXT_KEVENT_ONESHOT) != 0) { + ev->write = NXT_EVENT_INACTIVE; + } + + if (nxt_slow_path(eof && err != 0)) { + error = 1; + } + + if (nxt_fast_path(!error)) { + handler = ev->write_handler; + wq = ev->write_work_queue; + } + + data = ev->data; + log = ev->log; + + break; + + case EVFILT_VNODE: + fev = obj; + handler = fev->handler; + data = fev->data; + break; + + case EVFILT_SIGNAL: + sigev = obj; + obj = (void *) kev->ident; + handler = sigev->handler; + data = (void *) sigev->name; + break; + +#if (NXT_HAVE_EVFILT_USER) + + case EVFILT_USER: + handler = ks->post_handler; + data = NULL; + break; + +#endif + + default: + +#if (NXT_DEBUG) + nxt_log_alert(thr->log, + "unexpected kevent(%d) filter %d on ident %d", + ks->kqueue, kev->filter, kev->ident); +#endif + + continue; + } + + nxt_thread_work_queue_add(thr, wq, handler, obj, data, log); + } +} + + +/* + * nxt_kqueue_event_conn_io_connect() eliminates the + * getsockopt() syscall to test pending connect() error. + */ + +static void +nxt_kqueue_event_conn_io_connect(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_work_handler_t handler; + const nxt_event_conn_state_t *state; + + c = obj; + + state = c->write_state; + + switch (nxt_socket_connect(c->socket.fd, c->remote) ){ + + case NXT_OK: + c->socket.write_ready = 1; + handler = state->ready_handler; + break; + + case NXT_AGAIN: + c->socket.write_handler = nxt_kqueue_event_conn_connected; + c->socket.error_handler = nxt_event_conn_connect_error; + + nxt_event_conn_timer(thr->engine, c, state, &c->write_timer); + + nxt_kqueue_enable_write(thr->engine->event_set, &c->socket); + return; + + case NXT_DECLINED: + handler = state->close_handler; + break; + + default: /* NXT_ERROR */ + handler = state->error_handler; + break; + } + + nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); +} + + +static void +nxt_kqueue_event_conn_connected(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "kqueue event conn connected fd:%d", c->socket.fd); + + c->socket.write = NXT_EVENT_BLOCKED; + + if (c->write_state->autoreset_timer) { + nxt_event_timer_disable(&c->write_timer); + } + + nxt_thread_work_queue_add(thr, c->write_work_queue, + c->write_state->ready_handler, + c, data, c->socket.log); +} + + +static void +nxt_kqueue_listen_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_listen_t *cls; + + cls = obj; + + nxt_log_debug(thr->log, "kevent fd:%d avail:%D", + cls->socket.fd, cls->socket.kq_available); + + cls->ready = nxt_min(cls->batch, (uint32_t) cls->socket.kq_available); + + nxt_kqueue_event_conn_io_accept(thr, cls, data); +} + + +static void +nxt_kqueue_event_conn_io_accept(nxt_thread_t *thr, void *obj, void *data) +{ + socklen_t len; + nxt_socket_t s; + struct sockaddr *sa; + nxt_event_conn_t *c; + nxt_event_conn_listen_t *cls; + + cls = obj; + c = data; + + cls->ready--; + cls->socket.read_ready = (cls->ready != 0); + + cls->socket.kq_available--; + cls->socket.read_ready = (cls->socket.kq_available != 0); + + len = nxt_socklen(c->remote); + + if (len >= sizeof(struct sockaddr)) { + sa = &c->remote->u.sockaddr; + + } else { + sa = NULL; + len = 0; + } + + s = accept(cls->socket.fd, sa, &len); + + if (s != -1) { + c->socket.fd = s; + + nxt_log_debug(thr->log, "accept(%d): %d", cls->socket.fd, s); + + nxt_event_conn_accept(thr, cls, c); + return; + } + + nxt_event_conn_accept_error(thr, cls, "accept", nxt_errno); +} + + +/* + * nxt_kqueue_event_conn_io_read() is just a wrapper to eliminate the + * readv() or recv() syscall if a remote side just closed connection. + */ + +static void +nxt_kqueue_event_conn_io_read(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "kqueue event conn read fd:%d", c->socket.fd); + + if (c->socket.kq_available == 0 && c->socket.kq_eof) { + nxt_log_debug(thr->log, "kevent fd:%d eof", c->socket.fd); + + c->socket.closed = 1; + nxt_thread_work_queue_add(thr, c->read_work_queue, + c->read_state->close_handler, + c, data, c->socket.log); + return; + } + + nxt_event_conn_io_read(thr, c, data); +} + + +/* + * nxt_kqueue_event_conn_io_recvbuf() is just wrapper around standard + * nxt_event_conn_io_recvbuf() to eliminate the readv() or recv() syscalls + * if there is no pending data or a remote side closed connection. + */ + +static ssize_t +nxt_kqueue_event_conn_io_recvbuf(nxt_event_conn_t *c, nxt_buf_t *b) +{ + ssize_t n; + + if (c->socket.kq_available == 0 && c->socket.kq_eof) { + c->socket.closed = 1; + return 0; + } + + n = nxt_event_conn_io_recvbuf(c, b); + + if (n > 0) { + c->socket.kq_available -= n; + + if (c->socket.kq_available < 0) { + c->socket.kq_available = 0; + } + + nxt_log_debug(c->socket.log, "kevent fd:%d avail:%D eof:%d", + c->socket.fd, c->socket.kq_available, c->socket.kq_eof); + + c->socket.read_ready = (c->socket.kq_available != 0 + || c->socket.kq_eof); + } + + return n; +} diff --git a/src/nxt_lib.c b/src/nxt_lib.c new file mode 100644 index 00000000..2013a4f6 --- /dev/null +++ b/src/nxt_lib.c @@ -0,0 +1,148 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_uint_t nxt_ncpu = 1; +nxt_uint_t nxt_pagesize; +nxt_random_t nxt_random_data; + +nxt_thread_declare_data(nxt_thread_t, nxt_thread_context); + + +#if (NXT_DEBUG && NXT_FREEBSD) +/* + * Fill memory with 0xA5 after malloc() and with 0x5A before free(). + * malloc() options variable has to override the libc symbol, otherwise + * it has no effect. + */ +#if __FreeBSD_version < 1000011 +const char *_malloc_options = "J"; +#else +const char *malloc_conf = "junk:true"; +#endif +#endif + + +nxt_int_t +nxt_lib_start(const char *app, char **argv, char ***envp) +{ + int n; + nxt_int_t flags; + nxt_bool_t update; + nxt_thread_t *thr; + + flags = nxt_stderr_start(); + + nxt_log_start(app); + + nxt_pid = getpid(); + nxt_ppid = getppid(); + +#if (NXT_DEBUG) + + nxt_main_log.level = NXT_LOG_DEBUG; + +#if (NXT_LINUX) + /* Fill memory with 0xAA after malloc() and with 0x55 before free(). */ + mallopt(M_PERTURB, 0x55); +#endif + +#if (NXT_MACOSX) + /* Fill memory with 0xAA after malloc() and with 0x55 before free(). */ + setenv("MallocScribble", "1", 0); +#endif + +#endif /* NXT_DEBUG */ + + /* Thread log is required for nxt_malloc() in nxt_strerror_start(). */ + + nxt_thread_init_data(nxt_thread_context); + thr = nxt_thread(); + thr->log = &nxt_main_log; + +#if (NXT_THREADS) + thr->handle = nxt_thread_handle(); + thr->time.signal = -1; +#endif + + if (nxt_strerror_start() != NXT_OK) { + return NXT_ERROR; + } + + if (flags != -1) { + nxt_log_debug(thr->log, "stderr flags: 0x%04Xd", flags); + } + +#ifdef _SC_NPROCESSORS_ONLN + /* Linux, FreeBSD, Solaris, MacOSX. */ + n = sysconf(_SC_NPROCESSORS_ONLN); + +#elif (NXT_HPUX) + n = mpctl(MPC_GETNUMSPUS, NULL, NULL); + +#endif + + nxt_log_debug(thr->log, "ncpu: %ui", n); + + if (n > 1) { + nxt_ncpu = n; + } + + nxt_thread_spin_init(nxt_ncpu, 0); + + nxt_random_init(&nxt_random_data); + + nxt_pagesize = getpagesize(); + + nxt_log_debug(thr->log, "pagesize: %ui", nxt_pagesize); + + if (argv != NULL) { + update = (argv[0] == app); + + nxt_process_arguments(argv, envp); + + if (update) { + nxt_log_start(nxt_process_argv[0]); + } + } + + return NXT_OK; +} + + +void +nxt_lib_stop(void) +{ + /* TODO: stop engines */ + +#if (NXT_THREADS0) + + for ( ;; ) { + nxt_thread_pool_t *tp; + + nxt_thread_spin_lock(&cycle->lock); + + tp = cycle->thread_pools; + cycle->thread_pools = (tp != NULL) ? tp->next : NULL; + + nxt_thread_spin_unlock(&cycle->lock); + + if (tp == NULL) { + break; + } + + nxt_thread_pool_destroy(tp); + } + +#else + + exit(0); + nxt_unreachable(); + +#endif +} diff --git a/src/nxt_linux_sendfile.c b/src/nxt_linux_sendfile.c new file mode 100644 index 00000000..9c8f563c --- /dev/null +++ b/src/nxt_linux_sendfile.c @@ -0,0 +1,240 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * sendfile() has been introduced in Linux 2.2. + * It supported 32-bit offsets only. + * + * Linux 2.4.21 has introduced sendfile64(). However, even on 64-bit + * platforms it returns EINVAL if the count argument is more than 2G-1 bytes. + * In Linux 2.6.17 sendfile() has been internally changed to splice() + * and this limitation has gone. + */ + +#ifdef NXT_TEST_BUILD_LINUX_SENDFILE + +#define MSG_NOSIGNAL 0x4000 +#define MSG_MORE 0x8000 + +ssize_t nxt_linux_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + +static ssize_t nxt_sys_sendfile(int out_fd, int in_fd, off_t *offset, + size_t count) +{ + return -1; +} + +#else +#define nxt_sys_sendfile sendfile +#endif + + +static ssize_t nxt_linux_send(nxt_event_conn_t *c, void *buf, size_t size, + nxt_uint_t flags); +static ssize_t nxt_linux_sendmsg(nxt_event_conn_t *c, + nxt_sendbuf_coalesce_t *sb, nxt_uint_t niov, nxt_uint_t flags); + + +ssize_t +nxt_linux_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit) +{ + size_t size; + ssize_t n; + nxt_buf_t *fb; + nxt_err_t err; + nxt_off_t offset; + nxt_uint_t niov, flags; + struct iovec iov[NXT_IOBUF_MAX]; + nxt_sendbuf_coalesce_t sb; + + sb.buf = b; + sb.iobuf = iov; + sb.nmax = NXT_IOBUF_MAX; + sb.sync = 0; + sb.size = 0; + sb.limit = limit; + + niov = nxt_sendbuf_mem_coalesce(&sb); + + if (niov == 0 && sb.sync) { + return 0; + } + + fb = (sb.buf != NULL && nxt_buf_is_file(sb.buf)) ? sb.buf : NULL; + + if (niov != 0) { + + flags = MSG_NOSIGNAL; + + if (fb != NULL) { + /* + * The Linux-specific MSG_MORE flag is cheaper + * than additional setsockopt(TCP_CORK) syscall. + */ + flags |= MSG_MORE; + } + + if (niov == 1) { + /* + * Disposal of surplus kernel msghdr + * and iovec copy-in operations. + */ + return nxt_linux_send(c, iov->iov_base, iov->iov_len, flags); + } + + return nxt_linux_sendmsg(c, &sb, niov, flags); + } + + size = nxt_sendbuf_file_coalesce(&sb); + + nxt_log_debug(c->socket.log, "sendfile(%d, %FD, @%O, %uz)", + c->socket.fd, fb->file->fd, fb->file_pos, size); + + offset = fb->file_pos; + + n = nxt_sys_sendfile(c->socket.fd, fb->file->fd, &offset, size); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(c->socket.log, "sendfile(): %d", n); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, + "sendfile(%d, %FD, %O, %uz) failed %E \"%FN\"", + c->socket.fd, fb->file->fd, fb->file_pos, size, + err, fb->file->name); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "sendfile() %E", err); + + return 0; + } + + if (n < (ssize_t) size) { + c->socket.write_ready = 0; + } + + return n; +} + + +static ssize_t +nxt_linux_send(nxt_event_conn_t *c, void *buf, size_t size, nxt_uint_t flags) +{ + ssize_t n; + nxt_err_t err; + + n = send(c->socket.fd, buf, size, flags); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(c->socket.log, "send(%d, %p, %uz, 0x%uXi): %z", + c->socket.fd, buf, size, flags, n); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "send(%d, %p, %uz, 0x%uXi) failed %E", + c->socket.fd, buf, size, flags, err); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "send() %E", err); + + return 0; + } + + if (n < (ssize_t) size) { + c->socket.write_ready = 0; + } + + return n; +} + + +static ssize_t +nxt_linux_sendmsg(nxt_event_conn_t *c, nxt_sendbuf_coalesce_t *sb, + nxt_uint_t niov, nxt_uint_t flags) +{ + ssize_t n; + nxt_err_t err; + struct msghdr msg; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = sb->iobuf; + msg.msg_iovlen = niov; + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = 0; + + n = sendmsg(c->socket.fd, &msg, flags); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(c->socket.log, "sendmsg(%d, %ui, 0x%uXi): %d", + c->socket.fd, niov, flags, n); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "sendmsg(%d, %ui, 0x%uXi) failed %E", + c->socket.fd, niov, flags, err); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "sendmsg() %E", err); + + return 0; + } + + if (n < (ssize_t) sb->size) { + c->socket.write_ready = 0; + } + + return n; +} diff --git a/src/nxt_list.c b/src/nxt_list.c new file mode 100644 index 00000000..6ba7c9fa --- /dev/null +++ b/src/nxt_list.c @@ -0,0 +1,108 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_list_t * +nxt_list_create(nxt_mem_pool_t *mp, nxt_uint_t n, size_t size) +{ + nxt_list_t *list; + + list = nxt_mem_alloc(mp, sizeof(nxt_list_t) + n * size); + + if (nxt_fast_path(list != NULL)) { + list->last = &list->part; + list->size = size; + list->nalloc = n; + list->mem_pool = mp; + list->part.next = NULL; + list->part.nelts = 0; + } + + return list; +} + + +void * +nxt_list_add(nxt_list_t *list) +{ + void *elt; + nxt_list_part_t *last; + + last = list->last; + + if (last->nelts == list->nalloc) { + + /* The last list part is filled up, allocating a new list part. */ + + last = nxt_mem_alloc(list->mem_pool, + sizeof(nxt_list_part_t) + list->nalloc * list->size); + + if (nxt_slow_path(last == NULL)) { + return NULL; + } + + last->next = NULL; + last->nelts = 0; + + list->last->next = last; + list->last = last; + } + + elt = nxt_list_data(last) + last->nelts * list->size; + last->nelts++; + + return elt; +} + + +void * +nxt_list_zero_add(nxt_list_t *list) +{ + void *p; + + p = nxt_list_add(list); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, list->size); + } + + return p; +} + + +void * +nxt_list_next(nxt_list_t *list, nxt_list_next_t *next) +{ + if (next->part != NULL) { + next->elt++; + + if (next->elt < next->part->nelts) { + return nxt_list_next_value(list, next); + } + + next->part = next->part->next; + + if (next->part != NULL) { + next->elt = 0; + return nxt_list_data(next->part); + } + + } else { + next->part = nxt_list_part(list); + /* + * The first list part is allocated together with + * a nxt_list_t itself and it may never be NULL. + */ + if (next->part->nelts != 0) { + return nxt_list_data(next->part); + } + + } + + return NULL; +} diff --git a/src/nxt_list.h b/src/nxt_list.h new file mode 100644 index 00000000..4d14f4d9 --- /dev/null +++ b/src/nxt_list.h @@ -0,0 +1,131 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_LIST_H_INCLUDED_ +#define _NXT_LIST_H_INCLUDED_ + + +typedef struct nxt_list_part_s nxt_list_part_t; + +struct nxt_list_part_s { + nxt_list_part_t *next; + uintptr_t nelts; +}; + + +typedef struct { + nxt_list_part_t *last; +#if (NXT_64BIT) + uint32_t size; + uint32_t nalloc; +#else + uint16_t size; + uint16_t nalloc; +#endif + nxt_mem_pool_t *mem_pool; + nxt_list_part_t part; +} nxt_list_t; + + +typedef struct { + nxt_list_part_t *part; + uintptr_t elt; +} nxt_list_next_t; + + +#define \ +nxt_list_part(list) \ + (&(list)->part) + + +#define \ +nxt_list_data(part) \ + ((char *) (part) + sizeof(nxt_list_part_t)) + + +#define \ +nxt_list_first(list) \ + (void *) nxt_list_data(nxt_list_part(list)) + + +nxt_inline void * +nxt_list_elt(nxt_list_t *list, nxt_uint_t n) +{ + nxt_list_part_t *part; + + if (nxt_fast_path((list) != NULL)) { + part = nxt_list_part(list); + + while (part != NULL) { + if (n < (nxt_uint_t) part->nelts) { + return ((void *) (nxt_list_data(part) + n * (list)->size)); + } + + n -= (nxt_uint_t) part->nelts; + part = part->next; + } + } + + return NULL; +} + + +#define nxt_list_each(elt, list) \ + do { \ + if (nxt_fast_path((list) != NULL)) { \ + void *_end; \ + nxt_list_part_t *_part = nxt_list_part(list); \ + \ + do { \ + elt = (void *) nxt_list_data(_part); \ + \ + for (_end = (elt + _part->nelts); elt != _end; elt++) { \ + +#define nxt_list_loop \ + } \ + \ + _part = _part->next; \ + \ + } while (_part != NULL); \ + } \ + } while (0) + + +NXT_EXPORT nxt_list_t *nxt_list_create(nxt_mem_pool_t *mp, nxt_uint_t n, + size_t size); +NXT_EXPORT void *nxt_list_add(nxt_list_t *list); +NXT_EXPORT void *nxt_list_zero_add(nxt_list_t *list); + +NXT_EXPORT void *nxt_list_next(nxt_list_t *list, nxt_list_next_t *next); + + +#define \ +nxt_list_next_value(list, next) \ + ((void *) (nxt_list_data((next)->part) + (next)->elt * (list)->size)) + + +nxt_inline nxt_uint_t +nxt_list_nelts(nxt_list_t *list) +{ + nxt_uint_t n; + nxt_list_part_t *part; + + n = 0; + + if (nxt_fast_path((list) != NULL)) { + part = nxt_list_part(list); + + do { + n += (nxt_uint_t) part->nelts; + part = part->next; + } while (part != NULL); + } + + return n; +} + + +#endif /* _NXT_LIST_H_INCLUDED_ */ diff --git a/src/nxt_listen_socket.c b/src/nxt_listen_socket.c new file mode 100644 index 00000000..22b5bf8d --- /dev/null +++ b/src/nxt_listen_socket.c @@ -0,0 +1,252 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static u_char *nxt_listen_socket_log_handler(void *ctx, u_char *pos, + u_char *last); + + +nxt_int_t +nxt_listen_socket_create(nxt_listen_socket_t *ls, nxt_bool_t bind_test) +{ + nxt_log_t log, *old; + nxt_uint_t family; + nxt_socket_t s; + nxt_thread_t *thr; + nxt_sockaddr_t *sa; + + sa = ls->sockaddr; + + thr = nxt_thread(); + old = thr->log; + log = *thr->log; + log.ctx_handler = nxt_listen_socket_log_handler; + log.ctx = sa; + thr->log = &log; + + family = sa->u.sockaddr.sa_family; + + s = nxt_socket_create(family, sa->type, 0, ls->flags); + if (s == -1) { + goto socket_fail; + } + + if (nxt_socket_setsockopt(s, SOL_SOCKET, SO_REUSEADDR, 1) != NXT_OK) { + goto fail; + } + +#if (NXT_INET6 && defined IPV6_V6ONLY) + + if (family == AF_INET6 && ls->ipv6only) { + int ipv6only; + + ipv6only = (ls->ipv6only == 1); + + /* Ignore possible error. TODO: why? */ + (void) nxt_socket_setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, ipv6only); + } + +#endif + +#if 0 + + /* Ignore possible error. TODO: why? */ + (void) nxt_socket_setsockopt(s, SOL_SOCKET, SO_SNDBUF, 8192); + +#endif + +#ifdef TCP_DEFER_ACCEPT + + if (ls->read_after_accept) { + /* Defer accept() maximum at 1 second. */ + /* Ignore possible error. TODO: why? */ + (void) nxt_socket_setsockopt(s, IPPROTO_TCP, TCP_DEFER_ACCEPT, 1); + } + +#endif + + switch (nxt_socket_bind(s, sa, bind_test)) { + + case NXT_OK: + break; + + case NXT_ERROR: + goto fail; + + default: /* NXT_DECLINED: EADDRINUSE on bind() test */ + return NXT_OK; + } + +#if (NXT_HAVE_UNIX_DOMAIN) + + if (family == AF_UNIX) { + nxt_file_name_t *name; + nxt_file_access_t access; + + name = (nxt_file_name_t *) sa->u.sockaddr_un.sun_path; + + access = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + + if (nxt_file_set_access(name, access) != NXT_OK) { + goto fail; + } + + if (bind_test && nxt_file_delete(name) != NXT_OK) { + goto fail; + } + } + +#endif + + nxt_log_debug(&log, "listen(%d, %d)", s, ls->backlog); + + if (listen(s, ls->backlog) != 0) { + nxt_log_alert(&log, "listen(%d, %d) failed %E", + s, ls->backlog, nxt_socket_errno); + goto fail; + } + + ls->socket = s; + thr->log = old; + + return NXT_OK; + +fail: + + nxt_socket_close(s); + +socket_fail: + + thr->log = old; + + return NXT_ERROR; +} + + +nxt_int_t +nxt_listen_socket_update(nxt_listen_socket_t *ls, nxt_listen_socket_t *prev) +{ + nxt_log_t log, *old; + nxt_thread_t *thr; + + ls->socket = prev->socket; + + thr = nxt_thread(); + old = thr->log; + log = *thr->log; + log.ctx_handler = nxt_listen_socket_log_handler; + log.ctx = ls->sockaddr; + thr->log = &log; + + nxt_log_debug(&log, "listen(%d, %d)", ls->socket, ls->backlog); + + if (listen(ls->socket, ls->backlog) != 0) { + nxt_log_alert(&log, "listen(%d, %d) failed %E", + ls->socket, ls->backlog, nxt_socket_errno); + goto fail; + } + + thr->log = old; + + return NXT_OK; + +fail: + + thr->log = old; + + return NXT_ERROR; +} + + +size_t +nxt_listen_socket_pool_min_size(nxt_listen_socket_t *ls) +{ + size_t size; + + /* + * The first nxt_sockaddr_t is intended for mandatory remote sockaddr + * and textual representaion with port. The second nxt_sockaddr_t + * is intended for local sockaddr without textual representaion which + * may be required to get specific address of connection received on + * wildcard AF_INET and AF_INET6 addresses. For AF_UNIX addresses + * the local sockaddr is not required. + */ + + switch (ls->sockaddr->u.sockaddr.sa_family) { + +#if (NXT_INET6) + + case AF_INET6: + ls->socklen = sizeof(struct sockaddr_in6); + + size = offsetof(nxt_sockaddr_t, u) + sizeof(struct sockaddr_in6) + + NXT_INET6_ADDR_STR_LEN + (sizeof(":65535") - 1); + + if (IN6_IS_ADDR_UNSPECIFIED(&ls->sockaddr->u.sockaddr_in6.sin6_addr)) { + size += offsetof(nxt_sockaddr_t, u) + sizeof(struct sockaddr_in6); + } + + break; + +#endif + +#if (NXT_HAVE_UNIX_DOMAIN) + + case AF_UNIX: + /* + * A remote socket is usually unbound and thus has unspecified Unix + * domain sockaddr_un which can be shortcut to 3 bytes. To handle + * a bound remote socket correctly ls->socklen should be at least + * sizeof(struct sockaddr_un), see comment in unix/nxt_socket.h. + */ + ls->socklen = 3; + size = ls->socklen + sizeof("unix:") - 1; + + break; + +#endif + + default: + ls->socklen = sizeof(struct sockaddr_in); + + size = offsetof(nxt_sockaddr_t, u) + sizeof(struct sockaddr_in) + + NXT_INET_ADDR_STR_LEN + (sizeof(":65535") - 1); + + if (ls->sockaddr->u.sockaddr_in.sin_addr.s_addr == INADDR_ANY) { + size += offsetof(nxt_sockaddr_t, u) + sizeof(struct sockaddr_in); + } + + break; + } + +#if (NXT_SSLTLS) + + if (ls->ssltls) { + size += 4 * sizeof(void *) /* SSL/TLS connection */ + + sizeof(nxt_buf_mem_t) + + sizeof(nxt_mem_pool_cleanup_t); + } + +#endif + + return size + sizeof(nxt_mem_pool_t) + + sizeof(nxt_event_conn_t) + + sizeof(nxt_log_t); +} + + +static u_char * +nxt_listen_socket_log_handler(void *ctx, u_char *pos, u_char *end) +{ + nxt_sockaddr_t *sa; + + sa = ctx; + + return nxt_sprintf(pos, end, " while creating listening socket on %*s", + sa->text_len, sa->text); +} diff --git a/src/nxt_listen_socket.h b/src/nxt_listen_socket.h new file mode 100644 index 00000000..2100f17f --- /dev/null +++ b/src/nxt_listen_socket.h @@ -0,0 +1,61 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_LISTEN_SOCKET_H_INCLUDED_ +#define _NXT_LISTEN_SOCKET_H_INCLUDED_ + + +typedef struct { + /* nxt_socket_t is int. */ + nxt_socket_t socket; + int backlog; + + nxt_work_queue_t *work_queue; + nxt_work_handler_t handler; + + nxt_sockaddr_t *sockaddr; + + uint8_t flags; + uint8_t read_after_accept; /* 1 bit */ + +#if (NXT_SSLTLS) + uint8_t ssltls; /* 1 bit */ +#endif +#if (NXT_INET6 && defined IPV6_V6ONLY) + uint8_t ipv6only; /* 2 bits */ +#endif + + void *servers; + + socklen_t socklen; + uint32_t mem_pool_size; +} nxt_listen_socket_t; + + +#if (NXT_FREEBSD || NXT_MACOSX || NXT_OPENBSD) +/* + * A backlog is limited by system-wide sysctl kern.ipc.somaxconn. + * This is supported by FreeBSD 2.2, OpenBSD 2.0, and MacOSX. + */ +#define NXT_LISTEN_BACKLOG -1 + +#else +/* + * Linux, Solaris, and NetBSD treat negative value as 0. + * 511 is a safe default. + */ +#define NXT_LISTEN_BACKLOG 511 +#endif + + +NXT_EXPORT nxt_int_t nxt_listen_socket_create(nxt_listen_socket_t *ls, + nxt_bool_t bind_test); +NXT_EXPORT nxt_int_t nxt_listen_socket_update(nxt_listen_socket_t *ls, + nxt_listen_socket_t *prev); +NXT_EXPORT size_t nxt_listen_socket_pool_min_size(nxt_listen_socket_t *ls); + + +#endif /* _NXT_LISTEN_SOCKET_H_INCLUDED_ */ diff --git a/src/nxt_log.c b/src/nxt_log.c new file mode 100644 index 00000000..95591f18 --- /dev/null +++ b/src/nxt_log.c @@ -0,0 +1,112 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_uint_t nxt_debug; + +nxt_log_t nxt_main_log = { + NXT_LOG_INFO, + 0, + nxt_log_handler, + NULL, + NULL +}; + + +nxt_str_t nxt_log_levels[8] = { + nxt_string("emerg"), + nxt_string("alert"), + nxt_string("crit"), + nxt_string("error"), + nxt_string("warn"), + nxt_string("notice"), + nxt_string("info"), + nxt_string("debug") +}; + + +static const u_char *nxt_log_prefix; + + +void +nxt_log_start(const char *prefix) +{ + if (prefix != NULL && *prefix != '\0') { + nxt_log_prefix = (u_char *) prefix; + } +} + + +/* STUB */ +nxt_log_t * +nxt_log_set_ctx(nxt_log_t *log, nxt_log_ctx_handler_t handler, void *ctx) +{ + nxt_log_t *old; + nxt_thread_t *thr; + + thr = nxt_thread(); + old = thr->log; + + log->level = old->level; + log->handler = old->handler; + log->ctx_handler = handler; + log->ctx = ctx; + + thr->log = log; + + return old; +} + + +void nxt_cdecl +nxt_log_handler(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...) +{ + u_char *p, *syslogmsg, *end; + va_list args; + u_char msg[NXT_MAX_ERROR_STR]; + + p = msg; + end = msg + NXT_MAX_ERROR_STR; + + if (nxt_log_prefix != NULL) { + p = nxt_cpystrn(p, nxt_log_prefix, end - p); + *p++ = ':'; + *p++ = ' '; + } + + syslogmsg = p; + + p = nxt_sprintf(p, end, (log->ident != 0) ? "[%V] *%D " : "[%V] ", + &nxt_log_levels[level], log->ident); + + va_start(args, fmt); + p = nxt_vsprintf(p, end, fmt, args); + va_end(args); + + if (level != NXT_LOG_DEBUG && log->ctx_handler != NULL) { + p = log->ctx_handler(log->ctx, p, end); + } + + if (p > end - NXT_LINEFEED_SIZE) { + p = end - NXT_LINEFEED_SIZE; + } + + nxt_linefeed(p); + + (void) nxt_write_console(nxt_stderr, msg, p - msg); + + if (level <= NXT_LOG_ALERT) { + *(p - NXT_LINEFEED_SIZE) = '\0'; + + /* + * Syslog LOG_ALERT level is enough, because + * LOG_EMERG level broadcast a message to all users. + */ + nxt_write_syslog(LOG_ALERT, syslogmsg); + } +} diff --git a/src/nxt_log.h b/src/nxt_log.h new file mode 100644 index 00000000..cbd7d09b --- /dev/null +++ b/src/nxt_log.h @@ -0,0 +1,126 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_LOG_H_INCLUDED_ +#define _NXT_LOG_H_INCLUDED_ + + +#define NXT_LOG_EMERG 0 +#define NXT_LOG_ALERT 1 +#define NXT_LOG_CRIT 2 +#define NXT_LOG_ERR 3 +#define NXT_LOG_WARN 4 +#define NXT_LOG_NOTICE 5 +#define NXT_LOG_INFO 6 +#define NXT_LOG_DEBUG 7 + + +#define NXT_MAX_ERROR_STR 2048 + + +typedef void nxt_cdecl (*nxt_log_handler_t)(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...); +typedef u_char *(*nxt_log_ctx_handler_t)(void *ctx, u_char *pos, u_char *end); + + +struct nxt_log_s { + uint32_t level; + uint32_t ident; + nxt_log_handler_t handler; + nxt_log_ctx_handler_t ctx_handler; + void *ctx; +}; + + +NXT_EXPORT void nxt_log_start(const char *name); +NXT_EXPORT nxt_log_t *nxt_log_set_ctx(nxt_log_t *log, + nxt_log_ctx_handler_t handler, void *ctx); + +NXT_EXPORT void nxt_cdecl nxt_log_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...); + + +#define \ +nxt_log_level_enough(log, level) \ + ((log)->level >= (level)) + + +#define \ +nxt_log_emerg(_log, ...) \ + do { \ + nxt_log_t *_log_ = (_log); \ + \ + _log_->handler(NXT_LOG_EMERG, _log_, __VA_ARGS__); \ + } while (0) + + +#define \ +nxt_log_alert(_log, ...) \ + do { \ + nxt_log_t *_log_ = (_log); \ + \ + _log_->handler(NXT_LOG_ALERT, _log_, __VA_ARGS__); \ + } while (0) + + +#define \ +nxt_log_error(_level, _log, ...) \ + do { \ + nxt_log_t *_log_ = (_log); \ + nxt_uint_t _level_ = (_level); \ + \ + if (nxt_slow_path(_log_->level >= _level_)) { \ + _log_->handler(_level_, _log_, __VA_ARGS__); \ + } \ + } while (0) + + +#if (NXT_DEBUG) + +#define \ +nxt_log_debug(_log, ...) \ + do { \ + nxt_log_t *_log_ = (_log); \ + \ + if (nxt_slow_path(_log_->level == NXT_LOG_DEBUG || nxt_debug)) { \ + _log_->handler(NXT_LOG_DEBUG, _log_, __VA_ARGS__); \ + } \ + } while (0) + +#else + +#define \ +nxt_log_debug(...) + +#endif + + +#define \ +nxt_main_log_emerg(...) \ + nxt_log_emerg(&nxt_main_log, __VA_ARGS__) + + +#define \ +nxt_main_log_alert(...) \ + nxt_log_alert(&nxt_main_log, __VA_ARGS__) + + +#define \ +nxt_main_log_error(level, ...) \ + nxt_log_error(level, &nxt_main_log, __VA_ARGS__) + + +#define \ +nxt_main_log_debug(...) \ + nxt_log_debug(&nxt_main_log, __VA_ARGS__) + + +NXT_EXPORT extern nxt_uint_t nxt_debug; +NXT_EXPORT extern nxt_log_t nxt_main_log; +NXT_EXPORT extern nxt_str_t nxt_log_levels[]; + + +#endif /* _NXT_LOG_H_INCLUDED_ */ diff --git a/src/nxt_log_moderation.c b/src/nxt_log_moderation.c new file mode 100644 index 00000000..bba379de --- /dev/null +++ b/src/nxt_log_moderation.c @@ -0,0 +1,96 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_log_moderate_timer_handler(nxt_thread_t *thr, void *obj, + void *data); + + +nxt_bool_t +nxt_log_moderate_allow(nxt_log_moderation_t *mod) +{ + nxt_uint_t n; + nxt_time_t now; + nxt_bool_t allow, timer; + nxt_thread_t *thr; + + thr = nxt_thread(); + now = nxt_thread_time(thr); + + allow = 0; + timer = 0; + + nxt_thread_spin_lock(&mod->lock); + + n = mod->count++; + + if (now != mod->last) { + + if (n <= mod->limit) { + mod->last = now; + mod->count = 1; + allow = 1; + } + + /* "n > mod->limit" means that timer has already been set. */ + + } else { + + if (n < mod->limit) { + allow = 1; + + } else if (n == mod->limit) { + /* + * There is a race condition on 32-bit many core system + * capable to fail an operation 2^32 times per second. + * This can be fixed by storing mod->count as uint64_t. + */ + timer = 1; + mod->pid = nxt_pid; + } + } + + nxt_thread_spin_unlock(&mod->lock); + + if (timer) { + mod->timer.work_queue = &thr->work_queue.main; + mod->timer.handler = nxt_log_moderate_timer_handler; + mod->timer.log = &nxt_main_log; + + nxt_event_timer_add(thr->engine, &mod->timer, 1000); + } + + return allow; +} + + +static void +nxt_log_moderate_timer_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_bool_t msg; + nxt_atomic_uint_t n; + nxt_event_timer_t *ev; + nxt_log_moderation_t *mod; + + ev = obj; + mod = nxt_event_timer_data(ev, nxt_log_moderation_t, timer); + + nxt_thread_spin_lock(&mod->lock); + + mod->last = nxt_thread_time(thr); + n = mod->count; + mod->count = 0; + msg = (mod->pid == nxt_pid); + + nxt_thread_spin_unlock(&mod->lock); + + if (msg) { + nxt_log_error(mod->level, &nxt_main_log, "%s %uA times", + mod->msg, n - mod->limit); + } +} diff --git a/src/nxt_log_moderation.h b/src/nxt_log_moderation.h new file mode 100644 index 00000000..5a794e79 --- /dev/null +++ b/src/nxt_log_moderation.h @@ -0,0 +1,40 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_LOG_MODERATION_H_INCLUDED_ +#define _NXT_LOG_MODERATION_H_INCLUDED_ + + +typedef struct { + uint32_t level; + uint32_t limit; + const char *msg; + nxt_thread_spinlock_t lock; + nxt_pid_t pid; + nxt_uint_t count; + nxt_time_t last; + nxt_event_timer_t timer; +} nxt_log_moderation_t; + + +#define NXT_LOG_MODERATION 0, -1, 0, 0, NXT_EVENT_TIMER + + +#define \ +nxt_log_moderate(_mod, _level, _log, ...) \ + do { \ + nxt_log_t *_log_ = _log; \ + \ + if (_log_->level >= (_level) && nxt_log_moderate_allow(_mod)) { \ + _log_->handler(_level, _log_, __VA_ARGS__); \ + } \ + } while (0) + + +nxt_bool_t nxt_log_moderate_allow(nxt_log_moderation_t *mod); + + +#endif /* _NXT_LOG_MODERATION_H_INCLUDED_ */ diff --git a/src/nxt_lvlhsh.c b/src/nxt_lvlhsh.c new file mode 100644 index 00000000..fc3c372b --- /dev/null +++ b/src/nxt_lvlhsh.c @@ -0,0 +1,890 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * The level hash consists of hierarchical levels of arrays of pointers. + * The pointers may point to another level, a bucket, or NULL. + * The levels and buckets must be allocated in manner alike posix_memalign() + * to bookkeep additional information in pointer low bits. + * + * A level is an array of pointers. Its size is a power of 2. Levels + * may be different sizes, but on the same level the sizes are the same. + * Level sizes are specified by number of bits per level in lvlhsh->shift + * array. A hash may have up to 7 levels. There are two predefined + * shift arrays given by the first two shift array values: + * + * 1) [0, 0]: [4, 4, 4, 4, 4, 4, 4] on a 64-bit platform or + * [5, 5, 5, 5, 5, 5, 0] on a 32-bit platform, + * so default size of levels is 128 bytes. + * + * 2) [0, 10]: [10, 4, 4, 4, 4, 4, 0] on a 64-bit platform or + * [10, 5, 5, 5, 5, 0, 0] on a 32-bit platform, + * so default size of levels is 128 bytes on all levels except + * the first level. The first level is 8K or 4K on 64-bit or 32-bit + * platforms respectively. + * + * All buckets in a hash are the same size which is a power of 2. + * A bucket contains several entries stored and tested sequentially. + * The bucket size should be one or two CPU cache line size, a minimum + * allowed size is 32 bytes. A default 128-byte bucket contains 10 64-bit + * entries or 15 32-bit entries. Each entry consists of pointer to value + * data and 32-bit key. If an entry value pointer is NULL, the entry is free. + * On a 64-bit platform entry value pointers are no aligned, therefore they + * are accessed as two 32-bit integers. The rest trailing space in a bucket + * is used as pointer to next bucket and this pointer is always aligned. + * Although the level hash allows to store a lot of values in a bucket chain, + * this is non optimal way. The large data set should be stored using + * several levels. + */ + +#define \ +nxt_lvlhsh_is_bucket(p) \ + ((uintptr_t) (p) & 1) + + +#define \ +nxt_lvlhsh_count_inc(n) \ + n = (void *) ((uintptr_t) (n) + 2) + + +#define \ +nxt_lvlhsh_count_dec(n) \ + n = (void *) ((uintptr_t) (n) - 2) + + +#define \ +nxt_lvlhsh_level_size(proto, nlvl) \ + ((uintptr_t) 1 << proto->shift[nlvl]) + + +#define \ +nxt_lvlhsh_level(lvl, mask) \ + (void **) ((uintptr_t) lvl & (~mask << 2)) + + +#define \ +nxt_lvlhsh_level_entries(lvl, mask) \ + ((uintptr_t) lvl & (mask << 1)) + + +#define \ +nxt_lvlhsh_store_bucket(slot, bkt) \ + slot = (void **) ((uintptr_t) bkt | 2 | 1) + + +#define \ +nxt_lvlhsh_bucket_size(proto) \ + proto->bucket_size + + +#define \ +nxt_lvlhsh_bucket(proto, bkt) \ + (uint32_t *) ((uintptr_t) bkt & ~(uintptr_t) proto->bucket_mask) + + +#define \ +nxt_lvlhsh_bucket_entries(proto, bkt) \ + (((uintptr_t) bkt & (uintptr_t) proto->bucket_mask) >> 1) + + +#define \ +nxt_lvlhsh_bucket_end(proto, bkt) \ + &bkt[proto->bucket_end] + + +#define \ +nxt_lvlhsh_free_entry(e) \ + (!(nxt_lvlhsh_valid_entry(e))) + + +#define \ +nxt_lvlhsh_next_bucket(proto, bkt) \ + ((void **) &bkt[proto->bucket_end]) + +#if (NXT_64BIT) + +#define \ +nxt_lvlhsh_valid_entry(e) \ + (((e)[0] | (e)[1]) != 0) + + +#define \ +nxt_lvlhsh_entry_value(e) \ + (void *) (((uintptr_t) (e)[1] << 32) + (e)[0]) + + +#define \ +nxt_lvlhsh_set_entry_value(e, n) \ + (e)[0] = (uint32_t) (uintptr_t) n; \ + (e)[1] = (uint32_t) ((uintptr_t) n >> 32) + + +#define \ +nxt_lvlhsh_entry_key(e) \ + (e)[2] + + +#define \ +nxt_lvlhsh_set_entry_key(e, n) \ + (e)[2] = n + +#else + +#define \ +nxt_lvlhsh_valid_entry(e) \ + ((e)[0] != 0) + + +#define \ +nxt_lvlhsh_entry_value(e) \ + (void *) (e)[0] + + +#define \ +nxt_lvlhsh_set_entry_value(e, n) \ + (e)[0] = (uint32_t) n + + +#define \ +nxt_lvlhsh_entry_key(e) \ + (e)[1] + + +#define \ +nxt_lvlhsh_set_entry_key(e, n) \ + (e)[1] = n + +#endif + + +#define NXT_LVLHSH_BUCKET_DONE ((void *) -1) + + +static nxt_int_t nxt_lvlhsh_level_find(nxt_lvlhsh_query_t *lhq, void **lvl, + uint32_t key, nxt_uint_t nlvl); +static nxt_int_t nxt_lvlhsh_bucket_find(nxt_lvlhsh_query_t *lhq, void **bkt); +static nxt_int_t nxt_lvlhsh_new_bucket(nxt_lvlhsh_query_t *lhq, void **slot); +static nxt_int_t nxt_lvlhsh_level_insert(nxt_lvlhsh_query_t *lhq, + void **slot, uint32_t key, nxt_uint_t nlvl); +static nxt_int_t nxt_lvlhsh_bucket_insert(nxt_lvlhsh_query_t *lhq, + void **slot, uint32_t key, nxt_int_t nlvl); +static nxt_int_t nxt_lvlhsh_convert_bucket_to_level(nxt_lvlhsh_query_t *lhq, + void **slot, nxt_uint_t nlvl, uint32_t *bucket); +static nxt_int_t nxt_lvlhsh_level_convertion_insert(nxt_lvlhsh_query_t *lhq, + void **parent, uint32_t key, nxt_uint_t nlvl); +static nxt_int_t nxt_lvlhsh_bucket_convertion_insert(nxt_lvlhsh_query_t *lhq, + void **slot, uint32_t key, nxt_int_t nlvl); +static nxt_int_t nxt_lvlhsh_free_level(nxt_lvlhsh_query_t *lhq, void **level, + nxt_uint_t size); +static nxt_int_t nxt_lvlhsh_level_delete(nxt_lvlhsh_query_t *lhq, void **slot, + uint32_t key, nxt_uint_t nlvl); +static nxt_int_t nxt_lvlhsh_bucket_delete(nxt_lvlhsh_query_t *lhq, void **bkt); +static void *nxt_lvlhsh_level_each(nxt_lvlhsh_each_t *lhe, void **level, + nxt_uint_t nlvl, nxt_uint_t shift); +static void *nxt_lvlhsh_bucket_each(nxt_lvlhsh_each_t *lhe); + + +nxt_int_t +nxt_lvlhsh_find(nxt_lvlhsh_t *lh, nxt_lvlhsh_query_t *lhq) +{ + void *slot; + + slot = lh->slot; + + if (nxt_fast_path(slot != NULL)) { + + if (nxt_lvlhsh_is_bucket(slot)) { + return nxt_lvlhsh_bucket_find(lhq, slot); + } + + return nxt_lvlhsh_level_find(lhq, slot, lhq->key_hash, 0); + } + + return NXT_DECLINED; +} + + +static nxt_int_t +nxt_lvlhsh_level_find(nxt_lvlhsh_query_t *lhq, void **lvl, uint32_t key, + nxt_uint_t nlvl) +{ + void **slot; + uintptr_t mask; + nxt_uint_t shift; + + shift = lhq->proto->shift[nlvl]; + mask = ((uintptr_t) 1 << shift) - 1; + + lvl = nxt_lvlhsh_level(lvl, mask); + slot = lvl[key & mask]; + + if (slot != NULL) { + + if (nxt_lvlhsh_is_bucket(slot)) { + return nxt_lvlhsh_bucket_find(lhq, slot); + } + + return nxt_lvlhsh_level_find(lhq, slot, key >> shift, nlvl + 1); + } + + return NXT_DECLINED; +} + + +static nxt_int_t +nxt_lvlhsh_bucket_find(nxt_lvlhsh_query_t *lhq, void **bkt) +{ + void *value; + uint32_t *bucket, *e; + nxt_uint_t n; + + do { + bucket = nxt_lvlhsh_bucket(lhq->proto, bkt); + n = nxt_lvlhsh_bucket_entries(lhq->proto, bkt); + e = bucket; + + do { + if (nxt_lvlhsh_valid_entry(e)) { + n--; + + if (nxt_lvlhsh_entry_key(e) == lhq->key_hash) { + + value = nxt_lvlhsh_entry_value(e); + + if (lhq->proto->test(lhq, value) == NXT_OK) { + lhq->value = value; + + return NXT_OK; + } + } + } + + e += NXT_LVLHSH_ENTRY_SIZE; + + } while (n != 0); + + bkt = *nxt_lvlhsh_next_bucket(lhq->proto, bucket); + + } while (bkt != NULL); + + return NXT_DECLINED; +} + + +nxt_int_t +nxt_lvlhsh_insert(nxt_lvlhsh_t *lh, nxt_lvlhsh_query_t *lhq) +{ + uint32_t key; + + if (nxt_fast_path(lh->slot != NULL)) { + + key = lhq->key_hash; + + if (nxt_lvlhsh_is_bucket(lh->slot)) { + return nxt_lvlhsh_bucket_insert(lhq, &lh->slot, key, -1); + } + + return nxt_lvlhsh_level_insert(lhq, &lh->slot, key, 0); + } + + return nxt_lvlhsh_new_bucket(lhq, &lh->slot); +} + + +static nxt_int_t +nxt_lvlhsh_new_bucket(nxt_lvlhsh_query_t *lhq, void **slot) +{ + uint32_t *bucket; + + bucket = lhq->proto->alloc(lhq->pool, nxt_lvlhsh_bucket_size(lhq->proto), + lhq->proto->nalloc); + + if (nxt_fast_path(bucket != NULL)) { + + nxt_lvlhsh_set_entry_value(bucket, lhq->value); + nxt_lvlhsh_set_entry_key(bucket, lhq->key_hash); + + *nxt_lvlhsh_next_bucket(lhq->proto, bucket) = NULL; + + nxt_lvlhsh_store_bucket(*slot, bucket); + + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_lvlhsh_level_insert(nxt_lvlhsh_query_t *lhq, void **parent, uint32_t key, + nxt_uint_t nlvl) +{ + void **slot, **lvl; + nxt_int_t ret; + uintptr_t mask; + nxt_uint_t shift; + + shift = lhq->proto->shift[nlvl]; + mask = ((uintptr_t) 1 << shift) - 1; + + lvl = nxt_lvlhsh_level(*parent, mask); + slot = &lvl[key & mask]; + + if (*slot != NULL) { + key >>= shift; + + if (nxt_lvlhsh_is_bucket(*slot)) { + return nxt_lvlhsh_bucket_insert(lhq, slot, key, nlvl); + } + + return nxt_lvlhsh_level_insert(lhq, slot, key, nlvl + 1); + } + + ret = nxt_lvlhsh_new_bucket(lhq, slot); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_lvlhsh_count_inc(*parent); + } + + return ret; +} + + +static nxt_int_t +nxt_lvlhsh_bucket_insert(nxt_lvlhsh_query_t *lhq, void **slot, uint32_t key, + nxt_int_t nlvl) +{ + void **bkt, **vacant_bucket, *value; + uint32_t *bucket, *e, *vacant_entry; + nxt_int_t ret; + uintptr_t n; + const void *new_value; + const nxt_lvlhsh_proto_t *proto; + + bkt = slot; + vacant_entry = NULL; + vacant_bucket = NULL; + proto = lhq->proto; + + /* Search for duplicate entry in bucket chain. */ + + do { + bucket = nxt_lvlhsh_bucket(proto, *bkt); + n = nxt_lvlhsh_bucket_entries(proto, *bkt); + e = bucket; + + do { + if (nxt_lvlhsh_valid_entry(e)) { + + if (nxt_lvlhsh_entry_key(e) == lhq->key_hash) { + + value = nxt_lvlhsh_entry_value(e); + + if (proto->test(lhq, value) == NXT_OK) { + + new_value = lhq->value; + lhq->value = value; + + if (lhq->replace) { + nxt_lvlhsh_set_entry_value(e, new_value); + + return NXT_OK; + } + + return NXT_DECLINED; + } + } + + n--; + + } else { + /* + * Save a hole vacant position in bucket + * and continue to search for duplicate entry. + */ + if (vacant_entry == NULL) { + vacant_entry = e; + vacant_bucket = bkt; + } + } + + e += NXT_LVLHSH_ENTRY_SIZE; + + } while (n != 0); + + if (e < nxt_lvlhsh_bucket_end(proto, bucket)) { + /* + * Save a vacant position on incomplete bucket's end + * and continue to search for duplicate entry. + */ + if (vacant_entry == NULL) { + vacant_entry = e; + vacant_bucket = bkt; + } + } + + bkt = nxt_lvlhsh_next_bucket(proto, bucket); + + } while (*bkt != NULL); + + if (vacant_entry != NULL) { + nxt_lvlhsh_set_entry_value(vacant_entry, lhq->value); + nxt_lvlhsh_set_entry_key(vacant_entry, lhq->key_hash); + nxt_lvlhsh_count_inc(*vacant_bucket); + + return NXT_OK; + } + + /* All buckets are full. */ + + nlvl++; + + if (nxt_fast_path(proto->shift[nlvl] != 0)) { + + ret = nxt_lvlhsh_convert_bucket_to_level(lhq, slot, nlvl, bucket); + + if (nxt_fast_path(ret == NXT_OK)) { + return nxt_lvlhsh_level_insert(lhq, slot, key, nlvl); + } + + return ret; + } + + /* The last allowed level, only buckets may be allocated here. */ + + return nxt_lvlhsh_new_bucket(lhq, bkt); +} + + +static nxt_int_t +nxt_lvlhsh_convert_bucket_to_level(nxt_lvlhsh_query_t *lhq, void **slot, + nxt_uint_t nlvl, uint32_t *bucket) +{ + void *lvl, **level; + uint32_t *e, *end, key; + nxt_int_t ret; + nxt_uint_t i, shift, size; + nxt_lvlhsh_query_t q; + const nxt_lvlhsh_proto_t *proto; + + proto = lhq->proto; + size = nxt_lvlhsh_level_size(proto, nlvl); + + lvl = proto->alloc(lhq->pool, size * (sizeof(void *)), proto->nalloc); + + if (nxt_slow_path(lvl == NULL)) { + return NXT_ERROR; + } + + nxt_memzero(lvl, size * (sizeof(void *))); + + level = lvl; + shift = 0; + + for (i = 0; i < nlvl; i++) { + /* + * Using SIMD operations in this trivial loop with maximum + * 8 iterations may increase code size by 170 bytes. + */ + nxt_pragma_loop_disable_vectorization; + + shift += proto->shift[i]; + } + + end = nxt_lvlhsh_bucket_end(proto, bucket); + + for (e = bucket; e < end; e += NXT_LVLHSH_ENTRY_SIZE) { + + q.proto = proto; + q.pool = lhq->pool; + q.value = nxt_lvlhsh_entry_value(e); + key = nxt_lvlhsh_entry_key(e); + q.key_hash = key; + + ret = nxt_lvlhsh_level_convertion_insert(&q, &lvl, key >> shift, nlvl); + + if (nxt_slow_path(ret != NXT_OK)) { + return nxt_lvlhsh_free_level(lhq, level, size); + } + } + + *slot = lvl; + + proto->free(lhq->pool, bucket, nxt_lvlhsh_bucket_size(proto)); + + return NXT_OK; +} + + +static nxt_int_t +nxt_lvlhsh_level_convertion_insert(nxt_lvlhsh_query_t *lhq, void **parent, + uint32_t key, nxt_uint_t nlvl) +{ + void **slot, **lvl; + nxt_int_t ret; + uintptr_t mask; + nxt_uint_t shift; + + shift = lhq->proto->shift[nlvl]; + mask = ((uintptr_t) 1 << shift) - 1; + + lvl = nxt_lvlhsh_level(*parent, mask); + slot = &lvl[key & mask]; + + if (*slot == NULL) { + ret = nxt_lvlhsh_new_bucket(lhq, slot); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_lvlhsh_count_inc(*parent); + } + + return ret; + } + + /* Only backets can be here. */ + + return nxt_lvlhsh_bucket_convertion_insert(lhq, slot, key >> shift, nlvl); +} + + +/* + * The special bucket insertion procedure is required because during + * convertion lhq->key contains garbage values and the test function + * cannot be called. Besides, the procedure can be simpler because + * a new entry is inserted just after occupied entries. + */ + +static nxt_int_t +nxt_lvlhsh_bucket_convertion_insert(nxt_lvlhsh_query_t *lhq, void **slot, + uint32_t key, nxt_int_t nlvl) +{ + void **bkt; + uint32_t *bucket, *e; + nxt_int_t ret; + uintptr_t n; + const nxt_lvlhsh_proto_t *proto; + + bkt = slot; + proto = lhq->proto; + + do { + bucket = nxt_lvlhsh_bucket(proto, *bkt); + n = nxt_lvlhsh_bucket_entries(proto, *bkt); + e = bucket + n * NXT_LVLHSH_ENTRY_SIZE; + + if (nxt_fast_path(e < nxt_lvlhsh_bucket_end(proto, bucket))) { + + nxt_lvlhsh_set_entry_value(e, lhq->value); + nxt_lvlhsh_set_entry_key(e, lhq->key_hash); + nxt_lvlhsh_count_inc(*bkt); + + return NXT_OK; + } + + bkt = nxt_lvlhsh_next_bucket(proto, bucket); + + } while (*bkt != NULL); + + /* All buckets are full. */ + + nlvl++; + + if (nxt_fast_path(proto->shift[nlvl] != 0)) { + + ret = nxt_lvlhsh_convert_bucket_to_level(lhq, slot, nlvl, bucket); + + if (nxt_fast_path(ret == NXT_OK)) { + return nxt_lvlhsh_level_insert(lhq, slot, key, nlvl); + } + + return ret; + } + + /* The last allowed level, only buckets may be allocated here. */ + + return nxt_lvlhsh_new_bucket(lhq, bkt); +} + + +static nxt_int_t +nxt_lvlhsh_free_level(nxt_lvlhsh_query_t *lhq, void **level, nxt_uint_t size) +{ + size_t bsize; + nxt_uint_t i; + const nxt_lvlhsh_proto_t *proto; + + proto = lhq->proto; + bsize = nxt_lvlhsh_bucket_size(proto); + + for (i = 0; i < size; i++) { + + if (level[i] != NULL) { + /* + * Chained buckets are not possible here, since even + * in the worst case one bucket cannot be converted + * in two chained buckets but remains the same bucket. + */ + proto->free(lhq->pool, nxt_lvlhsh_bucket(proto, level[i]), bsize); + } + } + + proto->free(lhq->pool, level, size * (sizeof(void *))); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_lvlhsh_delete(nxt_lvlhsh_t *lh, nxt_lvlhsh_query_t *lhq) +{ + if (nxt_fast_path(lh->slot != NULL)) { + + if (nxt_lvlhsh_is_bucket(lh->slot)) { + return nxt_lvlhsh_bucket_delete(lhq, &lh->slot); + } + + return nxt_lvlhsh_level_delete(lhq, &lh->slot, lhq->key_hash, 0); + } + + return NXT_DECLINED; +} + + +static nxt_int_t +nxt_lvlhsh_level_delete(nxt_lvlhsh_query_t *lhq, void **parent, uint32_t key, + nxt_uint_t nlvl) +{ + size_t size; + void **slot, **lvl; + uintptr_t mask; + nxt_int_t ret; + nxt_uint_t shift; + + shift = lhq->proto->shift[nlvl]; + mask = ((uintptr_t) 1 << shift) - 1; + + lvl = nxt_lvlhsh_level(*parent, mask); + slot = &lvl[key & mask]; + + if (*slot != NULL) { + + if (nxt_lvlhsh_is_bucket(*slot)) { + ret = nxt_lvlhsh_bucket_delete(lhq, slot); + + } else { + key >>= shift; + ret = nxt_lvlhsh_level_delete(lhq, slot, key, nlvl + 1); + } + + if (*slot == NULL) { + nxt_lvlhsh_count_dec(*parent); + + if (nxt_lvlhsh_level_entries(*parent, mask) == 0) { + *parent = NULL; + size = nxt_lvlhsh_level_size(lhq->proto, nlvl); + lhq->proto->free(lhq->pool, lvl, size * sizeof(void *)); + } + } + + return ret; + } + + return NXT_DECLINED; +} + + +static nxt_int_t +nxt_lvlhsh_bucket_delete(nxt_lvlhsh_query_t *lhq, void **bkt) +{ + void *value; + size_t size; + uint32_t *bucket, *e; + uintptr_t n; + const nxt_lvlhsh_proto_t *proto; + + proto = lhq->proto; + + do { + bucket = nxt_lvlhsh_bucket(proto, *bkt); + n = nxt_lvlhsh_bucket_entries(proto, *bkt); + e = bucket; + + do { + if (nxt_lvlhsh_valid_entry(e)) { + + if (nxt_lvlhsh_entry_key(e) == lhq->key_hash) { + + value = nxt_lvlhsh_entry_value(e); + + if (proto->test(lhq, value) == NXT_OK) { + + if (nxt_lvlhsh_bucket_entries(proto, *bkt) == 1) { + *bkt = *nxt_lvlhsh_next_bucket(proto, bucket); + size = nxt_lvlhsh_bucket_size(proto); + proto->free(lhq->pool, bucket, size); + + } else { + nxt_lvlhsh_count_dec(*bkt); + nxt_lvlhsh_set_entry_value(e, NULL); + } + + lhq->value = value; + + return NXT_OK; + } + } + + n--; + } + + e += NXT_LVLHSH_ENTRY_SIZE; + + } while (n != 0); + + bkt = nxt_lvlhsh_next_bucket(proto, bucket); + + } while (*bkt != NULL); + + return NXT_DECLINED; +} + + +void * +nxt_lvlhsh_each(nxt_lvlhsh_t *lh, nxt_lvlhsh_each_t *lhe) +{ + void **slot; + + if (lhe->bucket == NXT_LVLHSH_BUCKET_DONE) { + slot = lh->slot; + + if (nxt_lvlhsh_is_bucket(slot)) { + return NULL; + } + + } else { + if (nxt_slow_path(lhe->bucket == NULL)) { + + /* The first iteration only. */ + + slot = lh->slot; + + if (slot == NULL) { + return NULL; + } + + if (!nxt_lvlhsh_is_bucket(slot)) { + goto level; + } + + lhe->bucket = nxt_lvlhsh_bucket(lhe->proto, slot); + lhe->entries = nxt_lvlhsh_bucket_entries(lhe->proto, slot); + } + + return nxt_lvlhsh_bucket_each(lhe); + } + +level: + + return nxt_lvlhsh_level_each(lhe, slot, 0, 0); +} + + +static void * +nxt_lvlhsh_level_each(nxt_lvlhsh_each_t *lhe, void **level, nxt_uint_t nlvl, + nxt_uint_t shift) +{ + void **slot, *value; + uintptr_t mask; + nxt_uint_t n, level_shift; + + level_shift = lhe->proto->shift[nlvl]; + mask = ((uintptr_t) 1 << level_shift) - 1; + + level = nxt_lvlhsh_level(level, mask); + + do { + n = (lhe->current >> shift) & mask; + slot = level[n]; + + if (slot != NULL) { + if (nxt_lvlhsh_is_bucket(slot)) { + + if (lhe->bucket != NXT_LVLHSH_BUCKET_DONE) { + + lhe->bucket = nxt_lvlhsh_bucket(lhe->proto, slot); + lhe->entries = nxt_lvlhsh_bucket_entries(lhe->proto, slot); + lhe->entry = 0; + + return nxt_lvlhsh_bucket_each(lhe); + } + + lhe->bucket = NULL; + + } else { + value = nxt_lvlhsh_level_each(lhe, slot, nlvl + 1, + shift + level_shift); + if (value != NULL) { + return value; + } + } + } + + lhe->current &= ~(mask << shift); + n = ((n + 1) & mask) << shift; + lhe->current |= n; + + } while (n != 0); + + return NULL; +} + + +static nxt_noinline void * +nxt_lvlhsh_bucket_each(nxt_lvlhsh_each_t *lhe) +{ + void *value, **next; + uint32_t *bucket; + + /* At least one valid entry must present here. */ + do { + bucket = &lhe->bucket[lhe->entry]; + lhe->entry += NXT_LVLHSH_ENTRY_SIZE; + + } while (nxt_lvlhsh_free_entry(bucket)); + + value = nxt_lvlhsh_entry_value(bucket); + + lhe->entries--; + + if (lhe->entries == 0) { + next = *nxt_lvlhsh_next_bucket(lhe->proto, lhe->bucket); + + lhe->bucket = (next == NULL) ? NXT_LVLHSH_BUCKET_DONE: + nxt_lvlhsh_bucket(lhe->proto, next); + + lhe->entries = nxt_lvlhsh_bucket_entries(lhe->proto, next); + lhe->entry = 0; + } + + return value; +} + + +void * +nxt_lvlhsh_alloc(void *data, size_t size, nxt_uint_t nalloc) +{ + return nxt_memalign(size, size); +} + + +void +nxt_lvlhsh_free(void *data, void *p, size_t size) +{ + nxt_free(p); +} diff --git a/src/nxt_lvlhsh.h b/src/nxt_lvlhsh.h new file mode 100644 index 00000000..554e958b --- /dev/null +++ b/src/nxt_lvlhsh.h @@ -0,0 +1,188 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_LEVEL_HASH_H_INCLUDED_ +#define _NXT_LEVEL_HASH_H_INCLUDED_ + + +typedef struct nxt_lvlhsh_query_s nxt_lvlhsh_query_t; + +typedef nxt_int_t (*nxt_lvlhsh_test_t)(nxt_lvlhsh_query_t *lhq, void *data); +typedef void *(*nxt_lvlhsh_alloc_t)(void *ctx, size_t size, nxt_uint_t nalloc); +typedef void (*nxt_lvlhsh_free_t)(void *ctx, void *p, size_t size); + + +#if (NXT_64BIT) + +#define NXT_LVLHSH_DEFAULT_BUCKET_SIZE 128 +#define NXT_LVLHSH_ENTRY_SIZE 3 +#define NXT_LVLHSH_BATCH_ALLOC 16 + +/* 3 is shift of 64-bit pointer. */ +#define NXT_LVLHSH_MEMALIGN_SHIFT (NXT_MAX_MEMALIGN_SHIFT - 3) + +#else + +#define NXT_LVLHSH_DEFAULT_BUCKET_SIZE 64 +#define NXT_LVLHSH_ENTRY_SIZE 2 +#define NXT_LVLHSH_BATCH_ALLOC 8 + +/* 2 is shift of 32-bit pointer. */ +#define NXT_LVLHSH_MEMALIGN_SHIFT (NXT_MAX_MEMALIGN_SHIFT - 2) + +#endif + + +#if (NXT_LVLHSH_MEMALIGN_SHIFT < 10) +#define NXT_LVLHSH_MAX_MEMALIGN_SHIFT NXT_LVLHSH_MEMALIGN_SHIFT +#else +#define NXT_LVLHSH_MAX_MEMALIGN_SHIFT 10 +#endif + + +#define NXT_LVLHSH_BUCKET_END(bucket_size) \ + (((bucket_size) - sizeof(void *)) \ + / (NXT_LVLHSH_ENTRY_SIZE * sizeof(uint32_t)) \ + * NXT_LVLHSH_ENTRY_SIZE) + + +#define NXT_LVLHSH_BUCKET_SIZE(bucket_size) \ + NXT_LVLHSH_BUCKET_END(bucket_size), bucket_size, (bucket_size - 1) + + +#define NXT_LVLHSH_DEFAULT \ + NXT_LVLHSH_BUCKET_SIZE(NXT_LVLHSH_DEFAULT_BUCKET_SIZE), \ + { 4, 4, 4, 4, 4, 4, 4, 0 } + + +#define NXT_LVLHSH_LARGE_SLAB \ + NXT_LVLHSH_BUCKET_SIZE(NXT_LVLHSH_DEFAULT_BUCKET_SIZE), \ + { 10, 4, 4, 4, 4, 4, 4, 0 } + + +#define NXT_LVLHSH_LARGE_MEMALIGN \ + NXT_LVLHSH_BUCKET_SIZE(NXT_LVLHSH_DEFAULT_BUCKET_SIZE), \ + { NXT_LVLHSH_MAX_MEMALIGN_SHIFT, 4, 4, 4, 4, 0, 0, 0 } + + +typedef struct { + uint32_t bucket_end; + uint32_t bucket_size; + uint32_t bucket_mask; + uint8_t shift[8]; + uint32_t nalloc; + + nxt_lvlhsh_test_t test; + nxt_lvlhsh_alloc_t alloc; + nxt_lvlhsh_free_t free; +} nxt_lvlhsh_proto_t; + + +typedef struct { + nxt_lvlhsh_test_t test; + nxt_lvlhsh_alloc_t alloc; + nxt_lvlhsh_free_t free; + + /* The maximum allowed aligned shift. */ + uint32_t max_shift; + uint32_t nalloc; +} nxt_lvlhsh_ctx_t; + + +typedef struct { + void *slot; +} nxt_lvlhsh_t; + + +struct nxt_lvlhsh_query_s { + uint32_t key_hash; + uint32_t replace; /* 1 bit */ + nxt_str_t key; + void *value; + + const nxt_lvlhsh_proto_t *proto; + void *pool; + + /* Opaque data passed for the test function. */ + void *data; +}; + + +#define \ +nxt_lvlhsh_is_empty(lh) \ + ((lh)->slot == NULL) + + +#define \ +nxt_lvlhsh_init(lh) \ + (lh)->slot = NULL + +/* + * nxt_lvlhsh_find() finds a hash element. If the element has been + * found then it is stored in the lhq->value and nxt_lvlhsh_find() + * returns NXT_OK. Otherwise NXT_DECLINED is returned. + * + * The required nxt_lvlhsh_query_t fields: key_hash, key, proto. + */ +NXT_EXPORT nxt_int_t nxt_lvlhsh_find(nxt_lvlhsh_t *lh, nxt_lvlhsh_query_t *lhq); + +/* + * nxt_lvlhsh_insert() adds a hash element. If the element already + * presents in lvlhsh and the lhq->replace flag is zero, then lhq->value + * is updated with the old element and NXT_DECLINED is returned. + * If the element already presents in lvlhsh and the lhq->replace flag + * is non-zero, then the old element is replaced with the new element. + * lhq->value is updated with the old element, and NXT_OK is returned. + * If the element is not present in lvlhsh, then it is inserted and + * NXT_OK is returned. The lhq->value is not changed. + * On memory allocation failure NXT_ERROR is returned. + * + * The required nxt_lvlhsh_query_t fields: key_hash, key, proto, replace, value. + * The optional nxt_lvlhsh_query_t fields: pool. + */ +NXT_EXPORT nxt_int_t nxt_lvlhsh_insert(nxt_lvlhsh_t *lh, + nxt_lvlhsh_query_t *lhq); + +/* + * nxt_lvlhsh_delete() deletes a hash element. If the element has been + * found then it is removed from lvlhsh and is stored in the lhq->value, + * and NXT_OK is returned. Otherwise NXT_DECLINED is returned. + * + * The required nxt_lvlhsh_query_t fields: key_hash, key, proto. + * The optional nxt_lvlhsh_query_t fields: pool. + */ +NXT_EXPORT nxt_int_t nxt_lvlhsh_delete(nxt_lvlhsh_t *lh, + nxt_lvlhsh_query_t *lhq); + + +typedef struct { + const nxt_lvlhsh_proto_t *proto; + + /* + * Fields to store current bucket entry position. They cannot be + * combined in a single bucket pointer with number of entries in low + * bits, because entry positions are not aligned. A current level is + * stored as key bit path from the root. + */ + uint32_t *bucket; + uint32_t current; + uint32_t entry; + uint32_t entries; +} nxt_lvlhsh_each_t; + + +NXT_EXPORT void *nxt_lvlhsh_each(nxt_lvlhsh_t *lh, nxt_lvlhsh_each_t *le); + + +NXT_EXPORT void *nxt_lvlhsh_alloc(void *data, size_t size, nxt_uint_t nalloc); +NXT_EXPORT void nxt_lvlhsh_free(void *data, void *p, size_t size); + +NXT_EXPORT void *nxt_lvlhsh_pool_alloc(void *ctx, size_t size, + nxt_uint_t nalloc); +NXT_EXPORT void nxt_lvlhsh_pool_free(void *ctx, void *p, size_t size); + + +#endif /* _NXT_LEVEL_HASH_H_INCLUDED_ */ diff --git a/src/nxt_lvlhsh_pool.c b/src/nxt_lvlhsh_pool.c new file mode 100644 index 00000000..5de319f9 --- /dev/null +++ b/src/nxt_lvlhsh_pool.c @@ -0,0 +1,153 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +typedef struct nxt_lvlhsh_pool_cache_s nxt_lvlhsh_pool_cache_t; + +struct nxt_lvlhsh_pool_cache_s { + uint32_t size; + uint32_t nalloc; + void *free; + nxt_lvlhsh_pool_cache_t *next; +}; + + +typedef struct { + nxt_mem_pool_t *mem_pool; + void *free; + nxt_lvlhsh_pool_cache_t *next; +} nxt_lvlhsh_pool_t; + + +/* + * lvlhsh requires allocations aligned to a size of the allocations. + * This is not issue for slab-like allocators, but glibc allocator may + * waste memory on such aligned allocations. So nxt_lvlhsh_pool_alloc() + * allocates memory in chunks specified by the "nalloc" parameter + * except the first allocation. The first lvlhsh allocation is a bucket + * allocation and it is enough for a small hash or for early stage of + * a large hash. By default lvlhsh uses 128-bytes or 64-bytes buckets + * and levels on 64-bit and 32-bit platforms respectively. + * This allows to search up to 10 entries in one memory access and + * up to 160 entries in two memory accesses on 64-bit platform. + * And on 32-bit platform up to 7 entries and up to 112 entries + * respectively. + * + * After the bucket has been filled up with 10 64-bit entries + * or 7 32-bit entries, lvlhsh expands it to a level and spreads + * content of the first bucket to the level's new buckets. + * Number of the new allocations may be up to 11 on 64-bit or + * 8 on 32-bit platforms. It's better to allocate them together + * to eliminate wasting memory and CPU time. + * + * The "nalloc" should be 16. + */ + + +static void *nxt_lvlhsh_pool_alloc_chunk(nxt_mem_pool_cache_t *cache, + size_t size, nxt_uint_t nalloc); + + +/* Allocation of lvlhsh level or bucket with specified size. */ + +void * +nxt_lvlhsh_pool_alloc(void *ctx, size_t size, nxt_uint_t nalloc) +{ + void *p, **pp; + nxt_mem_pool_t *mp; + nxt_mem_pool_cache_t *cache; + + mp = ctx; + + for (cache = mp->cache; cache != NULL; cache = cache->next) { + + if (cache->size == size && cache->nalloc != 0) { + + if (cache->free != NULL) { + pp = cache->free; + cache->free = *pp; + return pp; + } + + return nxt_lvlhsh_pool_alloc_chunk(cache, size, nalloc); + } + } + + cache = nxt_mem_alloc(mp, sizeof(nxt_mem_pool_cache_t)); + + if (nxt_fast_path(cache != NULL)) { + + p = nxt_memalign(size, size); + + if (nxt_fast_path(p != NULL)) { + cache->size = size; + cache->nalloc = nalloc; + cache->free = NULL; + cache->next = mp->cache; + mp->cache = cache; + return p; + } + } + + return NULL; +} + + +static void * +nxt_lvlhsh_pool_alloc_chunk(nxt_mem_pool_cache_t *cache, size_t size, + nxt_uint_t nalloc) +{ + char *m, *p, *end; + void **pp; + size_t n; + + n = (nalloc == 0) ? 1 : nalloc; + n *= size; + + m = nxt_memalign(size, n); + + if (nxt_fast_path(m != NULL)) { + + pp = &cache->free; + end = m + n; + + for (p = m + size; p < end; p = p + size) { + *pp = p; + pp = (void **) p; + } + + *pp = NULL; + } + + return m; +} + + + +/* Deallocation of lvlhsh level or bucket with specified size. */ + +void +nxt_lvlhsh_pool_free(void *ctx, void *p, size_t size) +{ + void **pp; + nxt_mem_pool_t *mp; + nxt_mem_pool_cache_t *cache; + + mp = ctx; + + pp = p; + + for (cache = mp->cache; cache != NULL; cache = cache->next) { + + if (cache->size == size && cache->nalloc != 0) { + *pp = cache->free; + cache->free = p; + return; + } + } +} diff --git a/src/nxt_macosx_sendfile.c b/src/nxt_macosx_sendfile.c new file mode 100644 index 00000000..94a76718 --- /dev/null +++ b/src/nxt_macosx_sendfile.c @@ -0,0 +1,393 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* sendfile() has been introduced in MacOSX 10.5 (Leopard) */ + +#ifdef NXT_TEST_BUILD_MACOSX_SENDFILE + +ssize_t nxt_macosx_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + +static int nxt_sys_sendfile(int fd, int s, off_t offset, off_t *len, + struct sf_hdtr *hdtr, int flags) +{ + return -1; +} + +#else +#define nxt_sys_sendfile sendfile +#endif + + +ssize_t +nxt_macosx_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit) +{ + size_t hd_size, file_size; + ssize_t n; + nxt_buf_t *fb; + nxt_err_t err; + nxt_off_t sent; + nxt_uint_t nhd, ntr; + struct iovec hd[NXT_IOBUF_MAX], tr[NXT_IOBUF_MAX]; + struct sf_hdtr hdtr, *ht; + nxt_sendbuf_coalesce_t sb; + + sb.buf = b; + sb.iobuf = hd; + sb.nmax = NXT_IOBUF_MAX; + sb.sync = 0; + sb.size = 0; + sb.limit = limit; + + nhd = nxt_sendbuf_mem_coalesce(&sb); + + if (nhd == 0 && sb.sync) { + return 0; + } + + if (sb.buf == NULL || !nxt_buf_is_file(sb.buf)) { + return nxt_event_conn_io_writev(c, hd, nhd); + } + + hd_size = sb.size; + fb = sb.buf; + + file_size = nxt_sendbuf_file_coalesce(&sb); + + if (file_size == 0) { + return nxt_event_conn_io_writev(c, hd, nhd); + } + + sb.iobuf = tr; + + ntr = nxt_sendbuf_mem_coalesce(&sb); + + /* + * Disposal of surplus kernel operations if there are no headers + * and trailers. Besides sendfile() returns EINVAL if a sf_hdtr's + * count is 0, but corresponding pointer is not NULL. + */ + + nxt_memzero(&hdtr, sizeof(struct sf_hdtr)); + ht = NULL; + + if (nhd != 0) { + ht = &hdtr; + hdtr.headers = hd; + hdtr.hdr_cnt = nhd; + } + + if (ntr != 0) { + ht = &hdtr; + hdtr.trailers = tr; + hdtr.trl_cnt = ntr; + } + + /* + * MacOSX has the same bug as old FreeBSD (http://bugs.freebsd.org/33771). + * However this bug has never been fixed and instead of this it has been + * documented as a feature in MacOSX 10.7 (Lion) sendfile(2): + * + * When a header or trailer is specified, the value of len argument + * indicates the maximum number of bytes in the header and/or file + * to be sent. It does not control the trailer; if a trailer exists, + * all of it will be sent. + */ + sent = hd_size + file_size; + + nxt_log_debug(c->socket.log, + "sendfile(%FD, %d, @%O, %O) hd:%ui tr:%ui hs:%uz", + fb->file->fd, c->socket.fd, fb->file_pos, sent, + nhd, ntr, hd_size); + + n = nxt_sys_sendfile(fb->file->fd, c->socket.fd, + fb->file_pos, &sent, ht, 0); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(c->socket.log, "sendfile(): %d sent:%O", n, sent); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "sendfile(%FD, %d, %O, %O) failed " + "%E \"%FN\" hd:%ui tr:%ui", fb->file->fd, + c->socket.fd, fb->file_pos, sent, err, + fb->file->name, nhd, ntr); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "sendfile() %E", err); + + return sent; + } + + if (sent == 0) { + nxt_log_error(NXT_LOG_ERR, c->socket.log, + "file \"%FN\" was truncated while sendfile()", + fb->file->name); + + return NXT_ERROR; + } + + if (sent < (nxt_off_t) sb.size) { + c->socket.write_ready = 0; + } + + return sent; +} + + +#if 0 + +typedef struct { + nxt_socket_t socket; + nxt_err_t error; + + uint8_t write_ready; /* 1 bit */ + uint8_t log_error; +} nxt_sendbuf_t; + + +ssize_t nxt_macosx_sendfile(nxt_thread_t *thr, nxt_sendbuf_t *sb, nxt_buf_t *b, + size_t limit); +ssize_t nxt_writev(nxt_thread_t *thr, nxt_sendbuf_t *sb, nxt_iobuf_t *iob, + nxt_uint_t niob); +ssize_t nxt_send(nxt_thread_t *thr, nxt_sendbuf_t *sb, void *buf, size_t size); + + +ssize_t +nxt_macosx_sendfile(nxt_thread_t *thr, nxt_sendbuf_t *sb, nxt_buf_t *b, + size_t limit) +{ + size_t hd_size, file_size; + ssize_t n; + nxt_buf_t *buf; + nxt_err_t err; + nxt_off_t sent; + nxt_uint_t nhd, ntr; + struct iovec hd[NXT_IOBUF_MAX], tr[NXT_IOBUF_MAX]; + struct sf_hdtr hdtr, *ht; + nxt_sendbuf_coalesce_t sbc; + + sbc.buf = b; + sbc.iobuf = hd; + sbc.nmax = NXT_IOBUF_MAX; + sbc.sync = 0; + sbc.size = 0; + sbc.limit = limit; + + nhd = nxt_sendbuf_mem_coalesce(&sbc); + + if (nhd == 0 && sbc.sync) { + return 0; + } + + if (sbc.buf == NULL || !nxt_buf_is_file(sbc.buf)) { + return nxt_writev(thr, sb, hd, nhd); + } + + hd_size = sbc.size; + buf = sbc.buf; + + file_size = nxt_sendbuf_file_coalesce(&sbc); + + if (file_size == 0) { + return nxt_writev(thr, sb, hd, nhd); + } + + sbc.iobuf = tr; + + ntr = nxt_sendbuf_mem_coalesce(&sbc); + + /* + * Disposal of surplus kernel operations if there are no headers + * and trailers. Besides sendfile() returns EINVAL if a sf_hdtr's + * count is 0, but corresponding pointer is not NULL. + */ + + nxt_memzero(&hdtr, sizeof(struct sf_hdtr)); + ht = NULL; + + if (nhd != 0) { + ht = &hdtr; + hdtr.headers = hd; + hdtr.hdr_cnt = nhd; + } + + if (ntr != 0) { + ht = &hdtr; + hdtr.trailers = tr; + hdtr.trl_cnt = ntr; + } + + /* + * MacOSX has the same bug as old FreeBSD (http://bugs.freebsd.org/33771). + * However this bug has never been fixed and instead of this it has been + * documented as a feature in MacOSX 10.7 (Lion) sendfile(2): + * + * When a header or trailer is specified, the value of len argument + * indicates the maximum number of bytes in the header and/or file + * to be sent. It does not control the trailer; if a trailer exists, + * all of it will be sent. + */ + sent = hd_size + file_size; + + nxt_log_debug(thr->log, "sendfile(%FD, %d, @%O, %O) hd:%ui tr:%ui hs:%uz", + buf->file->fd, sb->socket, buf->file_pos, sent, + nhd, ntr, hd_size); + + n = nxt_sys_sendfile(buf->file->fd, sb->socket, + buf->file_pos, &sent, ht, 0); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(thr->log, "sendfile(): %d sent:%O", n, sent); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + sb->write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + sb->error = err; + nxt_log_error(nxt_socket_error_level(err, sb->log_error), thr->log, + "sendfile(%FD, %d, %O, %O) failed %E \"%FN\" " + "hd:%ui tr:%ui", buf->file->fd, sb->socket, + buf->file_pos, sent, err, buf->file->name, nhd, ntr); + + return NXT_ERROR; + } + + nxt_log_debug(thr->log, "sendfile() %E", err); + + return sent; + } + + if (sent == 0) { + nxt_log_error(NXT_LOG_ERR, thr->log, + "file \"%FN\" was truncated while sendfile()", + buf->file->name); + + return NXT_ERROR; + } + + if (sent < (nxt_off_t) sbc.size) { + sb->write_ready = 0; + } + + return sent; +} + + +ssize_t +nxt_writev(nxt_thread_t *thr, nxt_sendbuf_t *sb, nxt_iobuf_t *iob, + nxt_uint_t niob) +{ + ssize_t n; + nxt_err_t err; + + if (niob == 1) { + /* Disposal of surplus kernel iovec copy-in operation. */ + return nxt_send(thr, sb, iob->iov_base, iob->iov_len); + } + + for ( ;; ) { + n = writev(sb->socket, iob, niob); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(thr->log, "writev(%d, %ui): %d", sb->socket, niob, n); + + if (n > 0) { + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(thr->log, "writev() %E", err); + sb->write_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(thr->log, "writev() %E", err); + continue; + + default: + sb->error = err; + nxt_log_error(nxt_socket_error_level(err, sb->log_error), thr->log, + "writev(%d, %ui) failed %E", sb->socket, niob, err); + return NXT_ERROR; + } + } +} + + +ssize_t +nxt_send(nxt_thread_t *thr, nxt_sendbuf_t *sb, void *buf, size_t size) +{ + ssize_t n; + nxt_err_t err; + + for ( ;; ) { + n = send(sb->socket, buf, size, 0); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(thr->log, "send(%d, %p, %uz): %z", + sb->socket, buf, size, n); + + if (n > 0) { + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(thr->log, "send() %E", err); + sb->write_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(thr->log, "send() %E", err); + continue; + + default: + sb->error = err; + nxt_log_error(nxt_socket_error_level(err, sb->log_error), thr->log, + "send(%d, %p, %uz) failed %E", + sb->socket, buf, size, err); + return NXT_ERROR; + } + } +} + +#endif diff --git a/src/nxt_main.c b/src/nxt_main.c new file mode 100644 index 00000000..ea2061d6 --- /dev/null +++ b/src/nxt_main.c @@ -0,0 +1,45 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_cycle.h> + + +extern char **environ; + + +int nxt_cdecl +main(int argc, char **argv) +{ + nxt_int_t ret; + nxt_thread_t *thr; + + static nxt_str_t nxt_config_name = nxt_string_zero("nginx.conf"); + + if (nxt_lib_start("nginman", argv, &environ) != NXT_OK) { + return 1; + } + +// nxt_main_log.level = NXT_LOG_INFO; + + thr = nxt_thread(); + nxt_thread_time_update(thr); + + nxt_main_log.handler = nxt_log_time_handler; + + nxt_log_error(NXT_LOG_INFO, thr->log, "nginman started"); + + ret = nxt_cycle_create(thr, NULL, NULL, &nxt_config_name, 0); + + if (ret != NXT_OK) { + return 1; + } + + nxt_event_engine_start(thr->engine); + + nxt_unreachable(); + return 0; +} diff --git a/src/nxt_main.h b/src/nxt_main.h new file mode 100644 index 00000000..24fb80cd --- /dev/null +++ b/src/nxt_main.h @@ -0,0 +1,180 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_LIB_H_INCLUDED_ +#define _NXT_LIB_H_INCLUDED_ + + +#include <nxt_auto_config.h> + +#include <nxt_unix.h> +#include <nxt_clang.h> +#include <nxt_types.h> +#include <nxt_time.h> +#include <nxt_process.h> + +typedef struct nxt_thread_s nxt_thread_t; +#include <nxt_thread_id.h> + +typedef struct nxt_mem_pool_s nxt_mem_pool_t; +#include <nxt_mem_pool.h> + +#include <nxt_errno.h> +#include <nxt_file.h> + +#include <nxt_random.h> +#include <nxt_string.h> +#include <nxt_utf8.h> +#include <nxt_file_name.h> + +typedef struct nxt_log_s nxt_log_t; +#include <nxt_log.h> + + +#include <nxt_atomic.h> +#include <nxt_queue.h> +#include <nxt_rbtree.h> +#include <nxt_sprintf.h> +#include <nxt_parse.h> + + +/* TODO: remove unused */ + +typedef struct nxt_event_fd_s nxt_event_fd_t; +typedef struct nxt_sockaddr_s nxt_sockaddr_t; + + +#include <nxt_malloc.h> +#include <nxt_mem_map.h> +#include <nxt_socket.h> +#include <nxt_spinlock.h> +#include <nxt_dyld.h> + +#include <nxt_work_queue.h> + + +typedef void *(*nxt_mem_proto_alloc_t)(void *pool, size_t size); +typedef void (*nxt_mem_proto_free_t)(void *pool, void *p); + +typedef struct { + nxt_mem_proto_alloc_t alloc; + nxt_mem_proto_free_t free; +} nxt_mem_proto_t; + + +#include <nxt_mem_cache_pool.h> +#include <nxt_mem_zone.h> +#include <nxt_mem_pool_cleanup.h> +#include <nxt_thread_time.h> + +typedef struct nxt_event_engine_s nxt_event_engine_t; +#include <nxt_event_timer.h> +#include <nxt_fiber.h> + +typedef struct nxt_thread_pool_s nxt_thread_pool_t; +#include <nxt_thread.h> + +#include <nxt_signal.h> +#if (NXT_THREADS) +#include <nxt_semaphore.h> +#endif + +#include <nxt_djb_hash.h> +#include <nxt_murmur_hash.h> +#include <nxt_lvlhsh.h> +#include <nxt_hash.h> + +#include <nxt_sort.h> +#include <nxt_array.h> +#include <nxt_vector.h> +#include <nxt_list.h> + +#include <nxt_service.h> + +typedef struct nxt_buf_s nxt_buf_t; +#include <nxt_buf.h> +#include <nxt_buf_pool.h> +#include <nxt_recvbuf.h> + +typedef struct nxt_event_conn_s nxt_event_conn_t; +#include <nxt_sendbuf.h> + +#include <nxt_log_moderation.h> + +#if (NXT_SSLTLS) +#include <nxt_ssltls.h> +#endif + + +#define \ +nxt_thread() \ + (nxt_thread_t *) nxt_thread_get_data(nxt_thread_context) + +nxt_thread_extern_data(nxt_thread_t, nxt_thread_context); + + +#include <nxt_thread_log.h> + +#include <nxt_event_fd.h> + +#include <nxt_chan.h> +#if (NXT_THREADS) +#include <nxt_thread_pool.h> +#endif + + +typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, + nxt_event_conn_t *c); +#include <nxt_listen_socket.h> + +#include <nxt_event_conn.h> + +#include <nxt_event_file.h> + +#include <nxt_event_set.h> +#include <nxt_event_engine.h> + +#include <nxt_job.h> +#include <nxt_job_file.h> +#include <nxt_buf_filter.h> + +#include <nxt_job_resolve.h> +#include <nxt_sockaddr.h> + +#include <nxt_cache.h> + +#include <nxt_source.h> +typedef struct nxt_upstream_source_s nxt_upstream_source_t; + +#include <nxt_stream_source.h> +#include <nxt_upstream.h> +#include <nxt_upstream_source.h> +#include <nxt_http_parse.h> +#include <nxt_http_source.h> +#include <nxt_fastcgi_source.h> + + +#if (NXT_LIB_UNIT_TEST) +#include <../test/nxt_lib_unit_test.h> +#else +#define NXT_LIB_UNIT_TEST_STATIC static +#endif + + +/* + * The envp argument must be &environ if application may + * change its process title with nxt_process_title(). + */ +NXT_EXPORT nxt_int_t nxt_lib_start(const char *app, char **argv, char ***envp); +NXT_EXPORT void nxt_lib_stop(void); + + +NXT_EXPORT extern nxt_uint_t nxt_ncpu; +NXT_EXPORT extern nxt_uint_t nxt_pagesize; +NXT_EXPORT extern nxt_random_t nxt_random_data; + + +#endif /* _NXT_LIB_H_INCLUDED_ */ diff --git a/src/nxt_malloc.c b/src/nxt_malloc.c new file mode 100644 index 00000000..568f04c6 --- /dev/null +++ b/src/nxt_malloc.c @@ -0,0 +1,210 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_log_moderation_t nxt_malloc_log_moderation = { + NXT_LOG_ALERT, 2, "memory allocation failed", NXT_LOG_MODERATION +}; + + +static nxt_log_t * +nxt_malloc_log(void) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + + if (thr != NULL && thr->log != NULL) { + return thr->log; + } + + return &nxt_main_log; +} + + +void * +nxt_malloc(size_t size) +{ + void *p; + + p = malloc(size); + + if (nxt_fast_path(p != NULL)) { + nxt_log_debug(nxt_malloc_log(), "malloc(%uz): %p", size, p); + + } else { + nxt_log_moderate(&nxt_malloc_log_moderation, + NXT_LOG_ALERT, nxt_malloc_log(), + "malloc(%uz) failed %E", size, nxt_errno); + } + + return p; +} + + +void * +nxt_zalloc(size_t size) +{ + void *p; + + p = nxt_malloc(size); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, size); + } + + return p; +} + + +void * +nxt_realloc(void *p, size_t size) +{ + void *n; + + n = realloc(p, size); + + if (nxt_fast_path(n != NULL)) { + nxt_log_debug(nxt_malloc_log(), "realloc(%p, %uz): %p", p, size, n); + + } else { + nxt_log_moderate(&nxt_malloc_log_moderation, + NXT_LOG_ALERT, nxt_malloc_log(), + "realloc(%p, %uz) failed %E", p, size, nxt_errno); + } + + return n; +} + + +#if (NXT_DEBUG) + +void +nxt_free(void *p) +{ + nxt_log_debug(nxt_malloc_log(), "free(%p)", p); + + free(p); +} + + +#endif + + +#if (NXT_HAVE_POSIX_MEMALIGN) + +/* + * posix_memalign() presents in Linux glibc 2.1.91, FreeBSD 7.0, + * Solaris 11, MacOSX 10.6 (Snow Leopard), NetBSD 5.0. + */ + +void * +nxt_memalign(size_t alignment, size_t size) +{ + void *p; + nxt_err_t err; + + err = posix_memalign(&p, alignment, size); + + if (nxt_fast_path(err == 0)) { + nxt_thread_log_debug("posix_memalign(%uz, %uz): %p", + alignment, size, p); + return p; + } + + nxt_log_moderate(&nxt_malloc_log_moderation, + NXT_LOG_ALERT, nxt_malloc_log(), + "posix_memalign(%uz, %uz) failed %E", + alignment, size, err); + return NULL; +} + +#elif (NXT_HAVE_MEMALIGN) + +/* memalign() presents in Solaris, HP-UX. */ + +void * +nxt_memalign(size_t alignment, size_t size) +{ + void *p; + + p = memalign(alignment, size); + + if (nxt_fast_path(p != NULL)) { + nxt_thread_log_debug("memalign(%uz, %uz): %p", + alignment, size, p); + return p; + } + + nxt_log_moderate(&nxt_malloc_log_moderation, + NXT_LOG_ALERT, nxt_malloc_log(), + "memalign(%uz, %uz) failed %E", + alignment, size, nxt_errno); + return NULL; +} + +#elif (NXT_FREEBSD) + +/* + * FreeBSD prior to 7.0 lacks posix_memalign(), but if a requested size + * is lesser than or equal to 4K, then phkmalloc aligns the size to the + * next highest power of 2 and allocates memory with the same alignment. + * Allocations larger than 2K are always aligned to 4K. + */ + +void * +nxt_memalign(size_t alignment, size_t size) +{ + size_t aligned_size; + u_char *p; + nxt_err_t err; + + if (nxt_slow_path((alignment - 1) & alignment) != 0) { + /* Alignment must be a power of 2. */ + err = NXT_EINVAL; + goto fail; + } + + if (nxt_slow_path(alignment > 4096)) { + err = NXT_EOPNOTSUPP; + goto fail; + } + + if (nxt_fast_path(size <= 2048)) { + aligned_size = nxt_max(size, alignment); + + } else { + /* Align to 4096. */ + aligned_size = size; + } + + p = malloc(aligned_size); + + if (nxt_fast_path(p != NULL)) { + nxt_thread_log_debug("nxt_memalign(%uz, %uz): %p", alignment, size, p); + + } else { + nxt_log_moderate(&nxt_malloc_log_moderation, + NXT_LOG_ALERT, nxt_malloc_log(), + "malloc(%uz) failed %E", size, nxt_errno); + } + + return p; + +fail: + + nxt_thread_log_alert("nxt_memalign(%uz, %uz) failed %E", + alignment, size, err); + return NULL; +} + +#else + +#error no memalign() implementation. + +#endif diff --git a/src/nxt_malloc.h b/src/nxt_malloc.h new file mode 100644 index 00000000..3b3de5f4 --- /dev/null +++ b/src/nxt_malloc.h @@ -0,0 +1,135 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_MALLOC_H_INCLUDED_ +#define _NXT_UNIX_MALLOC_H_INCLUDED_ + + +NXT_EXPORT void *nxt_malloc(size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_zalloc(size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_realloc(void *p, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_memalign(size_t alignment, size_t size) + NXT_MALLOC_LIKE; + + +#if (NXT_DEBUG) + +NXT_EXPORT void nxt_free(void *p); + +#else + +#define \ +nxt_free(p) \ + free(p) + +#endif + + +#if (NXT_HAVE_MALLOC_USABLE_SIZE) + +/* + * Due to allocation strategies malloc() allocators may allocate more + * memory than is requested, so malloc_usable_size() allows to use all + * allocated memory. It is helpful for socket buffers or unaligned disk + * file I/O. However, they may be suboptimal for aligned disk file I/O. + */ + +#if (NXT_LINUX) + +/* + * Linux glibc stores bookkeeping information together with allocated + * memory itself. Size of the bookkeeping information is 12 or 24 bytes + * on 32-bit and 64-bit platforms respectively. Due to alignment there + * are usually 4 or 8 spare bytes respectively. However, if allocation + * is larger than about 128K, spare size may be up to one page: glibc aligns + * sum of allocation and bookkeeping size to a page. So if requirement + * of the large allocation size is not strict it is better to allocate + * with small cutback and then to adjust size with malloc_usable_size(). + * Glibc malloc_usable_size() is fast operation. + */ + +#define \ +nxt_malloc_usable_size(p, size) \ + size = malloc_usable_size(p) + +#define \ +nxt_malloc_cutback(cutback, size) \ + size = ((cutback) && size > 127 * 1024) ? size - 32 : size + +#elif (NXT_FREEBSD) + +/* + * FreeBSD prior to 7.0 (phkmalloc) aligns sizes to + * 16 - 2048 a power of two + * 2049 - ... aligned to 4K + * + * FreeBSD 7.0 (jemalloc) aligns sizes to: + * 2 - 8 a power of two + * 9 - 512 aligned to 16 + * 513 - 2048 a power of two, i.e. aligned to 1K + * 2049 - 1M aligned to 4K + * 1M- ... aligned to 1M + * See table in src/lib/libc/stdlib/malloc.c + * + * FreeBSD 7.0 malloc_usable_size() is fast for allocations, which + * are lesser than 1M. Larger allocations require mutex acquiring. + */ + +#define \ +nxt_malloc_usable_size(p, size) \ + size = malloc_usable_size(p) + +#define \ +nxt_malloc_cutback(cutback, size) + +#endif + +#elif (NXT_HAVE_MALLOC_GOOD_SIZE) + +/* + * MacOSX aligns sizes to + * 16 - 496 aligned to 16, 32-bit + * 16 - 992 aligned to 16, 64-bit + * 497/993 - 15K aligned to 512, if lesser than 1G RAM + * 497/993 - 127K aligned to 512, otherwise + * 15K/127K- ... aligned to 4K + * + * malloc_good_size() is faster than malloc_size() + */ + +#define \ +nxt_malloc_usable_size(p, size) \ + size = malloc_good_size(size) + +#define \ +nxt_malloc_cutback(cutback, size) + +#else + +#define \ +nxt_malloc_usable_size(p, size) + +#define \ +nxt_malloc_cutback(cutback, size) + +#endif + + +#if (NXT_HAVE_POSIX_MEMALIGN || NXT_HAVE_MEMALIGN) +#define NXT_MAX_MEMALIGN_SHIFT 32 + +#elif (NXT_FREEBSD) +#define NXT_MAX_MEMALIGN_SHIFT 12 + +#else +#define NXT_MAX_MEMALIGN_SHIFT 3 +#endif + + +#endif /* _NXT_UNIX_MALLOC_H_INCLUDED_ */ diff --git a/src/nxt_master_process.c b/src/nxt_master_process.c new file mode 100644 index 00000000..df2934c0 --- /dev/null +++ b/src/nxt_master_process.c @@ -0,0 +1,650 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_cycle.h> +#include <nxt_process_chan.h> +#include <nxt_master_process.h> + + +static nxt_int_t nxt_master_process_chan_create(nxt_cycle_t *cycle); +static void nxt_master_process_title(void); +static nxt_int_t nxt_master_start_worker_processes(nxt_cycle_t *cycle); +static nxt_int_t nxt_master_create_worker_process(nxt_cycle_t *cycle); +static void nxt_master_stop_previous_worker_processes(nxt_thread_t *thr, + void *obj, void *data); +static void nxt_master_process_sighup_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_master_process_new_cycle(nxt_thread_t *thr, nxt_cycle_t *cycle); +static void nxt_master_process_sigterm_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_master_process_sigquit_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_master_process_sigusr1_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_master_process_sigusr2_handler(nxt_thread_t *thr, void *obj, + void *data); +static char **nxt_master_process_upgrade_environment(nxt_cycle_t *cycle); +static char **nxt_master_process_upgrade_environment_create(nxt_cycle_t *cycle); +static void nxt_master_process_sigchld_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_master_cleanup_worker_process(nxt_thread_t *thr, nxt_pid_t pid); + + +const nxt_event_sig_t nxt_master_process_signals[] = { + nxt_event_signal(SIGHUP, nxt_master_process_sighup_handler), + nxt_event_signal(SIGINT, nxt_master_process_sigterm_handler), + nxt_event_signal(SIGQUIT, nxt_master_process_sigquit_handler), + nxt_event_signal(SIGTERM, nxt_master_process_sigterm_handler), + nxt_event_signal(SIGCHLD, nxt_master_process_sigchld_handler), + nxt_event_signal(SIGUSR1, nxt_master_process_sigusr1_handler), + nxt_event_signal(SIGUSR2, nxt_master_process_sigusr2_handler), + nxt_event_signal_end, +}; + + +static nxt_bool_t nxt_exiting; + + +nxt_int_t +nxt_master_process_start(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + cycle->type = NXT_PROCESS_MASTER; + + if (nxt_master_process_chan_create(cycle) != NXT_OK) { + return NXT_ERROR; + } + + nxt_master_process_title(); + + return nxt_master_start_worker_processes(cycle); +} + + +static nxt_int_t +nxt_master_process_chan_create(nxt_cycle_t *cycle) +{ + nxt_process_chan_t *proc; + + proc = nxt_array_add(cycle->processes); + if (nxt_slow_path(proc == NULL)) { + return NXT_ERROR; + } + + proc->pid = nxt_pid; + proc->engine = 0; + + proc->chan = nxt_chan_create(0); + if (nxt_slow_path(proc->chan == NULL)) { + return NXT_ERROR; + } + + /* + * A master process chan. A write chan is not closed + * since it should be inherited by worker processes. + */ + nxt_chan_read_enable(nxt_thread(), proc->chan); + + return NXT_OK; +} + + +static void +nxt_master_process_title(void) +{ + u_char *p, *end; + nxt_uint_t i; + u_char title[2048]; + + end = title + sizeof(title); + + p = nxt_sprintf(title, end, "nginman: master process %s", + nxt_process_argv[0]); + + for (i = 1; nxt_process_argv[i] != NULL; i++) { + p = nxt_sprintf(p, end, " %s", nxt_process_argv[i]); + } + + *p = '\0'; + + nxt_process_title((char *) title); +} + + +static nxt_int_t +nxt_master_start_worker_processes(nxt_cycle_t *cycle) +{ + nxt_int_t ret; + nxt_uint_t n; + + cycle->process_generation++; + + n = cycle->worker_processes; + + while (n-- != 0) { + ret = nxt_master_create_worker_process(cycle); + + if (ret != NXT_OK) { + return ret; + } + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_master_create_worker_process(nxt_cycle_t *cycle) +{ + nxt_pid_t pid; + nxt_process_chan_t *proc; + + proc = nxt_array_add(cycle->processes); + if (nxt_slow_path(proc == NULL)) { + return NXT_ERROR; + } + + cycle->current_process = cycle->processes->nelts - 1; + + proc->engine = 0; + proc->generation = cycle->process_generation; + + proc->chan = nxt_chan_create(0); + if (nxt_slow_path(proc->chan == NULL)) { + return NXT_ERROR; + } + + pid = nxt_process_create(nxt_worker_process_start, cycle, + "start worker process"); + + switch (pid) { + + case -1: + return NXT_ERROR; + + case 0: + /* A worker process, return to the event engine work queue loop. */ + return NXT_AGAIN; + + default: + /* The master process created a new process. */ + proc->pid = pid; + + nxt_chan_read_close(proc->chan); + nxt_chan_write_enable(nxt_thread(), proc->chan); + + nxt_process_new_chan(cycle, proc); + return NXT_OK; + } +} + + +static void +nxt_master_process_sighup_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_cycle_t *cycle; + + cycle = nxt_thread_cycle(); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "signal %d (%s) recevied, %s", + (int) (uintptr_t) obj, data, + cycle->reconfiguring ? "ignored" : "reconfiguring"); + + if (!cycle->reconfiguring) { + (void) nxt_cycle_create(thr, cycle, nxt_master_process_new_cycle, + cycle->config_name, 0); + } +} + + +static void +nxt_master_process_new_cycle(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + nxt_log_debug(thr->log, "new cycle"); + + /* A safe place to free the previous cycle. */ + nxt_mem_pool_destroy(cycle->previous->mem_pool); + + switch (nxt_master_start_worker_processes(cycle)) { + + case NXT_OK: + /* + * The master process, allow old worker processes to accept new + * connections yet 500ms in parallel with new worker processes. + */ + cycle->timer.handler = nxt_master_stop_previous_worker_processes; + cycle->timer.log = &nxt_main_log; + nxt_event_timer_ident(&cycle->timer, -1); + + cycle->timer.work_queue = &thr->work_queue.main; + + nxt_event_timer_add(thr->engine, &cycle->timer, 500); + + return; + + case NXT_ERROR: + /* + * The master process, one or more new worker processes + * could not be created, there is no fallback. + */ + return; + + default: /* NXT_AGAIN */ + /* A worker process, return to the event engine work queue loop. */ + return; + } +} + + +static void +nxt_master_stop_previous_worker_processes(nxt_thread_t *thr, void *obj, + void *data) +{ + uint32_t generation; + nxt_uint_t i, n; + nxt_cycle_t *cycle; + nxt_process_chan_t *proc; + + cycle = nxt_thread_cycle(); + + proc = cycle->processes->elts; + n = cycle->processes->nelts; + + generation = cycle->process_generation - 1; + + /* The proc[0] is the master process. */ + + for (i = 1; i < n; i++) { + if (proc[i].generation == generation) { + (void) nxt_chan_write(proc[i].chan, NXT_CHAN_MSG_QUIT, -1, 0, NULL); + } + } + + cycle->reconfiguring = 0; +} + + +void +nxt_master_stop_worker_processes(nxt_cycle_t *cycle) +{ + nxt_process_chan_write(cycle, NXT_CHAN_MSG_QUIT, -1, 0, NULL); +} + + + +static void +nxt_master_process_sigterm_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_log_debug(thr->log, "sigterm handler signo:%d (%s)", + (int) (uintptr_t) obj, data); + + /* TODO: fast exit. */ + + nxt_exiting = 1; + + nxt_cycle_quit(thr, NULL); +} + + +static void +nxt_master_process_sigquit_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_log_debug(thr->log, "sigquit handler signo:%d (%s)", + (int) (uintptr_t) obj, data); + + /* TODO: graceful exit. */ + + nxt_exiting = 1; + + nxt_cycle_quit(thr, NULL); +} + + +static void +nxt_master_process_sigusr1_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_int_t ret; + nxt_uint_t n; + nxt_file_t *file, *new_file; + nxt_cycle_t *cycle; + nxt_array_t *new_files; + nxt_mem_pool_t *mp; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "signal %d (%s) recevied, %s", + (int) (uintptr_t) obj, data, "log files rotation"); + + mp = nxt_mem_pool_create(1024); + if (mp == NULL) { + return; + } + + cycle = nxt_thread_cycle(); + + n = nxt_list_nelts(cycle->log_files); + + new_files = nxt_array_create(mp, n, sizeof(nxt_file_t)); + if (new_files == NULL) { + nxt_mem_pool_destroy(mp); + return; + } + + nxt_list_each(file, cycle->log_files) { + + /* This allocation cannot fail. */ + new_file = nxt_array_add(new_files); + + new_file->name = file->name; + new_file->fd = NXT_FILE_INVALID; + new_file->log_level = NXT_LOG_CRIT; + + ret = nxt_file_open(new_file, NXT_FILE_APPEND, NXT_FILE_CREATE_OR_OPEN, + NXT_FILE_OWNER_ACCESS); + + if (ret != NXT_OK) { + goto fail; + } + + } nxt_list_loop; + + new_file = new_files->elts; + + ret = nxt_file_stderr(&new_file[0]); + + if (ret == NXT_OK) { + n = 0; + + nxt_list_each(file, cycle->log_files) { + + nxt_process_chan_change_log_file(cycle, n, new_file[n].fd); + /* + * The old log file descriptor must be closed at the moment + * when no other threads use it. dup2() allows to use the + * old file descriptor for new log file. This change is + * performed atomically in the kernel. + */ + (void) nxt_file_redirect(file, new_file[n].fd); + + n++; + + } nxt_list_loop; + + nxt_mem_pool_destroy(mp); + return; + } + +fail: + + new_file = new_files->elts; + n = new_files->nelts; + + while (n != 0) { + if (new_file->fd != NXT_FILE_INVALID) { + nxt_file_close(new_file); + } + + new_file++; + n--; + } + + nxt_mem_pool_destroy(mp); +} + + +static void +nxt_master_process_sigusr2_handler(nxt_thread_t *thr, void *obj, void *data) +{ + char **env; + nxt_int_t ret; + nxt_pid_t pid, ppid; + nxt_bool_t ignore; + nxt_cycle_t *cycle; + + cycle = nxt_thread_cycle(); + + /* Is upgrade or reconfiguring in progress? */ + ignore = (cycle->new_binary != 0) || cycle->reconfiguring; + + ppid = getppid(); + + if (ppid == nxt_ppid && ppid != 1) { + /* + * Ignore the upgrade signal in a new master process if an old + * master process is still running. After the old process's exit + * getppid() will return 1 (init process pid) or pid of zsched (zone + * scheduler) if the processes run in Solaris zone. There is little + * race condition between the parent process exit and getting getppid() + * for the very start of the new master process execution, so init or + * zsched pid may be stored in nxt_ppid. For this reason pid 1 is + * tested explicitly. There is no workaround for this race condition + * in Solaris zons. To eliminate this race condition in Solaris + * zone the old master process should be quit only when both + * "nginman.pid.oldbin" (created by the old master process) and + * "nginman.pid" (created by the new master process) files exists. + */ + ignore = 1; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "signal %d (%s) recevied, %s, parent pid: %PI", + (int) (uintptr_t) obj, data, + ignore ? "ignored" : "online binary file upgrade", ppid); + + if (ignore) { + return; + } + + env = nxt_master_process_upgrade_environment(cycle); + if (nxt_slow_path(env == NULL)) { + return; + } + + cycle->new_binary = -1; + + ret = nxt_cycle_pid_file_create(cycle->oldbin_file, 0); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + + pid = nxt_process_execute(nxt_process_argv[0], nxt_process_argv, env); + + if (pid == -1) { + cycle->new_binary = 0; + (void) nxt_file_delete(cycle->oldbin_file); + + } else { + cycle->new_binary = pid; + } + +fail: + + /* Zero slot is NGINX variable slot, all other slots must not be free()d. */ + nxt_free(env[0]); + nxt_free(env); +} + + +static char ** +nxt_master_process_upgrade_environment(nxt_cycle_t *cycle) +{ + size_t len; + char **env; + u_char *p, *end; + nxt_uint_t n; + nxt_listen_socket_t *ls; + + env = nxt_master_process_upgrade_environment_create(cycle); + if (nxt_slow_path(env == NULL)) { + return NULL; + } + + ls = cycle->listen_sockets->elts; + n = cycle->listen_sockets->nelts; + + len = sizeof("NGINX=") + n * (NXT_INT_T_LEN + 1); + + p = nxt_malloc(len); + + if (nxt_slow_path(p == NULL)) { + nxt_free(env); + return NULL; + } + + env[0] = (char *) p; + end = p + len; + + p = nxt_cpymem(p, "NGINX=", sizeof("NGINX=") - 1); + + do { + p = nxt_sprintf(p, end, "%ud;", ls->socket); + + ls++; + n--; + } while (n != 0); + + *p = '\0'; + + return env; +} + + +static char ** +nxt_master_process_upgrade_environment_create(nxt_cycle_t *cycle) +{ + char **env; + nxt_uint_t n; + + /* 2 is for "NGINX" variable and the last NULL slot. */ + n = 2; + +#if (NXT_SETPROCTITLE_ARGV) + n++; +#endif + + env = nxt_malloc(n * sizeof(char *)); + if (nxt_slow_path(env == NULL)) { + return NULL; + } + + /* Zero slot is reserved for "NGINX" variable. */ + n = 1; + + /* TODO: copy env values */ + +#if (NXT_SETPROCTITLE_ARGV) + + /* 300 spare bytes for new process title. */ + env[n++] = (char *) + "SPARE=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; + +#endif + + env[n] = NULL; + + return env; +} + + +static void +nxt_master_process_sigchld_handler(nxt_thread_t *thr, void *obj, void *data) +{ + int status; + nxt_err_t err; + nxt_pid_t pid; + + nxt_log_debug(thr->log, "sigchld handler signo:%d (%s)", + (int) (uintptr_t) obj, data); + + for ( ;; ) { + pid = waitpid(-1, &status, WNOHANG); + + if (pid == -1) { + + switch (err = nxt_errno) { + + case NXT_ECHILD: + return; + + case NXT_EINTR: + continue; + + default: + nxt_log_alert(thr->log, "waitpid() failed: %E", err); + return; + } + } + + nxt_log_debug(thr->log, "waitpid(): %PI", pid); + + if (pid == 0) { + return; + } + + if (WTERMSIG(status)) { +#ifdef WCOREDUMP + nxt_log_alert(thr->log, "process %PI exited on signal %d%s", + pid, WTERMSIG(status), + WCOREDUMP(status) ? " (core dumped)" : ""); +#else + nxt_log_alert(thr->log, "process %PI exited on signal %d", + pid, WTERMSIG(status)); +#endif + + } else { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "process %PI exited with code %d", + pid, WEXITSTATUS(status)); + } + + nxt_master_cleanup_worker_process(thr, pid); + } +} + + +static void +nxt_master_cleanup_worker_process(nxt_thread_t *thr, nxt_pid_t pid) +{ + nxt_uint_t i, n, generation; + nxt_cycle_t *cycle; + nxt_process_chan_t *proc; + + cycle = nxt_thread_cycle(); + + if (cycle->new_binary == pid) { + cycle->new_binary = 0; + + (void) nxt_file_rename(cycle->oldbin_file, cycle->pid_file); + return; + } + + proc = cycle->processes->elts; + n = cycle->processes->nelts; + + for (i = 0; i < n; i++) { + + if (pid == proc[i].pid) { + generation = proc[i].generation; + + nxt_array_remove(cycle->processes, &proc[i]); + + if (nxt_exiting) { + nxt_log_debug(thr->log, "processes %d", n); + + if (n == 2) { + nxt_cycle_quit(thr, cycle); + } + + } else if (generation == cycle->process_generation) { + (void) nxt_master_create_worker_process(cycle); + } + + return; + } + } +} diff --git a/src/nxt_master_process.h b/src/nxt_master_process.h new file mode 100644 index 00000000..8018c6b8 --- /dev/null +++ b/src/nxt_master_process.h @@ -0,0 +1,19 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_MASTER_PROCESS_H_INCLUDED_ +#define _NXT_UNIX_MASTER_PROCESS_H_INCLUDED_ + + +nxt_int_t nxt_master_process_start(nxt_thread_t *thr, nxt_cycle_t *cycle); +void nxt_master_stop_worker_processes(nxt_cycle_t *cycle); +void nxt_worker_process_start(void *data); + + +extern const nxt_event_sig_t nxt_master_process_signals[]; + + +#endif /* _NXT_UNIX_MASTER_PROCESS_H_INCLUDED_ */ diff --git a/src/nxt_mem_cache_pool.c b/src/nxt_mem_cache_pool.c new file mode 100644 index 00000000..e949dff9 --- /dev/null +++ b/src/nxt_mem_cache_pool.c @@ -0,0 +1,767 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * A memory cache pool allocates memory in clusters of specified size and + * aligned to page_alignment. A cluster is divided on pages of specified + * size. Page size must be a power of 2. A page can be used entirely or + * can be divided on chunks of equal size. Chunk size must be a power of 2. + * A cluster can contains pages with different chunk sizes. Cluster size + * must be multiple of page size and may be not a power of 2. Allocations + * greater than page are allocated outside clusters. Start addresses and + * sizes of clusters and large allocations are stored in rbtree to find + * them on free operations. The rbtree nodes are sorted by start addresses. + */ + + +typedef struct nxt_mem_cache_page_s nxt_mem_cache_page_t; + +struct nxt_mem_cache_page_s { + /* Chunk bitmap. There can be no more than 32 chunks in a page. */ + uint8_t map[4]; + + /* Number of free chunks of a chunked page. */ + uint8_t chunks; + + /* + * Size of chunks or page shifted by pool->chunk_size_shift. + * Zero means that page is free. + */ + uint8_t size; + + /* + * Page number in page cluster. + * There can be no more than 65536 pages in a cluster. + */ + uint16_t number; + + /* + * Used to link pages with free chunks in pool chunk slot list + * or to link free pages in clusters. + */ + nxt_queue_link_t link; +}; + + +typedef struct { + NXT_RBTREE_NODE (node); + uint8_t type; + uint32_t size; + + u_char *start; + nxt_mem_cache_page_t pages[]; +} nxt_mem_cache_block_t; + + +typedef struct { + nxt_queue_t pages; +#if (NXT_64BIT) + uint32_t size; + uint32_t chunks; +#else + uint16_t size; + uint16_t chunks; +#endif +} nxt_mem_cache_slot_t; + + +struct nxt_mem_cache_pool_s { + /* rbtree of nxt_mem_cache_block_t. */ + nxt_rbtree_t pages; + + nxt_queue_t free_pages; + + uint8_t chunk_size_shift; + uint8_t page_size_shift; + uint32_t page_size; + uint32_t page_alignment; + uint32_t cluster_size; + + nxt_mem_cache_slot_t slots[]; +}; + + +/* A cluster cache block. */ +#define NXT_MEM_CACHE_CLUSTER_BLOCK 0 + +/* A discrete cache block of large allocation. */ +#define NXT_MEM_CACHE_DISCRETE_BLOCK 1 +/* + * An embedded cache block allocated together with large allocation + * just after the allocation. + */ +#define NXT_MEM_CACHE_EMBEDDED_BLOCK 2 + + +#define \ +nxt_mem_cache_chunk_is_free(map, chunk) \ + ((map[chunk / 8] & (0x80 >> (chunk & 7))) == 0) + + +#define \ +nxt_mem_cache_chunk_set_free(map, chunk) \ + map[chunk / 8] &= ~(0x80 >> (chunk & 7)) + + +#define \ +nxt_mem_cache_free_junk(p, size) \ + nxt_memset((p), 0x5A, size) + + +static nxt_uint_t nxt_mem_cache_shift(nxt_uint_t n); +static void *nxt_mem_cache_alloc_small(nxt_mem_cache_pool_t *pool, size_t size); +static nxt_uint_t nxt_mem_cache_alloc_chunk(u_char *map, nxt_uint_t size); +static nxt_mem_cache_page_t * + nxt_mem_cache_alloc_page(nxt_mem_cache_pool_t *pool); +static nxt_mem_cache_block_t * + nxt_mem_cache_alloc_cluster(nxt_mem_cache_pool_t *pool); +static void *nxt_mem_cache_alloc_large(nxt_mem_cache_pool_t *pool, + size_t alignment, size_t size); +static nxt_int_t nxt_mem_cache_rbtree_compare(nxt_rbtree_node_t *node1, + nxt_rbtree_node_t *node2); +static nxt_mem_cache_block_t *nxt_mem_cache_find_block(nxt_rbtree_t *tree, + u_char *p); +static const char *nxt_mem_cache_chunk_free(nxt_mem_cache_pool_t *pool, + nxt_mem_cache_block_t *cluster, u_char *p); + + +nxt_mem_cache_pool_t * +nxt_mem_cache_pool_create(size_t cluster_size, size_t page_alignment, + size_t page_size, size_t min_chunk_size) +{ + /* Alignment and sizes must be a power of 2. */ + + if (nxt_slow_path((page_alignment & (page_alignment - 1)) != 0 + || (page_size & (page_size - 1)) != 0 + || (min_chunk_size & (min_chunk_size - 1)) != 0)) + { + return NULL; + } + + page_alignment = nxt_max(page_alignment, NXT_MAX_ALIGNMENT); + + if (nxt_slow_path(page_size < 64 + || page_size < page_alignment + || page_size < min_chunk_size + || min_chunk_size * 32 < page_size + || cluster_size < page_size + || cluster_size % page_size != 0)) + { + return NULL; + } + + return nxt_mem_cache_pool_fast_create(cluster_size, page_alignment, + page_size, min_chunk_size); +} + + +nxt_mem_cache_pool_t * +nxt_mem_cache_pool_fast_create(size_t cluster_size, size_t page_alignment, + size_t page_size, size_t min_chunk_size) +{ + nxt_uint_t slots, chunk_size; + nxt_mem_cache_slot_t *slot; + nxt_mem_cache_pool_t *pool; + + slots = 0; + chunk_size = page_size; + + do { + slots++; + chunk_size /= 2; + } while (chunk_size > min_chunk_size); + + pool = nxt_zalloc(sizeof(nxt_mem_cache_pool_t) + + slots * sizeof(nxt_mem_cache_slot_t)); + + if (nxt_fast_path(pool != NULL)) { + + pool->page_size = page_size; + pool->page_alignment = nxt_max(page_alignment, NXT_MAX_ALIGNMENT); + pool->cluster_size = cluster_size; + + slot = pool->slots; + + do { + nxt_queue_init(&slot->pages); + + slot->size = chunk_size; + /* slot->chunks should be one less than actual number of chunks. */ + slot->chunks = (page_size / chunk_size) - 1; + + slot++; + chunk_size *= 2; + } while (chunk_size < page_size); + + pool->chunk_size_shift = nxt_mem_cache_shift(min_chunk_size); + pool->page_size_shift = nxt_mem_cache_shift(page_size); + + nxt_rbtree_init(&pool->pages, nxt_mem_cache_rbtree_compare, NULL); + + nxt_queue_init(&pool->free_pages); + } + + return pool; +} + + +static nxt_uint_t +nxt_mem_cache_shift(nxt_uint_t n) +{ + nxt_uint_t shift; + + shift = 0; + n /= 2; + + do { + shift++; + n /= 2; + } while (n != 0); + + return shift; +} + + +nxt_bool_t +nxt_mem_cache_pool_is_empty(nxt_mem_cache_pool_t *pool) +{ + return (nxt_rbtree_is_empty(&pool->pages) + && nxt_queue_is_empty(&pool->free_pages)); +} + + +void +nxt_mem_cache_pool_destroy(nxt_mem_cache_pool_t *pool) +{ + void *p; + nxt_rbtree_node_t *node, *next; + nxt_mem_cache_block_t *block; + + for (node = nxt_rbtree_min(&pool->pages); + nxt_rbtree_is_there_successor(&pool->pages, node); + node = next) + { + next = nxt_rbtree_node_successor(&pool->pages, node); + + block = (nxt_mem_cache_block_t *) node; + + nxt_rbtree_delete(&pool->pages, &block->node); + + p = block->start; + + if (block->type != NXT_MEM_CACHE_EMBEDDED_BLOCK) { + nxt_free(block); + } + + nxt_free(p); + } + + nxt_free(pool); +} + + +nxt_inline u_char * +nxt_mem_cache_page_addr(nxt_mem_cache_pool_t *pool, nxt_mem_cache_page_t *page) +{ + nxt_mem_cache_block_t *block; + + block = (nxt_mem_cache_block_t *) + ((u_char *) page - page->number * sizeof(nxt_mem_cache_page_t) + - offsetof(nxt_mem_cache_block_t, pages)); + + return block->start + (page->number << pool->page_size_shift); +} + + +void * +nxt_mem_cache_alloc(nxt_mem_cache_pool_t *pool, size_t size) +{ + nxt_thread_log_debug("mem cache alloc: %uz", size); + + if (size <= pool->page_size) { + return nxt_mem_cache_alloc_small(pool, size); + } + + return nxt_mem_cache_alloc_large(pool, NXT_MAX_ALIGNMENT, size); +} + + +void * +nxt_mem_cache_zalloc(nxt_mem_cache_pool_t *pool, size_t size) +{ + void *p; + + p = nxt_mem_cache_alloc(pool, size); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, size); + } + + return p; +} + + +void * +nxt_mem_cache_align(nxt_mem_cache_pool_t *pool, size_t alignment, size_t size) +{ + nxt_thread_log_debug("mem cache align: @%uz:%uz", alignment, size); + + /* Alignment must be a power of 2. */ + + if (nxt_fast_path((alignment - 1) & alignment) == 0) { + + if (size <= pool->page_size && alignment <= pool->page_alignment) { + size = nxt_max(size, alignment); + + if (size <= pool->page_size) { + return nxt_mem_cache_alloc_small(pool, size); + } + } + + return nxt_mem_cache_alloc_large(pool, alignment, size); + } + + return NULL; +} + + +void * +nxt_mem_cache_zalign(nxt_mem_cache_pool_t *pool, size_t alignment, size_t size) +{ + void *p; + + p = nxt_mem_cache_align(pool, alignment, size); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, size); + } + + return p; +} + + +static void * +nxt_mem_cache_alloc_small(nxt_mem_cache_pool_t *pool, size_t size) +{ + u_char *p; + nxt_queue_link_t *link; + nxt_mem_cache_page_t *page; + nxt_mem_cache_slot_t *slot; + + p = NULL; + + if (size <= pool->page_size / 2) { + + /* Find a slot with appropriate chunk size. */ + for (slot = pool->slots; slot->size < size; slot++) { /* void */ } + + size = slot->size; + + if (nxt_fast_path(!nxt_queue_is_empty(&slot->pages))) { + + link = nxt_queue_first(&slot->pages); + page = nxt_queue_link_data(link, nxt_mem_cache_page_t, link); + + p = nxt_mem_cache_page_addr(pool, page); + p += nxt_mem_cache_alloc_chunk(page->map, size); + + page->chunks--; + + if (page->chunks == 0) { + /* + * Remove full page from the pool chunk slot list + * of pages with free chunks. + */ + nxt_queue_remove(&page->link); + } + + } else { + page = nxt_mem_cache_alloc_page(pool); + + if (nxt_fast_path(page != NULL)) { + + nxt_queue_insert_head(&slot->pages, &page->link); + + /* Mark the first chunk as busy. */ + page->map[0] = 0x80; + page->map[1] = 0; + page->map[2] = 0; + page->map[3] = 0; + + /* slot->chunks are already one less. */ + page->chunks = slot->chunks; + page->size = size >> pool->chunk_size_shift; + + p = nxt_mem_cache_page_addr(pool, page); + } + } + + } else { + page = nxt_mem_cache_alloc_page(pool); + + if (nxt_fast_path(page != NULL)) { + page->size = pool->page_size >> pool->chunk_size_shift; + + p = nxt_mem_cache_page_addr(pool, page); + } + +#if (NXT_DEBUG) + size = pool->page_size; +#endif + } + + nxt_thread_log_debug("mem cache chunk:%uz alloc: %p", size, p); + + return p; +} + + +static nxt_uint_t +nxt_mem_cache_alloc_chunk(uint8_t *map, nxt_uint_t size) +{ + uint8_t mask; + nxt_uint_t n, offset; + + offset = 0; + n = 0; + + /* The page must have at least one free chunk. */ + + for ( ;; ) { + if (map[n] != 0xff) { + + mask = 0x80; + + do { + if ((map[n] & mask) == 0) { + /* A free chunk is found. */ + map[n] |= mask; + return offset; + } + + offset += size; + mask >>= 1; + + } while (mask != 0); + + } else { + /* Fast-forward: all 8 chunks are occupied. */ + offset += size * 8; + } + + n++; + } +} + + +static nxt_mem_cache_page_t * +nxt_mem_cache_alloc_page(nxt_mem_cache_pool_t *pool) +{ + nxt_queue_link_t *link; + nxt_mem_cache_page_t *page; + nxt_mem_cache_block_t *cluster; + + if (nxt_queue_is_empty(&pool->free_pages)) { + cluster = nxt_mem_cache_alloc_cluster(pool); + if (nxt_slow_path(cluster == NULL)) { + return NULL; + } + } + + link = nxt_queue_first(&pool->free_pages); + nxt_queue_remove(link); + + page = nxt_queue_link_data(link, nxt_mem_cache_page_t, link); + + return page; +} + + +static nxt_mem_cache_block_t * +nxt_mem_cache_alloc_cluster(nxt_mem_cache_pool_t *pool) +{ + nxt_uint_t n; + nxt_mem_cache_block_t *cluster; + + n = pool->cluster_size >> pool->page_size_shift; + + cluster = nxt_zalloc(sizeof(nxt_mem_cache_block_t) + + n * sizeof(nxt_mem_cache_page_t)); + + if (nxt_slow_path(cluster == NULL)) { + return NULL; + } + + /* NXT_MEM_CACHE_CLUSTER_BLOCK type is zero. */ + + cluster->size = pool->cluster_size; + + cluster->start = nxt_memalign(pool->page_alignment, pool->cluster_size); + if (nxt_slow_path(cluster->start == NULL)) { + nxt_free(cluster); + return NULL; + } + + n--; + cluster->pages[n].number = n; + nxt_queue_insert_head(&pool->free_pages, &cluster->pages[n].link); + + while (n != 0) { + n--; + cluster->pages[n].number = n; + nxt_queue_insert_before(&cluster->pages[n + 1].link, + &cluster->pages[n].link); + } + + nxt_rbtree_insert(&pool->pages, &cluster->node); + + return cluster; +} + + +static void * +nxt_mem_cache_alloc_large(nxt_mem_cache_pool_t *pool, size_t alignment, + size_t size) +{ + u_char *p; + size_t aligned_size; + uint8_t type; + nxt_mem_cache_block_t *block; + + if (nxt_slow_path((size - 1) & size) != 0) { + aligned_size = nxt_align_size(size, sizeof(uintptr_t)); + + p = nxt_memalign(alignment, + aligned_size + sizeof(nxt_mem_cache_block_t)); + + if (nxt_slow_path(p == NULL)) { + return NULL; + } + + block = (nxt_mem_cache_block_t *) (p + aligned_size); + type = NXT_MEM_CACHE_EMBEDDED_BLOCK; + + } else { + block = nxt_malloc(sizeof(nxt_mem_cache_block_t)); + if (nxt_slow_path(block == NULL)) { + nxt_free(block); + return NULL; + } + + p = nxt_memalign(alignment, size); + if (nxt_slow_path(p == NULL)) { + return NULL; + } + + type = NXT_MEM_CACHE_DISCRETE_BLOCK; + } + + block->type = type; + block->size = size; + block->start = p; + + nxt_rbtree_insert(&pool->pages, &block->node); + + return p; +} + + +static nxt_int_t +nxt_mem_cache_rbtree_compare(nxt_rbtree_node_t *node1, nxt_rbtree_node_t *node2) +{ + nxt_mem_cache_block_t *block1, *block2; + + block1 = (nxt_mem_cache_block_t *) node1; + block2 = (nxt_mem_cache_block_t *) node2; + + return (uintptr_t) block1->start - (uintptr_t) block2->start; +} + + +void +nxt_mem_cache_free(nxt_mem_cache_pool_t *pool, void *p) +{ + const char *err; + nxt_mem_cache_block_t *block; + + nxt_thread_log_debug("mem cache free %p", p); + + block = nxt_mem_cache_find_block(&pool->pages, p); + + if (nxt_fast_path(block != NULL)) { + + if (block->type == NXT_MEM_CACHE_CLUSTER_BLOCK) { + err = nxt_mem_cache_chunk_free(pool, block, p); + + } else if (nxt_fast_path(p == block->start)) { + nxt_rbtree_delete(&pool->pages, &block->node); + + if (block->type == NXT_MEM_CACHE_DISCRETE_BLOCK) { + nxt_free(block); + } + + nxt_free(p); + + err = NULL; + + } else { + err = "pointer to wrong page"; + } + + } else { + err = "pointer is out of pool"; + } + + if (nxt_slow_path(err != NULL)) { + nxt_thread_log_alert("nxt_mem_cache_pool_free(%p): %s", p, err); + } +} + + +static nxt_mem_cache_block_t * +nxt_mem_cache_find_block(nxt_rbtree_t *tree, u_char *p) +{ + nxt_rbtree_node_t *node, *sentinel; + nxt_mem_cache_block_t *block; + + node = nxt_rbtree_root(tree); + sentinel = nxt_rbtree_sentinel(tree); + + while (node != sentinel) { + + block = (nxt_mem_cache_block_t *) node; + + if (p < block->start) { + node = node->left; + + } else if (p >= block->start + block->size) { + node = node->right; + + } else { + return block; + } + } + + return NULL; +} + + +static const char * +nxt_mem_cache_chunk_free(nxt_mem_cache_pool_t *pool, + nxt_mem_cache_block_t *cluster, u_char *p) +{ + u_char *start; + uintptr_t offset; + nxt_uint_t n, size, chunk; + nxt_mem_cache_page_t *page; + nxt_mem_cache_slot_t *slot; + + n = (p - cluster->start) >> pool->page_size_shift; + start = cluster->start + (n << pool->page_size_shift); + + page = &cluster->pages[n]; + + if (page->size == 0) { + return "page is already free"; + } + + size = page->size << pool->chunk_size_shift; + + if (size != pool->page_size) { + + offset = (uintptr_t) (p - start) & (pool->page_size - 1); + chunk = offset / size; + + if (nxt_slow_path(offset != chunk * size)) { + return "pointer to wrong chunk"; + } + + if (nxt_slow_path(nxt_mem_cache_chunk_is_free(page->map, chunk))) { + return "chunk is already free"; + } + + nxt_mem_cache_chunk_set_free(page->map, chunk); + + /* Find a slot with appropriate chunk size. */ + for (slot = pool->slots; slot->size < size; slot++) { /* void */ } + + if (page->chunks != slot->chunks) { + page->chunks++; + + if (page->chunks == 1) { + /* + * Add the page to the head of pool chunk slot list + * of pages with free chunks. + */ + nxt_queue_insert_head(&slot->pages, &page->link); + } + + nxt_mem_cache_free_junk(p, size); + + return NULL; + + } else { + /* + * All chunks are free, remove the page from pool chunk slot + * list of pages with free chunks. + */ + nxt_queue_remove(&page->link); + } + + } else if (nxt_slow_path(p != start)) { + return "invalid pointer to chunk"; + } + + /* Add the free page to the pool's free pages tree. */ + + page->size = 0; + nxt_queue_insert_head(&pool->free_pages, &page->link); + + nxt_mem_cache_free_junk(p, size); + + /* Test if all pages in the cluster are free. */ + + page = cluster->pages; + n = pool->cluster_size >> pool->page_size_shift; + + do { + if (page->size != 0) { + return NULL; + } + + page++; + n--; + } while (n != 0); + + /* Free cluster. */ + + page = cluster->pages; + n = pool->cluster_size >> pool->page_size_shift; + + do { + nxt_queue_remove(&page->link); + page++; + n--; + } while (n != 0); + + nxt_rbtree_delete(&pool->pages, &cluster->node); + + p = cluster->start; + + nxt_free(cluster); + nxt_free(p); + + return NULL; +} + + +const nxt_mem_proto_t nxt_mem_cache_proto = { + (nxt_mem_proto_alloc_t) nxt_mem_cache_alloc, + (nxt_mem_proto_free_t) nxt_mem_cache_free, +}; diff --git a/src/nxt_mem_cache_pool.h b/src/nxt_mem_cache_pool.h new file mode 100644 index 00000000..4df73624 --- /dev/null +++ b/src/nxt_mem_cache_pool.h @@ -0,0 +1,40 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_MEM_CACHE_POOL_H_INCLUDED_ +#define _NXT_MEM_CACHE_POOL_H_INCLUDED_ + + +typedef struct nxt_mem_cache_pool_s nxt_mem_cache_pool_t; + + +NXT_EXPORT nxt_mem_cache_pool_t *nxt_mem_cache_pool_create(size_t cluster_size, + size_t page_alignment, size_t page_size, size_t min_chunk_size) + NXT_MALLOC_LIKE; +NXT_EXPORT nxt_mem_cache_pool_t * + nxt_mem_cache_pool_fast_create(size_t cluster_size, + size_t page_alignment, size_t page_size, size_t min_chunk_size) + NXT_MALLOC_LIKE; +NXT_EXPORT nxt_bool_t nxt_mem_cache_pool_is_empty(nxt_mem_cache_pool_t *pool); +NXT_EXPORT void nxt_mem_cache_pool_destroy(nxt_mem_cache_pool_t *pool); + +NXT_EXPORT void *nxt_mem_cache_alloc(nxt_mem_cache_pool_t *pool, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_mem_cache_zalloc(nxt_mem_cache_pool_t *pool, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_mem_cache_align(nxt_mem_cache_pool_t *pool, + size_t alignment, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_mem_cache_zalign(nxt_mem_cache_pool_t *pool, + size_t alignment, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void nxt_mem_cache_free(nxt_mem_cache_pool_t *pool, void *p); + + +extern const nxt_mem_proto_t nxt_mem_cache_proto; + + +#endif /* _NXT_MEM_CACHE_POOL_H_INCLUDED_ */ diff --git a/src/nxt_mem_map.c b/src/nxt_mem_map.c new file mode 100644 index 00000000..f9caf54f --- /dev/null +++ b/src/nxt_mem_map.c @@ -0,0 +1,40 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +void * +nxt_mem_mmap(void *addr, size_t len, nxt_uint_t protection, nxt_uint_t flags, + nxt_fd_t fd, nxt_off_t offset) +{ + void *p; + + p = mmap(addr, len, protection, flags, fd, offset); + + if (nxt_fast_path(p != MAP_FAILED)) { + nxt_thread_log_debug("mmap(%p, %uz, %uxi, %uxi, %FD, %O): %p", + addr, len, protection, flags, fd, offset, p); + + } else { + nxt_thread_log_alert("mmap(%p, %uz, %ui, %ui, %FD, %O) failed %E", + addr, len, protection, flags, fd, offset, nxt_errno); + } + + return p; +} + + +void +nxt_mem_munmap(void *addr, size_t len) +{ + if (nxt_fast_path(munmap(addr, len) == 0)) { + nxt_thread_log_debug("munmap(%p, %uz)", addr, len); + + } else { + nxt_thread_log_alert("munmap(%p, %uz) failed %E", addr, len, nxt_errno); + } +} diff --git a/src/nxt_mem_map.h b/src/nxt_mem_map.h new file mode 100644 index 00000000..a4a10cc4 --- /dev/null +++ b/src/nxt_mem_map.h @@ -0,0 +1,65 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_MEM_MAP_H_INCLUDED_ +#define _NXT_UNIX_MEM_MAP_H_INCLUDED_ + + +#define NXT_MEM_MAP_FAILED MAP_FAILED + + +#define NXT_MEM_MAP_READ PROT_READ +#define NXT_MEM_MAP_WRITE PROT_WRITE + + +#if (NXT_HAVE_MAP_ANONYMOUS) +#define NXT_MEM_MAP_ANON MAP_ANONYMOUS +#else +#define NXT_MEM_MAP_ANON MAP_ANON +#endif + +#define NXT_MEM_MAP_SHARED (MAP_SHARED | NXT_MEM_MAP_ANON) + + +#if (NXT_HAVE_MAP_POPULATE) +/* + * Linux MAP_POPULATE reads ahead and wires pages. + * (MAP_POPULATE | MAP_NONBLOCK) wires only resident pages + * without read ahead but it does not work since Linux 2.6.23. + */ +#define NXT_MEM_MAP_PREFAULT MAP_POPULATE + +#elif (NXT_HAVE_MAP_PREFAULT_READ) +/* FreeBSD MAP_PREFAULT_READ wires resident pages without read ahead. */ +#define NXT_MEM_MAP_PREFAULT MAP_PREFAULT_READ + +#else +#define NXT_MEM_MAP_PREFAULT 0 +#endif + +#define NXT_MEM_MAP_FILE (MAP_SHARED | NXT_MEM_MAP_PREFAULT) + + +#define \ + nxt_mem_map_file_ctx_t(ctx) + + +#define \ +nxt_mem_map(addr, ctx, len, protection, flags, fd, offset) \ + nxt_mem_mmap(addr, len, protection, flags, fd, offset) + + +#define \ +nxt_mem_unmap(addr, ctx, len) \ + nxt_mem_munmap(addr, len) + + +NXT_EXPORT void *nxt_mem_mmap(void *addr, size_t len, nxt_uint_t protection, + nxt_uint_t flags, nxt_fd_t fd, nxt_off_t offset); +NXT_EXPORT void nxt_mem_munmap(void *addr, size_t len); + + +#endif /* _NXT_UNIX_MEM_MAP_H_INCLUDED_ */ diff --git a/src/nxt_mem_pool.c b/src/nxt_mem_pool.c new file mode 100644 index 00000000..0d4b5688 --- /dev/null +++ b/src/nxt_mem_pool.c @@ -0,0 +1,641 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * The pool allocator provides cheap allocation of small objects. + * The objects are allocated from larger preallocated chunks. + * + * aligned and non-aligned allocations, + * cache of reusable objects, lvlhsh-specific cache + * eliminating align padding + * data locality + * freeing on pool destruction + * freeing large allocations + */ + + +static void *nxt_mem_pool_align(nxt_mem_pool_t *mp, size_t alignment, + size_t size); +static void *nxt_mem_pool_ext(nxt_mem_pool_t *mp, size_t size); +static nxt_mem_pool_chunk_t *nxt_mem_pool_next_chunk(nxt_mem_pool_t *mp, + nxt_mem_pool_chunk_t *chunk); +static nxt_mem_pool_chunk_t *nxt_mem_pool_chunk(nxt_mem_pool_t *mp); +static void *nxt_mem_lvlhsh_alloc_chunk(nxt_mem_pool_cache_t *cache, + size_t size, nxt_uint_t nalloc); + + +#if (NXT_DEBUG) + +static nxt_bool_t +nxt_mem_pool_thread_is_invalid(nxt_mem_pool_t *mp) +{ + nxt_tid_t tid; + nxt_thread_t *thr; + + thr = nxt_thread(); + tid = nxt_thread_tid(thr); + + if (nxt_slow_path(mp->tid != tid)) { + + if (mp->pid == nxt_pid) { + nxt_log_alert(thr->log, "mem_pool locked by thread %PT", mp->tid); + nxt_abort(); + return 1; + } + + mp->pid = nxt_pid; + mp->tid = tid; + } + + return 0; +} + + +/* SunC does not support C99 variadic macro with empty __VA_ARGS__. */ + +#define \ +nxt_mem_pool_thread_assert(mp) \ + if (nxt_mem_pool_thread_is_invalid(mp)) \ + return + + +#define \ +nxt_mem_pool_thread_assert_return(mp, ret) \ + if (nxt_mem_pool_thread_is_invalid(mp)) \ + return ret + + +#else /* !(NXT_DEBUG) */ + +#define \ +nxt_mem_pool_thread_assert(mp) + +#define \ +nxt_mem_pool_thread_assert_return(mp, ret) + +#endif + + +nxt_mem_pool_t * +nxt_mem_pool_create(size_t size) +{ + u_char *p; + size_t min_ext_size; + nxt_mem_pool_t *mp; + + mp = nxt_malloc(size); + + if (nxt_fast_path(mp != NULL)) { + + mp->chunk_size = (uint32_t) size; + + min_ext_size = size - sizeof(nxt_mem_pool_t) + 1; + mp->min_ext_size = (uint32_t) nxt_min(min_ext_size, + NXT_MEM_POOL_MIN_EXT_SIZE); + + nxt_malloc_usable_size(mp, size); + + p = (u_char *) mp; + + mp->chunk.free = p + sizeof(nxt_mem_pool_t); + mp->chunk.end = p + size; + mp->chunk.next = NULL; + mp->chunk.fails = 0; + + mp->current = &mp->chunk; + mp->ext = NULL; + mp->cleanup = NULL; + mp->cache = NULL; + + nxt_thread_log_debug("mem pool chunk size:%uz avail:%uz", + size, mp->chunk.end - mp->chunk.free); + + nxt_mem_pool_debug_lock(mp, nxt_thread_tid(NULL)); + } + + return mp; +} + + +void +nxt_mem_pool_destroy(nxt_mem_pool_t *mp) +{ + nxt_mem_pool_ext_t *ext; + nxt_mem_pool_chunk_t *chunk, *next; + nxt_mem_pool_cleanup_t *mpcl; + + nxt_mem_pool_thread_assert(mp); + + for (mpcl = mp->cleanup; mpcl != NULL; mpcl = mpcl->next) { + if (mpcl->handler != NULL) { + nxt_thread_log_debug("mem pool cleanup: %p", mpcl); + mpcl->handler(mpcl->data); + } + } + + for (ext = mp->ext; ext != NULL; ext = ext->next) { + if (ext->data != NULL) { + nxt_free(ext->data); + } + } + + chunk = &mp->chunk; + + do { + nxt_thread_log_debug("mem pool chunk fails:%uD unused:%uz", + chunk->fails, chunk->end - chunk->free); + next = chunk->next; + nxt_free(chunk); + chunk = next; + + } while (chunk != NULL); +} + + +void * +nxt_mem_align(nxt_mem_pool_t *mp, size_t alignment, size_t size) +{ + nxt_mem_pool_thread_assert_return(mp, NULL); + + if (nxt_fast_path(size < mp->min_ext_size)) { + return nxt_mem_pool_align(mp, alignment, size); + } + + return nxt_mem_pool_ext(mp, size); +} + + +void * +nxt_mem_zalign(nxt_mem_pool_t *mp, size_t alignment, size_t size) +{ + void *p; + + p = nxt_mem_align(mp, alignment, size); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, size); + } + + return p; +} + + +/* + * Zero-filled aligned allocation, suitable for struct + * allocation without long double and SIMD values. + */ + +void * +nxt_mem_zalloc(nxt_mem_pool_t *mp, size_t size) +{ + void *p; + + p = nxt_mem_alloc(mp, size); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, size); + } + + return p; +} + + +void * +nxt_mem_buf(nxt_mem_pool_t *mp, size_t *sizep, nxt_uint_t flags) +{ + u_char *p; + size_t size; + + nxt_mem_pool_thread_assert_return(mp, NULL); + + size = *sizep; + + if (nxt_fast_path(size >= mp->min_ext_size)) { + + nxt_malloc_cutback(flags & NXT_MEM_BUF_CUTBACK, size); + + /* Windows only: try to minimize number of allocated pages. */ + p = nxt_mem_pool_ext(mp, size); + if (p != NULL) { + + if (flags & NXT_MEM_BUF_USABLE) { + nxt_malloc_usable_size(p, size); + } + + *sizep = size; + } + + return p; + } + + return nxt_mem_pool_align(mp, NXT_ALIGNMENT, size); +} + + +/* Non-aligned allocation, suitable for string allocation. */ + +void * +nxt_mem_nalloc(nxt_mem_pool_t *mp, size_t size) +{ + u_char *p; + nxt_mem_pool_chunk_t *chunk; + + nxt_mem_pool_thread_assert_return(mp, NULL); + + if (nxt_slow_path(size >= mp->min_ext_size)) { + return nxt_mem_pool_ext(mp, size); + } + + chunk = mp->current; + + for ( ;; ) { + p = chunk->end - size; + + if (nxt_fast_path(p >= chunk->free)) { + chunk->end = p; + return p; + } + + chunk = nxt_mem_pool_next_chunk(mp, chunk); + + if (nxt_slow_path(chunk == NULL)) { + return NULL; + } + } +} + + +/* An attempt to deallocate a large allocation outside pool. */ + +nxt_int_t +nxt_mem_free(nxt_mem_pool_t *mp, void *p) +{ + nxt_mem_pool_ext_t *ext; + + nxt_mem_pool_thread_assert_return(mp, NXT_DECLINED); + + for (ext = mp->ext; ext != NULL; ext = ext->next) { + + if (p == ext->data) { + nxt_free(ext->data); + ext->data = NULL; + + return NXT_OK; + } + } + + return NXT_DECLINED; +} + + +static void * +nxt_mem_pool_ext(nxt_mem_pool_t *mp, size_t size) +{ + void *p; + nxt_mem_pool_ext_t *ext; + + ext = nxt_mem_pool_align(mp, sizeof(void *), sizeof(nxt_mem_pool_ext_t)); + + if (nxt_fast_path(ext != NULL)) { + p = nxt_malloc(size); + + if (nxt_fast_path(p != NULL)) { + ext->data = p; + ext->next = mp->ext; + mp->ext = ext; + + return p; + } + } + + return NULL; +} + + +static void * +nxt_mem_pool_align(nxt_mem_pool_t *mp, size_t alignment, size_t size) +{ + u_char *p, *f; + nxt_mem_pool_chunk_t *chunk; + + chunk = mp->current; + + for ( ;; ) { + + p = nxt_align_ptr(chunk->free, alignment); + f = p + size; + + if (nxt_fast_path(f <= chunk->end)) { + chunk->free = f; + return p; + } + + chunk = nxt_mem_pool_next_chunk(mp, chunk); + + if (nxt_slow_path(chunk == NULL)) { + return NULL; + } + } +} + + +static nxt_mem_pool_chunk_t * +nxt_mem_pool_next_chunk(nxt_mem_pool_t *mp, nxt_mem_pool_chunk_t *chunk) +{ + nxt_bool_t full; + + full = (chunk->free == chunk->end || chunk->fails++ > 10); + + chunk = chunk->next; + + if (chunk == NULL) { + chunk = nxt_mem_pool_chunk(mp); + + if (nxt_slow_path(chunk == NULL)) { + return NULL; + } + } + + if (full) { + mp->current = chunk; + } + + return chunk; +} + + +static nxt_mem_pool_chunk_t * +nxt_mem_pool_chunk(nxt_mem_pool_t *mp) +{ + u_char *p; + size_t size; + nxt_mem_pool_chunk_t *ch, *chunk; + + size = mp->chunk_size; + + chunk = nxt_malloc(size); + + if (nxt_fast_path(chunk != NULL)) { + + nxt_malloc_usable_size(chunk, size); + + p = (u_char *) chunk; + + chunk->free = p + sizeof(nxt_mem_pool_chunk_t); + chunk->end = p + size; + chunk->next = NULL; + chunk->fails = 0; + + for (ch = mp->current; ch->next; ch = ch->next) { /* void */ } + + ch->next = chunk; + } + + return chunk; +} + + +nxt_mem_pool_cleanup_t * +nxt_mem_pool_cleanup(nxt_mem_pool_t *mp, size_t size) +{ + nxt_mem_pool_cleanup_t *mpcl; + + nxt_mem_pool_thread_assert_return(mp, NULL); + + mpcl = nxt_mem_pool_align(mp, sizeof(void *), + sizeof(nxt_mem_pool_cleanup_t)); + if (nxt_fast_path(mpcl != NULL)) { + + mpcl->handler = NULL; + mpcl->data = NULL; + + if (size != 0) { + mpcl->data = nxt_mem_alloc(mp, size); + if (nxt_slow_path(mpcl->data == NULL)) { + return NULL; + } + } + + mpcl->next = mp->cleanup; + mp->cleanup = mpcl; + + nxt_thread_log_debug("mem pool cleanup add: %p", mpcl); + } + + return mpcl; +} + + +/* Allocation of reusable object with specified size. */ + +void * +nxt_mem_cache_alloc0(nxt_mem_pool_t *mp, size_t size) +{ + void **pp; + nxt_mem_pool_cache_t *cache; + + nxt_mem_pool_thread_assert_return(mp, NULL); + + for (cache = mp->cache; cache != NULL; cache = cache->next) { + + if (cache->size == size && cache->nalloc == 0) { + + if (cache->free != NULL) { + pp = cache->free; + cache->free = *pp; + return pp; + } + + break; + } + } + + return nxt_mem_alloc(mp, size); +} + + +void * +nxt_mem_cache_zalloc0(nxt_mem_pool_t *mp, size_t size) +{ + void *p; + + p = nxt_mem_cache_alloc0(mp, size); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, size); + } + + return p; +} + + +/* Deallocation of reusable object with specified size. */ + +void +nxt_mem_cache_free0(nxt_mem_pool_t *mp, void *p, size_t size) +{ + void **pp; + nxt_mem_pool_cache_t *cache, **pcache; + + nxt_mem_pool_thread_assert(mp); + + pp = p; + + pcache = &mp->cache; + for (cache = mp->cache; cache != NULL; cache = cache->next) { + + if (cache->size == size && cache->nalloc == 0) { + *pp = cache->free; + cache->free = p; + return; + } + + pcache = &cache->next; + } + + /* Non-lvlhash caches are created only on return. */ + + cache = nxt_mem_pool_align(mp, sizeof(void *), + sizeof(nxt_mem_pool_cache_t)); + if (nxt_fast_path(cache != NULL)) { + *pp = NULL; + cache->size = (uint32_t) size; + cache->nalloc = 0; + cache->free = p; + cache->next = NULL; + *pcache = cache; + } +} + + +/* + * lvlhsh requires allocations aligned to a size of the allocations. + * This is not issue for slab-like allocators, but glibc allocator may + * waste memory on such aligned allocations. So nxt_mem_lvlhsh_alloc() + * allocates memory in chunks specified by the "nalloc" parameter + * except the first allocation. The first lvlhsh allocation is a bucket + * allocation and it is enough for small hashes and for early stage + * of a hash. By default lvlhsh uses 128-bytes buckets and levels. + * This allows to search up to 10 entries in one memory access and + * up to 160 entries in two memory accesses on 64-bit platform. + * And on 32-bit platform up to 15 entries and up to 480 entries + * respectively. + * + * After the bucket will be filled up with 10 64-bit entries or 15 + * 32-bit entries, lvlhsh will expand it to a level and content + * of the first bucket will spread to the level's new buckets. + * Number of the new buckets may be up to 11 on 64-bit or 16 on 32-bit + * platforms. It's better to allocate them together to eliminate + * wasting memory and CPU time. + * + * The "nalloc" should be 16 if bucket size is 128 bytes. + */ + + +/* Allocation of lvlhsh level or bucket with specified size. */ + +void * +nxt_mem_lvlhsh_alloc(void *ctx, size_t size, nxt_uint_t nalloc) +{ + void *p, **pp; + nxt_mem_pool_t *mp; + nxt_mem_pool_cache_t *cache; + + mp = ctx; + + nxt_mem_pool_thread_assert_return(mp, NULL); + + for (cache = mp->cache; cache != NULL; cache = cache->next) { + + if (cache->size == size && cache->nalloc != 0) { + + if (cache->free != NULL) { + pp = cache->free; + cache->free = *pp; + return pp; + } + + return nxt_mem_lvlhsh_alloc_chunk(cache, size, nalloc); + } + } + + cache = nxt_mem_pool_align(mp, sizeof(void *), + sizeof(nxt_mem_pool_cache_t)); + if (nxt_fast_path(cache != NULL)) { + + p = nxt_memalign(size, size); + + if (nxt_fast_path(p != NULL)) { + cache->size = (uint32_t) size; + cache->nalloc = nalloc; + cache->free = NULL; + cache->next = mp->cache; + mp->cache = cache; + return p; + } + } + + return NULL; +} + + +static void * +nxt_mem_lvlhsh_alloc_chunk(nxt_mem_pool_cache_t *cache, size_t size, + nxt_uint_t nalloc) +{ + char *m, *p, *end; + void **pp; + size_t n; + + n = (nalloc == 0) ? 1 : nalloc; + n *= size; + + m = nxt_memalign(size, n); + + if (nxt_fast_path(m != NULL)) { + + pp = &cache->free; + end = m + n; + + for (p = m + size; p < end; p = p + size) { + *pp = p; + pp = (void **) p; + } + + *pp = NULL; + } + + return m; +} + + +/* Deallocation of lvlhsh level or bucket with specified size. */ + +void +nxt_mem_lvlhsh_free(void *ctx, void *p, size_t size) +{ + void **pp; + nxt_mem_pool_t *mp; + nxt_mem_pool_cache_t *cache; + + mp = ctx; + + nxt_mem_pool_thread_assert(mp); + + pp = p; + + for (cache = mp->cache; cache != NULL; cache = cache->next) { + + if (cache->size == size && cache->nalloc != 0) { + *pp = cache->free; + cache->free = p; + return; + } + } +} diff --git a/src/nxt_mem_pool.h b/src/nxt_mem_pool.h new file mode 100644 index 00000000..f59e29e1 --- /dev/null +++ b/src/nxt_mem_pool.h @@ -0,0 +1,150 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_MEM_POOL_H_INCLUDED_ +#define _NXT_MEM_POOL_H_INCLUDED_ + + +#define NXT_MEM_POOL_MIN_EXT_SIZE nxt_pagesize + + +typedef void (*nxt_mem_pool_cleanup_handler_t)(void *data); +typedef struct nxt_mem_pool_cleanup_s nxt_mem_pool_cleanup_t; +typedef struct nxt_mem_pool_cache_s nxt_mem_pool_cache_t; +typedef struct nxt_mem_pool_chunk_s nxt_mem_pool_chunk_t; +typedef struct nxt_mem_pool_ext_s nxt_mem_pool_ext_t; + + +struct nxt_mem_pool_cleanup_s { + nxt_mem_pool_cleanup_handler_t handler; + void *data; + nxt_mem_pool_cleanup_t *next; +}; + + +struct nxt_mem_pool_ext_s { + void *data; + nxt_mem_pool_ext_t *next; +}; + + +struct nxt_mem_pool_chunk_s { + u_char *free; + u_char *end; + nxt_mem_pool_chunk_t *next; + uint32_t fails; /* 8 bits */ +}; + + +struct nxt_mem_pool_cache_s { + uint32_t size; + uint32_t nalloc; + void *free; + nxt_mem_pool_cache_t *next; +}; + + +struct nxt_mem_pool_s { + nxt_mem_pool_chunk_t chunk; + uint32_t min_ext_size; + uint32_t chunk_size; + nxt_mem_pool_chunk_t *current; + nxt_mem_pool_ext_t *ext; + nxt_mem_pool_cache_t *cache; + nxt_mem_pool_cleanup_t *cleanup; + +#if (NXT_DEBUG) + nxt_pid_t pid; + nxt_tid_t tid; +#endif +}; + + +NXT_EXPORT nxt_mem_pool_t *nxt_mem_pool_create(size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void nxt_mem_pool_destroy(nxt_mem_pool_t *mp); + + +/* + * Generic aligned allocation, suitable for struct allocations + * without "long double" and SIMD values. + */ +#define \ +nxt_mem_alloc(mp, size) \ + nxt_mem_align((mp), NXT_ALIGNMENT, (size)) + + +NXT_EXPORT void *nxt_mem_align(nxt_mem_pool_t *mp, size_t alignment, + size_t size) + NXT_MALLOC_LIKE; + +NXT_EXPORT void *nxt_mem_zalign(nxt_mem_pool_t *mp, size_t alignment, + size_t size) + NXT_MALLOC_LIKE; + +NXT_EXPORT void *nxt_mem_nalloc(nxt_mem_pool_t *mp, size_t size) + NXT_MALLOC_LIKE; + +NXT_EXPORT void *nxt_mem_zalloc(nxt_mem_pool_t *mp, size_t size) + NXT_MALLOC_LIKE; + + +/* + * nxt_mem_buf() is intended to allocate I/O buffers. + * Unix network buffers usually have no size restrictions, so + * NXT_MEM_BUF_CUTBACK and NXT_MEM_BUF_USABLE options allow to + * utilize better allocated memory (details in unix/nxt_malloc.h). + * Windows locks buffers in kernel memory on page basis for both + * network and file operations, so nxt_mem_buf() should minimize + * number of allocated pages. However, these allocations are not + * necessary page-aligned. + */ +#define NXT_MEM_BUF_CUTBACK 1 +#define NXT_MEM_BUF_USABLE 2 + +NXT_EXPORT void *nxt_mem_buf(nxt_mem_pool_t *mp, size_t *sizep, + nxt_uint_t flags); + + +/* + * Aligned allocation, suitable for generic allocations compatible + * with malloc() alignment. + */ +#define \ +nxt_mem_malloc(mp, size) \ + nxt_mem_align((mp), NXT_MAX_ALIGNMENT, (size)) + + +NXT_EXPORT nxt_int_t nxt_mem_free(nxt_mem_pool_t *mp, void *p); +NXT_EXPORT nxt_mem_pool_cleanup_t *nxt_mem_pool_cleanup(nxt_mem_pool_t *mp, + size_t size); + +NXT_EXPORT void *nxt_mem_cache_alloc0(nxt_mem_pool_t *mp, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_mem_cache_zalloc0(nxt_mem_pool_t *mp, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void nxt_mem_cache_free0(nxt_mem_pool_t *mp, void *p, size_t size); + +NXT_EXPORT void *nxt_mem_lvlhsh_alloc(void *ctx, size_t size, + nxt_uint_t nalloc); +NXT_EXPORT void nxt_mem_lvlhsh_free(void *ctx, void *p, size_t size); + + +#if (NXT_DEBUG) + +#define \ +nxt_mem_pool_debug_lock(_mp, _tid) \ + (_mp->tid) = _tid + +#else + +#define \ +nxt_mem_pool_debug_lock(_mp, _tid) + +#endif + + +#endif /* _NXT_MEM_POOL_H_INCLUDED_ */ diff --git a/src/nxt_mem_pool_cleanup.c b/src/nxt_mem_pool_cleanup.c new file mode 100644 index 00000000..febfc959 --- /dev/null +++ b/src/nxt_mem_pool_cleanup.c @@ -0,0 +1,39 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_mem_pool_file_cleanup_handler(void *data); + + +nxt_mem_pool_cleanup_t * +nxt_mem_pool_file_cleanup(nxt_mem_pool_t *mp, nxt_file_t *file) +{ + nxt_mem_pool_cleanup_t *mpcl; + + mpcl = nxt_mem_pool_cleanup(mp, 0); + + if (nxt_fast_path(mpcl != NULL)) { + mpcl->handler = nxt_mem_pool_file_cleanup_handler; + mpcl->data = file; + } + + return mpcl; +} + + +static void +nxt_mem_pool_file_cleanup_handler(void *data) +{ + nxt_file_t *file; + + file = data; + + if (file->fd != NXT_FILE_INVALID) { + nxt_file_close(file); + } +} diff --git a/src/nxt_mem_pool_cleanup.h b/src/nxt_mem_pool_cleanup.h new file mode 100644 index 00000000..f84395d0 --- /dev/null +++ b/src/nxt_mem_pool_cleanup.h @@ -0,0 +1,15 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_MEM_POOL_CLEANUP_H_INCLUDED_ +#define _NXT_MEM_POOL_CLEANUP_H_INCLUDED_ + + +NXT_EXPORT nxt_mem_pool_cleanup_t *nxt_mem_pool_file_cleanup(nxt_mem_pool_t *mp, + nxt_file_t *file); + + +#endif /* _NXT_MEM_POOL_CLEANUP_H_INCLUDED_ */ diff --git a/src/nxt_mem_zone.c b/src/nxt_mem_zone.c new file mode 100644 index 00000000..02dd4328 --- /dev/null +++ b/src/nxt_mem_zone.c @@ -0,0 +1,958 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#define NXT_MEM_ZONE_PAGE_FREE 0 +/* + * A page was never allocated before so it should be filled with + * junk on the first time allocation if memory debugging is enabled. + */ +#define NXT_MEM_ZONE_PAGE_FRESH 1 + +/* An entire page is currently used, no chunks inside the page. */ +#define NXT_MEM_ZONE_PAGE_USED 2 + + +typedef struct nxt_mem_zone_page_s nxt_mem_zone_page_t; + +struct nxt_mem_zone_page_s { + /* + * A size of page chunks if value is greater than or equal to 16. + * Otherwise it is used to mark page state: NXT_MEM_ZONE_PAGE_FREE, + * NXT_MEM_ZONE_PAGE_FRESH, and NXT_MEM_ZONE_PAGE_USED. + */ + uint16_t size; + + /* A number of free chunks of a chunked page. */ + uint16_t chunks; + + union { + /* A chunk bitmap if a number of chunks is lesser than 32. */ + uint8_t map[4]; + /* + * The count is a number of successive occupied pages in the first + * page. In the next occupied pages and in all free pages the count + * is zero, because a number of successive free pages is stored in + * free block size resided in beginning of the first free page. + */ + uint32_t count; + } u; + + /* Used for slot list of pages with free chunks. */ + nxt_mem_zone_page_t *next; + + /* + * Used to link of all pages including free, chunked and occupied + * pages to coalesce free pages. + */ + nxt_queue_link_t link; +}; + + +typedef struct { + uint32_t size; + uint32_t chunks; + uint32_t start; + uint32_t map_size; + nxt_mem_zone_page_t *pages; +} nxt_mem_zone_slot_t; + + +typedef struct { + NXT_RBTREE_NODE (node); + uint32_t size; +} nxt_mem_zone_free_block_t; + + +struct nxt_mem_zone_s { + nxt_thread_spinlock_t lock; + nxt_mem_zone_page_t *pages; + nxt_mem_zone_page_t sentinel_page; + nxt_rbtree_t free_pages; + + uint32_t page_size_shift; + uint32_t page_size_mask; + uint32_t max_chunk_size; + uint32_t small_bitmap_min_size; + + u_char *start; + u_char *end; + + nxt_mem_zone_slot_t slots[]; +}; + + +#define \ +nxt_mem_zone_page_addr(zone, page) \ + (void *) (zone->start + ((page - zone->pages) << zone->page_size_shift)) + + +#define \ +nxt_mem_zone_addr_page(zone, addr) \ + &zone->pages[((u_char *) addr - zone->start) >> zone->page_size_shift] + + +#define \ +nxt_mem_zone_page_is_free(page) \ + (page->size < NXT_MEM_ZONE_PAGE_USED) + + +#define \ +nxt_mem_zone_page_is_chunked(page) \ + (page->size >= 16) + + +#define \ +nxt_mem_zone_page_bitmap(zone, slot) \ + (slot->size < zone->small_bitmap_min_size) + + +#define \ +nxt_mem_zone_set_chunk_free(map, chunk) \ + map[chunk / 8] &= ~(0x80 >> (chunk & 7)) + + +#define \ +nxt_mem_zone_chunk_is_free(map, chunk) \ + ((map[chunk / 8] & (0x80 >> (chunk & 7))) == 0) + + +#define \ +nxt_mem_zone_fresh_junk(p, size) \ + nxt_memset((p), 0xA5, size) + + +#define \ +nxt_mem_zone_free_junk(p, size) \ + nxt_memset((p), 0x5A, size) + + +static uint32_t nxt_mem_zone_pages(u_char *start, size_t zone_size, + nxt_uint_t page_size); +static void *nxt_mem_zone_slots_init(nxt_mem_zone_t *zone, + nxt_uint_t page_size); +static void nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot, + nxt_uint_t page_size); +static nxt_int_t nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1, + nxt_rbtree_node_t *node2); +static void *nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone, + nxt_mem_zone_slot_t *slot, size_t size); +static nxt_uint_t nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset, + nxt_uint_t size); +static void *nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment, + size_t size); +static nxt_mem_zone_page_t *nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, + size_t alignment, uint32_t pages); +static nxt_mem_zone_free_block_t * + nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node, + uint32_t alignment, uint32_t pages); +static const char *nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone, + nxt_mem_zone_page_t *page, void *p); +static void nxt_mem_zone_free_pages(nxt_mem_zone_t *zone, + nxt_mem_zone_page_t *page, nxt_uint_t count); + + +static nxt_log_moderation_t nxt_mem_zone_log_moderation = { + NXT_LOG_ALERT, 2, "mem_zone_alloc() failed, not enough memory", + NXT_LOG_MODERATION +}; + + +nxt_mem_zone_t * +nxt_mem_zone_init(u_char *start, size_t zone_size, nxt_uint_t page_size) +{ + uint32_t pages; + nxt_uint_t n; + nxt_mem_zone_t *zone; + nxt_mem_zone_page_t *page; + nxt_mem_zone_free_block_t *block; + + if (nxt_slow_path((page_size & (page_size - 1)) != 0)) { + nxt_thread_log_alert("mem zone page size must be a power of 2"); + return NULL; + } + + pages = nxt_mem_zone_pages(start, zone_size, page_size); + if (pages == 0) { + return NULL; + } + + zone = (nxt_mem_zone_t *) start; + + /* The function returns address after all slots. */ + page = nxt_mem_zone_slots_init(zone, page_size); + + zone->pages = page; + + for (n = 0; n < pages; n++) { + page[n].size = NXT_MEM_ZONE_PAGE_FRESH; + } + + /* + * A special sentinel page entry marked as used does not correspond + * to a real page. The entry simplifies neighbour queue nodes check + * in nxt_mem_zone_free_pages(). + */ + zone->sentinel_page.size = NXT_MEM_ZONE_PAGE_USED; + nxt_queue_sentinel(&zone->sentinel_page.link); + nxt_queue_insert_after(&zone->sentinel_page.link, &page->link); + + /* rbtree of free pages. */ + + nxt_rbtree_init(&zone->free_pages, nxt_mem_zone_rbtree_compare, NULL); + + block = (nxt_mem_zone_free_block_t *) zone->start; + block->size = pages; + + nxt_rbtree_insert(&zone->free_pages, &block->node); + + return zone; +} + + +static uint32_t +nxt_mem_zone_pages(u_char *start, size_t zone_size, nxt_uint_t page_size) +{ + u_char *end; + size_t reserved; + nxt_uint_t n, pages, size, chunks, last; + nxt_mem_zone_t *zone; + + /* + * Find all maximum chunk sizes which zone page can be split on + * with minimum 16-byte step. + */ + last = page_size / 16; + n = 0; + size = 32; + + do { + chunks = page_size / size; + + if (last != chunks) { + last = chunks; + n++; + } + + size += 16; + + } while (chunks > 1); + + /* + * Find number of usable zone pages except zone bookkeeping data, + * slots, and pages entries. + */ + reserved = sizeof(nxt_mem_zone_t) + (n * sizeof(nxt_mem_zone_slot_t)); + + end = nxt_trunc_ptr(start + zone_size, page_size); + zone_size = end - start; + + pages = (zone_size - reserved) / (page_size + sizeof(nxt_mem_zone_page_t)); + + if (reserved > zone_size || pages == 0) { + nxt_thread_log_alert("mem zone size is too small: %uz", zone_size); + return 0; + } + + reserved += pages * sizeof(nxt_mem_zone_page_t); + nxt_memzero(start, reserved); + + zone = (nxt_mem_zone_t *) start; + + zone->start = nxt_align_ptr(start + reserved, page_size); + zone->end = end; + + nxt_thread_log_debug("mem zone pages: %uD, unused:%z", pages, + end - (zone->start + pages * page_size)); + + /* + * If a chunk size is lesser than zone->small_bitmap_min_size + * bytes, a page's chunk bitmap is larger than 32 bits and the + * bimap is placed at the start of the page. + */ + zone->small_bitmap_min_size = page_size / 32; + + zone->page_size_mask = page_size - 1; + zone->max_chunk_size = page_size / 2; + + n = zone->max_chunk_size; + + do { + zone->page_size_shift++; + n /= 2; + } while (n != 0); + + return (uint32_t) pages; +} + + +static void * +nxt_mem_zone_slots_init(nxt_mem_zone_t *zone, nxt_uint_t page_size) +{ + nxt_uint_t n, size, chunks; + nxt_mem_zone_slot_t *slot; + + slot = zone->slots; + + slot[0].chunks = page_size / 16; + slot[0].size = 16; + + n = 0; + size = 32; + + for ( ;; ) { + chunks = page_size / size; + + if (slot[n].chunks != chunks) { + + nxt_mem_zone_slot_init(&slot[n], page_size); + + nxt_thread_log_debug( + "mem zone size:%uD chunks:%uD start:%uD map:%uD", + slot[n].size, slot[n].chunks + 1, + slot[n].start, slot[n].map_size); + + n++; + + if (chunks == 1) { + return &slot[n]; + } + } + + slot[n].chunks = chunks; + slot[n].size = size; + size += 16; + } +} + + +static void +nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot, nxt_uint_t page_size) +{ + /* + * Calculate number of bytes required to store a chunk bitmap + * and align it to 4 bytes. + */ + slot->map_size = nxt_align_size(((slot->chunks + 7) / 8), 4); + + /* If chunk size is not a multiple of zone page size, there + * is surplus space which can be used for the chunk's bitmap. + */ + slot->start = page_size - slot->chunks * slot->size; + + /* slot->chunks should be one less than actual number of chunks. */ + slot->chunks--; + + if (slot->map_size > 4) { + /* A page's chunks bitmap is placed at the start of the page. */ + + if (slot->start < slot->map_size) { + /* + * There is no surplus space or the space is too + * small for chunks bitmap, so use the first chunks. + */ + if (slot->size < slot->map_size) { + /* The first chunks are occupied by bitmap. */ + slot->chunks -= slot->map_size / slot->size; + slot->start = nxt_align_size(slot->map_size, 16); + + } else { + /* The first chunk is occupied by bitmap. */ + slot->chunks--; + slot->start = slot->size; + } + } + } +} + + +/* + * Round up to the next highest power of 2. The algorithm is + * described in "Bit Twiddling Hacks" by Sean Eron Anderson. + */ + +nxt_inline uint32_t +nxt_next_highest_power_of_two(uint32_t n) +{ + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n++; + + return n; +} + + +static nxt_int_t +nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1, nxt_rbtree_node_t *node2) +{ + u_char *start1, *end1, *start2, *end2; + uint32_t n, size, size1, size2; + nxt_mem_zone_free_block_t *block1, *block2; + + block1 = (nxt_mem_zone_free_block_t *) node1; + block2 = (nxt_mem_zone_free_block_t *) node2; + + size1 = block1->size; + size2 = block2->size; + + /* + * This subtractions do not overflow if number of pages of a free + * block is below 2^31-1. This allows to use blocks up to 128G if + * a zone page size is just 64 bytes. + */ + n = size1 - size2; + + if (n != 0) { + return n; + } + + /* + * Sort equally sized blocks by their capability to allocate memory with + * alignment equal to the size rounded the previous higest power of 2. + */ + + /* Round the size to the previous higest power of two. */ + size = nxt_next_highest_power_of_two(size1) >> 1; + + /* Align the blocks' start and end to the rounded size. */ + start1 = nxt_align_ptr(block1, size); + end1 = nxt_trunc_ptr((u_char *) block1 + size1, size); + + start2 = nxt_align_ptr(block2, size); + end2 = nxt_trunc_ptr((u_char *) block2 + size2, size); + + return (end1 - start1) - (end2 - start2); +} + + +void * +nxt_mem_zone_zalloc(nxt_mem_zone_t *zone, size_t size) +{ + void *p; + + p = nxt_mem_zone_align(zone, 1, size); + + if (nxt_fast_path(p != NULL)) { + nxt_memzero(p, size); + } + + return p; +} + + +void * +nxt_mem_zone_align(nxt_mem_zone_t *zone, size_t alignment, size_t size) +{ + void *p; + nxt_mem_zone_slot_t *slot; + + if (nxt_slow_path((alignment - 1) & alignment) != 0) { + /* Alignment must be a power of 2. */ + return NULL; + } + + if (size <= zone->max_chunk_size && alignment <= zone->max_chunk_size) { + /* All chunks are aligned to 16. */ + + if (alignment > 16) { + /* + * Chunks which size is power of 2 are aligned to the size. + * So allocation size should be increased to the next highest + * power of two. This can waste memory, but a main consumer + * of aligned allocations is lvlhsh which anyway allocates + * memory with alignment equal to size. + */ + size = nxt_next_highest_power_of_two(size); + size = nxt_max(size, alignment); + } + + /* + * Find a zone slot with appropriate chunk size. + * This operation can be performed without holding lock. + */ + for (slot = zone->slots; slot->size < size; slot++) { /* void */ } + + nxt_thread_log_debug("mem zone alloc: @%uz:%uz chunk:%uD", + alignment, size, slot->size); + + nxt_thread_spin_lock(&zone->lock); + + p = nxt_mem_zone_alloc_small(zone, slot, size); + + } else { + + nxt_thread_log_debug("mem zone alloc: @%uz:%uz", alignment, size); + + nxt_thread_spin_lock(&zone->lock); + + p = nxt_mem_zone_alloc_large(zone, alignment, size); + } + + nxt_thread_spin_unlock(&zone->lock); + + if (nxt_fast_path(p != NULL)) { + nxt_thread_log_debug("mem zone alloc: %p", p); + + } else { + nxt_log_moderate(&nxt_mem_zone_log_moderation, + NXT_LOG_ALERT, nxt_thread_log(), + "nxt_mem_zone_alloc(%uz, %uz) failed, not enough memory", + alignment, size); + } + + return p; +} + + +static void * +nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone, nxt_mem_zone_slot_t *slot, + size_t size) +{ + u_char *p; + uint8_t *map; + nxt_mem_zone_page_t *page; + + page = slot->pages; + + if (nxt_fast_path(page != NULL)) { + + p = nxt_mem_zone_page_addr(zone, page); + + if (nxt_mem_zone_page_bitmap(zone, slot)) { + /* A page's chunks bitmap is placed at the start of the page. */ + map = p; + + } else { + map = page->u.map; + } + + p += nxt_mem_zone_alloc_chunk(map, slot->start, slot->size); + + page->chunks--; + + if (page->chunks == 0) { + /* + * Remove full page from the zone slot list of pages with + * free chunks. + */ + slot->pages = page->next; +#if (NXT_DEBUG) + page->next = NULL; +#endif + } + + return p; + } + + page = nxt_mem_zone_alloc_pages(zone, 1, 1); + + if (nxt_fast_path(page != NULL)) { + + slot->pages = page; + + page->size = slot->size; + /* slot->chunks are already one less. */ + page->chunks = slot->chunks; + page->u.count = 0; + page->next = NULL; + + p = nxt_mem_zone_page_addr(zone, page); + + if (nxt_mem_zone_page_bitmap(zone, slot)) { + /* A page's chunks bitmap is placed at the start of the page. */ + map = p; + nxt_memzero(map, slot->map_size); + + } else { + map = page->u.map; + } + + /* Mark the first chunk as busy. */ + map[0] = 0x80; + + return p + slot->start; + } + + return NULL; +} + + +static nxt_uint_t +nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset, nxt_uint_t size) +{ + uint8_t mask; + nxt_uint_t n; + + n = 0; + + /* The page must have at least one free chunk. */ + + for ( ;; ) { + /* The bitmap is always aligned to uint32_t. */ + + if (*(uint32_t *) &map[n] != 0xffffffff) { + + do { + if (map[n] != 0xff) { + + mask = 0x80; + + do { + if ((map[n] & mask) == 0) { + /* The free chunk is found. */ + map[n] |= mask; + return offset; + } + + offset += size; + mask >>= 1; + + } while (mask != 0); + + } else { + /* Fast-forward: all 8 chunks are occupied. */ + offset += size * 8; + } + + n++; + + } while (n % 4 != 0); + + } else { + /* Fast-forward: all 32 chunks are occupied. */ + offset += size * 32; + n += 4; + } + } +} + + +static void * +nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment, size_t size) +{ + uint32_t pages; + nxt_mem_zone_page_t *page; + + pages = (size + zone->page_size_mask) >> zone->page_size_shift; + + page = nxt_mem_zone_alloc_pages(zone, alignment, pages); + + if (nxt_fast_path(page != NULL)) { + return nxt_mem_zone_page_addr(zone, page); + } + + return NULL; +} + + +static nxt_mem_zone_page_t * +nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, size_t alignment, uint32_t pages) +{ + u_char *p; + size_t prev_size; + uint32_t prev_pages, node_pages, next_pages; + nxt_uint_t n; + nxt_mem_zone_page_t *prev_page, *page, *next_page; + nxt_mem_zone_free_block_t *block, *next_block; + + block = nxt_mem_zone_find_free_block(zone, + nxt_rbtree_root(&zone->free_pages), + alignment, pages); + + if (nxt_slow_path(block == NULL)) { + return NULL; + } + + node_pages = block->size; + + nxt_rbtree_delete(&zone->free_pages, &block->node); + + p = nxt_align_ptr(block, alignment); + page = nxt_mem_zone_addr_page(zone, p); + + prev_size = p - (u_char *) block; + + if (prev_size != 0) { + prev_pages = prev_size >>= zone->page_size_shift; + node_pages -= prev_pages; + + block->size = prev_pages; + nxt_rbtree_insert(&zone->free_pages, &block->node); + + prev_page = nxt_mem_zone_addr_page(zone, block); + nxt_queue_insert_after(&prev_page->link, &page->link); + } + + next_pages = node_pages - pages; + + if (next_pages != 0) { + next_page = &page[pages]; + next_block = nxt_mem_zone_page_addr(zone, next_page); + next_block->size = next_pages; + + nxt_rbtree_insert(&zone->free_pages, &next_block->node); + nxt_queue_insert_after(&page->link, &next_page->link); + } + + /* Go through pages after all rbtree operations to not trash CPU cache. */ + + page[0].u.count = pages; + + for (n = 0; n < pages; n++) { + + if (page[n].size == NXT_MEM_ZONE_PAGE_FRESH) { + nxt_mem_zone_fresh_junk(nxt_mem_zone_page_addr(zone, &page[n]), + zone->page_size_mask + 1); + } + + page[n].size = NXT_MEM_ZONE_PAGE_USED; + } + + return page; +} + + +/* + * Free blocks are sorted by size and then if the sizes are equal + * by aligned allocation capabilty. The former criterion is just + * comparison with a requested size and it can be used for iteractive + * search. The later criterion cannot be tested only by the requested + * size and alignment, so recursive in-order tree traversal is required + * to find a suitable free block. nxt_mem_zone_find_free_block() uses + * only recursive in-order tree traversal because anyway the slowest part + * of the algorithm are CPU cache misses. Besides the last tail recursive + * call may be optimized by compiler into iteractive search. + */ + +static nxt_mem_zone_free_block_t * +nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node, + uint32_t alignment, uint32_t pages) +{ + u_char *aligned, *end; + nxt_mem_zone_free_block_t *block, *free_block; + + if (node == nxt_rbtree_sentinel(&zone->free_pages)) { + return NULL; + } + + block = (nxt_mem_zone_free_block_t *) node; + + if (pages <= block->size) { + + free_block = nxt_mem_zone_find_free_block(zone, block->node.left, + alignment, pages); + if (free_block != NULL) { + return free_block; + } + + aligned = nxt_align_ptr(block, alignment); + + if (pages == block->size) { + if (aligned == (u_char *) block) { + /* Exact match. */ + return block; + } + + } else { /* pages < block->size */ + aligned += pages << zone->page_size_shift; + end = (u_char *) block + (block->size << zone->page_size_shift); + + if (aligned <= end) { + return block; + } + } + } + + return nxt_mem_zone_find_free_block(zone, block->node.right, + alignment, pages); +} + + +void +nxt_mem_zone_free(nxt_mem_zone_t *zone, void *p) +{ + nxt_uint_t count; + const char *err; + nxt_mem_zone_page_t *page; + + nxt_thread_log_debug("mem zone free: %p", p); + + if (nxt_fast_path(zone->start <= (u_char *) p + && (u_char *) p < zone->end)) + { + page = nxt_mem_zone_addr_page(zone, p); + + nxt_thread_spin_lock(&zone->lock); + + if (nxt_mem_zone_page_is_chunked(page)) { + err = nxt_mem_zone_free_chunk(zone, page, p); + + } else if (nxt_slow_path(nxt_mem_zone_page_is_free(page))) { + err = "page is already free"; + + } else if (nxt_slow_path((uintptr_t) p & zone->page_size_mask) != 0) { + err = "invalid pointer to chunk"; + + } else { + count = page->u.count; + + if (nxt_fast_path(count != 0)) { + nxt_mem_zone_free_junk(p, count * zone->page_size_mask + 1); + nxt_mem_zone_free_pages(zone, page, count); + err = NULL; + + } else { + /* Not the first allocated page. */ + err = "pointer to wrong page"; + } + } + + nxt_thread_spin_unlock(&zone->lock); + + } else { + err = "pointer is out of zone"; + } + + if (nxt_slow_path(err != NULL)) { + nxt_thread_log_alert("nxt_mem_zone_free(%p): %s", p, err); + } +} + + +static const char * +nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page, + void *p) +{ + u_char *map; + uint32_t size, offset, chunk; + nxt_mem_zone_page_t *pg, **ppg; + nxt_mem_zone_slot_t *slot; + + size = page->size; + + /* Find a zone slot with appropriate chunk size. */ + for (slot = zone->slots; slot->size < size; slot++) { /* void */ } + + offset = (uintptr_t) p & zone->page_size_mask; + offset -= slot->start; + + chunk = offset / size; + + if (nxt_slow_path(offset != chunk * size)) { + return "pointer to wrong chunk"; + } + + if (nxt_mem_zone_page_bitmap(zone, slot)) { + /* A page's chunks bitmap is placed at the start of the page. */ + map = (u_char *) ((uintptr_t) p & ~((uintptr_t) zone->page_size_mask)); + + } else { + map = page->u.map; + } + + if (nxt_mem_zone_chunk_is_free(map, chunk)) { + return "chunk is already free"; + } + + nxt_mem_zone_set_chunk_free(map, chunk); + + nxt_mem_zone_free_junk(p, page->size); + + if (page->chunks == 0) { + page->chunks = 1; + + /* Add the page to the head of slot list of pages with free chunks. */ + page->next = slot->pages; + slot->pages = page; + + } else if (page->chunks != slot->chunks) { + page->chunks++; + + } else { + + if (map != page->u.map) { + nxt_mem_zone_free_junk(map, slot->map_size); + } + + /* + * All chunks are free, remove the page from the slot list of pages + * with free chunks and add the page to the free pages tree. + */ + ppg = &slot->pages; + + for (pg = slot->pages; pg != NULL; pg = pg->next) { + + if (pg == page) { + *ppg = page->next; + break; + } + + ppg = &pg->next; + } + + nxt_mem_zone_free_pages(zone, page, 1); + } + + return NULL; +} + + +static void +nxt_mem_zone_free_pages(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page, + nxt_uint_t count) +{ + nxt_mem_zone_page_t *prev_page, *next_page; + nxt_mem_zone_free_block_t *block, *prev_block, *next_block; + + page->size = NXT_MEM_ZONE_PAGE_FREE; + page->chunks = 0; + page->u.count = 0; + page->next = NULL; + + nxt_memzero(&page[1], (count - 1) * sizeof(nxt_mem_zone_page_t)); + + next_page = nxt_queue_link_data(page->link.next, nxt_mem_zone_page_t, link); + + if (nxt_mem_zone_page_is_free(next_page)) { + + /* Coalesce with the next free pages. */ + + nxt_queue_remove(&next_page->link); + nxt_memzero(next_page, sizeof(nxt_mem_zone_page_t)); + + next_block = nxt_mem_zone_page_addr(zone, next_page); + count += next_block->size; + nxt_rbtree_delete(&zone->free_pages, &next_block->node); + } + + prev_page = nxt_queue_link_data(page->link.prev, nxt_mem_zone_page_t, link); + + if (nxt_mem_zone_page_is_free(prev_page)) { + + /* Coalesce with the previous free pages. */ + + nxt_queue_remove(&page->link); + + prev_block = nxt_mem_zone_page_addr(zone, prev_page); + count += prev_block->size; + nxt_rbtree_delete(&zone->free_pages, &prev_block->node); + + prev_block->size = count; + nxt_rbtree_insert(&zone->free_pages, &prev_block->node); + + return; + } + + block = nxt_mem_zone_page_addr(zone, page); + block->size = count; + nxt_rbtree_insert(&zone->free_pages, &block->node); +} diff --git a/src/nxt_mem_zone.h b/src/nxt_mem_zone.h new file mode 100644 index 00000000..3f078c2d --- /dev/null +++ b/src/nxt_mem_zone.h @@ -0,0 +1,29 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_MEM_ZONE_H_INCLUDED_ +#define _NXT_MEM_ZONE_H_INCLUDED_ + + +typedef struct nxt_mem_zone_s nxt_mem_zone_t; + + +NXT_EXPORT nxt_mem_zone_t *nxt_mem_zone_init(u_char *start, size_t zone_size, + nxt_uint_t page_size); + +#define \ +nxt_mem_zone_alloc(zone, size) \ + nxt_mem_zone_align((zone), 1, (size)) + +NXT_EXPORT void *nxt_mem_zone_align(nxt_mem_zone_t *zone, size_t alignment, + size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void *nxt_mem_zone_zalloc(nxt_mem_zone_t *zone, size_t size) + NXT_MALLOC_LIKE; +NXT_EXPORT void nxt_mem_zone_free(nxt_mem_zone_t *zone, void *p); + + +#endif /* _NXT_MEM_ZONE_H_INCLUDED_ */ diff --git a/src/nxt_murmur_hash.c b/src/nxt_murmur_hash.c new file mode 100644 index 00000000..e9adabfa --- /dev/null +++ b/src/nxt_murmur_hash.c @@ -0,0 +1,84 @@ + +/* + * The code is based on the code by Austin Appleby, + * released to the public domain. + */ + +#include <nxt_main.h> + + +uint32_t +nxt_murmur_hash2(const void *data, size_t len) +{ + uint32_t h, k; + const u_char *p; + const uint32_t m = 0x5bd1e995; + + p = data; + h = 0 ^ (uint32_t) len; + + while (len >= 4) { + k = p[0]; + k |= p[1] << 8; + k |= p[2] << 16; + k |= p[3] << 24; + + k *= m; + k ^= k >> 24; + k *= m; + + h *= m; + h ^= k; + + p += 4; + len -= 4; + } + + switch (len) { + case 3: + h ^= p[2] << 16; + case 2: + h ^= p[1] << 8; + case 1: + h ^= p[0]; + h *= m; + } + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + + +/* The MurmurHash2 for fixed 4 byte length. */ + +uint32_t +nxt_murmur_hash2_uint32(const void *data) +{ + uint32_t h, k; + const u_char *p; + const uint32_t m = 0x5bd1e995; + + p = data; + + k = p[0]; + k |= p[1] << 8; + k |= p[2] << 16; + k |= p[3] << 24; + + k *= m; + k ^= k >> 24; + k *= m; + + h = 0 ^ 4; + h *= m; + h ^= k; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} diff --git a/src/nxt_murmur_hash.h b/src/nxt_murmur_hash.h new file mode 100644 index 00000000..289dc5b0 --- /dev/null +++ b/src/nxt_murmur_hash.h @@ -0,0 +1,15 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_MURMUR_HASH_H_INCLUDED_ +#define _NXT_MURMUR_HASH_H_INCLUDED_ + + +NXT_EXPORT uint32_t nxt_murmur_hash2(const void *data, size_t len); +NXT_EXPORT uint32_t nxt_murmur_hash2_uint32(const void *data); + + +#endif /* _NXT_MURMUR_HASH_H_INCLUDED_ */ diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c new file mode 100644 index 00000000..fcdd2876 --- /dev/null +++ b/src/nxt_openssl.c @@ -0,0 +1,855 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <openssl/ssl.h> +#include <openssl/conf.h> +#include <openssl/err.h> + + +typedef struct { + SSL *session; + + int ssl_error; + uint8_t times; /* 2 bits */ + + nxt_buf_mem_t buffer; +} nxt_openssl_conn_t; + + +static nxt_int_t nxt_openssl_server_init(nxt_ssltls_conf_t *conf); + +static void nxt_openssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c); +static void nxt_openssl_session_cleanup(void *data); +static void nxt_openssl_conn_handshake(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_openssl_conn_io_read(nxt_thread_t *thr, void *obj, void *data); +static void nxt_openssl_conn_io_shutdown(nxt_thread_t *thr, void *obj, + void *data); +static ssize_t nxt_openssl_conn_io_write_chunk(nxt_thread_t *thr, + nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); +static ssize_t nxt_openssl_conn_io_send(nxt_event_conn_t *c, void *buf, + size_t size); +static nxt_int_t nxt_openssl_conn_test_error(nxt_thread_t *thr, + nxt_event_conn_t *c, int ret, nxt_err_t sys_err, + nxt_work_handler_t handler); +static void nxt_cdecl nxt_openssl_conn_error(nxt_event_conn_t *c, nxt_err_t err, + const char *fmt, ...); +static nxt_uint_t nxt_openssl_log_error_level(nxt_event_conn_t *c, + nxt_err_t err); +static void nxt_cdecl nxt_openssl_log_error(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...); +static u_char *nxt_openssl_copy_error(u_char *p, u_char *end); + + +const nxt_ssltls_lib_t nxt_openssl_lib = { + nxt_openssl_server_init, + NULL, +}; + + +static nxt_event_conn_io_t nxt_openssl_event_conn_io = { + NULL, + NULL, + + nxt_openssl_conn_io_read, + NULL, + NULL, + + nxt_event_conn_io_write, + nxt_openssl_conn_io_write_chunk, + NULL, + NULL, + nxt_openssl_conn_io_send, + + nxt_openssl_conn_io_shutdown, +}; + + +static long nxt_openssl_version; +static int nxt_openssl_connection_index; + + +static nxt_int_t +nxt_openssl_start(nxt_thread_t *thr) +{ + int index; + + if (nxt_fast_path(nxt_openssl_version != 0)) { + return NXT_OK; + } + + SSL_load_error_strings(); + + OPENSSL_config(NULL); + + /* + * SSL_library_init(3): + * + * SSL_library_init() always returns "1", + * so it is safe to discard the return value. + */ + (void) SSL_library_init(); + + nxt_openssl_version = SSLeay(); + + nxt_log_error(NXT_LOG_INFO, thr->log, "%s, %xl", + SSLeay_version(SSLEAY_VERSION), nxt_openssl_version); + +#ifndef SSL_OP_NO_COMPRESSION + { + /* + * Disable gzip compression in OpenSSL prior to 1.0.0 + * version, this saves about 522K per connection. + */ + int n; + STACK_OF(SSL_COMP) *ssl_comp_methods; + + ssl_comp_methods = SSL_COMP_get_compression_methods(); + + for (n = sk_SSL_COMP_num(ssl_comp_methods); n != 0; n--) { + (void) sk_SSL_COMP_pop(ssl_comp_methods); + } + } +#endif + + index = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL); + + if (index == -1) { + nxt_openssl_log_error(NXT_LOG_CRIT, thr->log, + "SSL_get_ex_new_index() failed"); + return NXT_ERROR; + } + + nxt_openssl_connection_index = index; + + return NXT_OK; +} + + +static nxt_int_t +nxt_openssl_server_init(nxt_ssltls_conf_t *conf) +{ + SSL_CTX *ctx; + const char *certificate, *key, *ciphers, *ca_certificate; + nxt_thread_t *thr; + STACK_OF(X509_NAME) *list; + + thr = nxt_thread(); + + if (nxt_openssl_start(thr) != NXT_OK) { + return NXT_ERROR; + } + + ctx = SSL_CTX_new(SSLv23_server_method()); + if (ctx == NULL) { + nxt_openssl_log_error(NXT_LOG_CRIT, thr->log, "SSL_CTX_new() failed"); + return NXT_ERROR; + } + + conf->ctx = ctx; + conf->conn_init = nxt_openssl_conn_init; + +#ifdef SSL_OP_NO_COMPRESSION + /* + * Disable gzip compression in OpenSSL 1.0.0, + * this saves about 522K per connection. + */ + SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION); +#endif + +#ifdef SSL_MODE_RELEASE_BUFFERS + + if (nxt_openssl_version >= 10001078) { + /* + * Allow to release read and write buffers in OpenSSL 1.0.0, + * this saves about 34K per idle connection. It is not safe + * before OpenSSL 1.0.1h (CVE-2010-5298). + */ + SSL_CTX_set_mode(ctx, SSL_MODE_RELEASE_BUFFERS); + } + +#endif + + certificate = conf->certificate; + + if (SSL_CTX_use_certificate_chain_file(ctx, certificate) == 0) { + nxt_openssl_log_error(NXT_LOG_CRIT, thr->log, + "SSL_CTX_use_certificate_file(\"%s\") failed", + certificate); + goto fail; + } + + key = conf->certificate_key; + + if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) == 0) { + nxt_openssl_log_error(NXT_LOG_CRIT, thr->log, + "SSL_CTX_use_PrivateKey_file(\"%s\") failed", + key); + goto fail; + } + + ciphers = (conf->ciphers != NULL) ? conf->ciphers : "HIGH:!aNULL:!MD5"; + + if (SSL_CTX_set_cipher_list(ctx, ciphers) == 0) { + nxt_openssl_log_error(NXT_LOG_CRIT, thr->log, + "SSL_CTX_set_cipher_list(\"%s\") failed", + ciphers); + goto fail; + } + + SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); + + if (conf->ca_certificate != NULL) { + + /* TODO: verify callback */ + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL); + + /* TODO: verify depth */ + SSL_CTX_set_verify_depth(ctx, 1); + + ca_certificate = conf->ca_certificate; + + if (SSL_CTX_load_verify_locations(ctx, ca_certificate, NULL) == 0) { + nxt_openssl_log_error(NXT_LOG_CRIT, thr->log, + "SSL_CTX_load_verify_locations(\"%s\") failed", + ca_certificate); + goto fail; + } + + list = SSL_load_client_CA_file(ca_certificate); + + if (list == NULL) { + nxt_openssl_log_error(NXT_LOG_CRIT, thr->log, + "SSL_load_client_CA_file(\"%s\") failed", + ca_certificate); + goto fail; + } + + /* + * SSL_load_client_CA_file() in OpenSSL prior to 0.9.7h and + * 0.9.8 versions always leaves an error in the error queue. + */ + ERR_clear_error(); + + SSL_CTX_set_client_CA_list(ctx, list); + } + + return NXT_OK; + +fail: + + SSL_CTX_free(ctx); + + return NXT_ERROR; +} + + +static void +nxt_openssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c) +{ + int ret; + SSL *s; + SSL_CTX *ctx; + nxt_openssl_conn_t *ssltls; + nxt_mem_pool_cleanup_t *mpcl; + + nxt_log_debug(c->socket.log, "openssl conn init"); + + ssltls = nxt_mem_zalloc(c->mem_pool, sizeof(nxt_openssl_conn_t)); + if (ssltls == NULL) { + goto fail; + } + + c->u.ssltls = ssltls; + nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size); + + mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0); + if (mpcl == NULL) { + goto fail; + } + + ctx = conf->ctx; + + s = SSL_new(ctx); + if (s == NULL) { + nxt_openssl_log_error(NXT_LOG_CRIT, c->socket.log, "SSL_new() failed"); + goto fail; + } + + ssltls->session = s; + mpcl->handler = nxt_openssl_session_cleanup; + mpcl->data = ssltls; + + ret = SSL_set_fd(s, c->socket.fd); + + if (ret == 0) { + nxt_openssl_log_error(NXT_LOG_CRIT, c->socket.log, + "SSL_set_fd(%d) failed", c->socket.fd); + goto fail; + } + + SSL_set_accept_state(s); + + if (SSL_set_ex_data(s, nxt_openssl_connection_index, c) == 0) { + nxt_openssl_log_error(NXT_LOG_CRIT, c->socket.log, + "SSL_set_ex_data() failed"); + goto fail; + } + + c->io = &nxt_openssl_event_conn_io; + c->sendfile = NXT_CONN_SENDFILE_OFF; + + nxt_openssl_conn_handshake(thr, c, c->socket.data); + return; + +fail: + + nxt_event_conn_io_handle(thr, c->read_work_queue, + c->read_state->error_handler, c, c->socket.data); +} + + +static void +nxt_openssl_session_cleanup(void *data) +{ + nxt_openssl_conn_t *ssltls; + + ssltls = data; + + nxt_thread_log_debug("openssl session cleanup"); + + nxt_free(ssltls->buffer.start); + + SSL_free(ssltls->session); +} + + +static void +nxt_openssl_conn_handshake(nxt_thread_t *thr, void *obj, void *data) +{ + int ret; + nxt_int_t n; + nxt_err_t err; + nxt_event_conn_t *c; + nxt_openssl_conn_t *ssltls; + + c = obj; + ssltls = c->u.ssltls; + + nxt_log_debug(thr->log, "openssl conn handshake: %d", ssltls->times); + + /* "ssltls->times == 1" is suitable to run SSL_do_handshake() in job. */ + + ret = SSL_do_handshake(ssltls->session); + + err = (ret <= 0) ? nxt_socket_errno : 0; + + nxt_thread_time_debug_update(thr); + + nxt_log_debug(thr->log, "SSL_do_handshake(%d): %d err:%d", + c->socket.fd, ret, err); + + if (ret > 0) { + /* ret == 1, the handshake was successfully completed. */ + nxt_openssl_conn_io_read(thr, c, data); + return; + } + + n = nxt_openssl_conn_test_error(thr, c, ret, err, + nxt_openssl_conn_handshake); + + if (n == NXT_ERROR) { + nxt_openssl_conn_error(c, err, "SSL_do_handshake(%d) failed", + c->socket.fd); + + nxt_event_conn_io_handle(thr, c->read_work_queue, + c->read_state->error_handler, c, data); + + } else if (ssltls->ssl_error == SSL_ERROR_WANT_READ && ssltls->times < 2) { + ssltls->times++; + } +} + + +static void +nxt_openssl_conn_io_read(nxt_thread_t *thr, void *obj, void *data) +{ + int ret; + nxt_buf_t *b; + nxt_int_t n; + nxt_err_t err; + nxt_event_conn_t *c; + nxt_work_handler_t handler; + nxt_openssl_conn_t *ssltls; + + c = obj; + + nxt_log_debug(thr->log, "openssl conn read"); + + handler = c->read_state->ready_handler; + b = c->read; + + /* b == NULL is used to test descriptor readiness. */ + + if (b != NULL) { + ssltls = c->u.ssltls; + + ret = SSL_read(ssltls->session, b->mem.free, b->mem.end - b->mem.free); + + err = (ret <= 0) ? nxt_socket_errno : 0; + + nxt_log_debug(thr->log, "SSL_read(%d, %p, %uz): %d err:%d", + c->socket.fd, b->mem.free, b->mem.end - b->mem.free, + ret, err); + + if (ret > 0) { + /* c->socket.read_ready is kept. */ + b->mem.free += ret; + handler = c->read_state->ready_handler; + + } else { + n = nxt_openssl_conn_test_error(thr, c, ret, err, + nxt_openssl_conn_io_read); + + if (nxt_fast_path(n != NXT_ERROR)) { + return; + } + + nxt_openssl_conn_error(c, err, "SSL_read(%d, %p, %uz) failed", + c->socket.fd, b->mem.free, + b->mem.end - b->mem.free); + + handler = c->read_state->error_handler; + } + } + + nxt_event_conn_io_handle(thr, c->read_work_queue, handler, c, data); +} + + +static ssize_t +nxt_openssl_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, + nxt_buf_t *b, size_t limit) +{ + nxt_openssl_conn_t *ssltls; + + nxt_log_debug(thr->log, "openssl conn write chunk"); + + ssltls = c->u.ssltls; + + return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit); +} + + +static ssize_t +nxt_openssl_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) +{ + int ret; + nxt_err_t err; + nxt_int_t n; + nxt_openssl_conn_t *ssltls; + + ssltls = c->u.ssltls; + + ret = SSL_write(ssltls->session, buf, size); + + if (ret <= 0) { + err = nxt_socket_errno; + c->socket.error = err; + + } else { + err = 0; + } + + nxt_log_debug(c->socket.log, "SSL_write(%d, %p, %uz): %d err:%d", + c->socket.fd, buf, size, ret, err); + + if (ret > 0) { + return ret; + } + + n = nxt_openssl_conn_test_error(nxt_thread(), c, ret, err, + nxt_event_conn_io_write); + + if (n == NXT_ERROR) { + nxt_openssl_conn_error(c, err, "SSL_write(%d, %p, %uz) failed", + c->socket.fd, buf, size); + } + + return n; +} + + +static void +nxt_openssl_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) +{ + int ret, mode; + SSL *s; + nxt_err_t err; + nxt_int_t n; + nxt_bool_t quiet, once; + nxt_event_conn_t *c; + nxt_work_handler_t handler; + nxt_openssl_conn_t *ssltls; + + c = obj; + + nxt_log_debug(thr->log, "openssl conn shutdown"); + + ssltls = c->u.ssltls; + s = ssltls->session; + + if (s == NULL) { + handler = c->write_state->close_handler; + goto done; + } + + mode = SSL_get_shutdown(s); + + if (c->socket.timedout || c->socket.error != 0) { + quiet = 1; + + } else if (c->socket.closed && !(mode & SSL_RECEIVED_SHUTDOWN)) { + quiet = 1; + + } else { + quiet = 0; + } + + SSL_set_quiet_shutdown(s, quiet); + + once = 1; + + for ( ;; ) { + SSL_set_shutdown(s, mode); + + ret = SSL_shutdown(s); + + err = (ret <= 0) ? nxt_socket_errno : 0; + + nxt_log_debug(thr->log, "SSL_shutdown(%d, %d, %b): %d err:%d", + c->socket.fd, mode, quiet, ret, err); + + if (ret > 0) { + /* ret == 1, the shutdown was successfully completed. */ + handler = c->write_state->close_handler; + goto done; + } + + if (ret == 0) { + /* + * If SSL_shutdown() returns 0 then it should be called + * again. The second SSL_shutdown() call should returns + * -1/SSL_ERROR_WANT_READ or -1/SSL_ERROR_WANT_WRITE. + * OpenSSL prior to 0.9.8m version however never returns + * -1 at all. Fortunately, OpenSSL internals preserve + * correct status available via SSL_get_error(-1). + */ + if (once) { + mode = SSL_get_shutdown(s); + once = 0; + continue; + } + + ret = -1; + } + + /* ret == -1 */ + + break; + } + + n = nxt_openssl_conn_test_error(thr, c, ret, err, + nxt_openssl_conn_io_shutdown); + + if (nxt_fast_path(n == 0)) { + return; + } + + if (n != NXT_ERROR) { /* n == NXT_AGAIN */ + c->socket.error_handler = c->read_state->error_handler; + nxt_event_timer_add(thr->engine, &c->read_timer, 5000); + return; + } + + nxt_openssl_conn_error(c, err, "SSL_shutdown(%d) failed", c->socket.fd); + + handler = c->write_state->error_handler; + +done: + + nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); +} + + +static nxt_int_t +nxt_openssl_conn_test_error(nxt_thread_t *thr, nxt_event_conn_t *c, int ret, + nxt_err_t sys_err, nxt_work_handler_t handler) +{ + u_long lib_err; + nxt_work_queue_t *wq; + nxt_openssl_conn_t *ssltls; + + ssltls = c->u.ssltls; + + ssltls->ssl_error = SSL_get_error(ssltls->session, ret); + + nxt_log_debug(c->socket.log, "SSL_get_error(): %d", ssltls->ssl_error); + + switch (ssltls->ssl_error) { + + case SSL_ERROR_WANT_READ: + nxt_event_fd_block_write(thr->engine, &c->socket); + + c->socket.read_ready = 0; + c->socket.read_handler = handler; + + if (nxt_event_fd_is_disabled(c->socket.read)) { + nxt_event_fd_enable_read(thr->engine, &c->socket); + } + + return NXT_AGAIN; + + case SSL_ERROR_WANT_WRITE: + nxt_event_fd_block_read(thr->engine, &c->socket); + + c->socket.write_ready = 0; + c->socket.write_handler = handler; + + if (nxt_event_fd_is_disabled(c->socket.write)) { + nxt_event_fd_enable_write(thr->engine, &c->socket); + } + + return NXT_AGAIN; + + case SSL_ERROR_SYSCALL: + + lib_err = ERR_peek_error(); + + nxt_log_debug(c->socket.log, "ERR_peek_error(): %l", lib_err); + + if (sys_err != 0 || lib_err != 0) { + return NXT_ERROR; + } + + /* A connection was just closed. */ + c->socket.closed = 1; + + /* Fall through. */ + + case SSL_ERROR_ZERO_RETURN: + /* A "close notify" alert. */ + + if (c->read_state != NULL) { + wq = c->read_work_queue; + handler = c->read_state->close_handler; + + } else { + wq = c->write_work_queue; + handler = c->write_state->close_handler; + } + + nxt_event_conn_io_handle(thr, wq, handler, c, c->socket.data); + + return 0; + + default: /* SSL_ERROR_SSL, etc. */ + c->socket.error = 1000; /* Nonexistent errno code. */ + return NXT_ERROR; + } +} + + +static void nxt_cdecl +nxt_openssl_conn_error(nxt_event_conn_t *c, nxt_err_t err, const char *fmt, ...) +{ + u_char *p, *end; + va_list args; + nxt_uint_t level; + u_char msg[NXT_MAX_ERROR_STR]; + + c->socket.error = err; + level = nxt_openssl_log_error_level(c, err); + + if (nxt_log_level_enough(c->socket.log, level)) { + + end = msg + sizeof(msg); + + va_start(args, fmt); + p = nxt_vsprintf(msg, end, fmt, args); + va_end(args); + + if (err != 0) { + p = nxt_sprintf(p, end, " %E", err); + } + + p = nxt_openssl_copy_error(p, end); + + nxt_log_error(level, c->socket.log, "%*s", p - msg, msg); + + } else { + ERR_clear_error(); + } +} + + +static nxt_uint_t +nxt_openssl_log_error_level(nxt_event_conn_t *c, nxt_err_t err) +{ + switch (ERR_GET_REASON(ERR_peek_error())) { + + case 0: + return nxt_socket_error_level(err, c->socket.log_error); + + case SSL_R_BAD_CHANGE_CIPHER_SPEC: /* 103 */ + case SSL_R_BLOCK_CIPHER_PAD_IS_WRONG: /* 129 */ + case SSL_R_DIGEST_CHECK_FAILED: /* 149 */ + case SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST: /* 151 */ + case SSL_R_EXCESSIVE_MESSAGE_SIZE: /* 152 */ + case SSL_R_LENGTH_MISMATCH: /* 159 */ + case SSL_R_NO_CIPHERS_PASSED: /* 182 */ + case SSL_R_NO_CIPHERS_SPECIFIED: /* 183 */ + case SSL_R_NO_COMPRESSION_SPECIFIED: /* 187 */ + case SSL_R_NO_SHARED_CIPHER: /* 193 */ + case SSL_R_RECORD_LENGTH_MISMATCH: /* 213 */ +#ifdef SSL_R_PARSE_TLSEXT + case SSL_R_PARSE_TLSEXT: /* 227 */ +#endif + case SSL_R_UNEXPECTED_MESSAGE: /* 244 */ + case SSL_R_UNEXPECTED_RECORD: /* 245 */ + case SSL_R_UNKNOWN_ALERT_TYPE: /* 246 */ + case SSL_R_UNKNOWN_PROTOCOL: /* 252 */ + case SSL_R_WRONG_VERSION_NUMBER: /* 267 */ + case SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC: /* 281 */ +#ifdef SSL_R_RENEGOTIATE_EXT_TOO_LONG + case SSL_R_RENEGOTIATE_EXT_TOO_LONG: /* 335 */ + case SSL_R_RENEGOTIATION_ENCODING_ERR: /* 336 */ + case SSL_R_RENEGOTIATION_MISMATCH: /* 337 */ +#endif +#ifdef SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED + case SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED: /* 338 */ +#endif +#ifdef SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING + case SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING: /* 345 */ +#endif + case 1000:/* SSL_R_SSLV3_ALERT_CLOSE_NOTIFY */ + case SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE: /* 1010 */ + case SSL_R_SSLV3_ALERT_BAD_RECORD_MAC: /* 1020 */ + case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: /* 1021 */ + case SSL_R_TLSV1_ALERT_RECORD_OVERFLOW: /* 1022 */ + case SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE: /* 1030 */ + case SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE: /* 1040 */ + case SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER: /* 1047 */ + break; + + case SSL_R_SSLV3_ALERT_NO_CERTIFICATE: /* 1041 */ + case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: /* 1042 */ + case SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE: /* 1043 */ + case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: /* 1044 */ + case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: /* 1045 */ + case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: /* 1046 */ + case SSL_R_TLSV1_ALERT_UNKNOWN_CA: /* 1048 */ + case SSL_R_TLSV1_ALERT_ACCESS_DENIED: /* 1049 */ + case SSL_R_TLSV1_ALERT_DECODE_ERROR: /* 1050 */ + case SSL_R_TLSV1_ALERT_DECRYPT_ERROR: /* 1051 */ + case SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION: /* 1060 */ + case SSL_R_TLSV1_ALERT_PROTOCOL_VERSION: /* 1070 */ + case SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY: /* 1071 */ + case SSL_R_TLSV1_ALERT_INTERNAL_ERROR: /* 1080 */ + case SSL_R_TLSV1_ALERT_USER_CANCELLED: /* 1090 */ + case SSL_R_TLSV1_ALERT_NO_RENEGOTIATION: /* 1100 */ + return NXT_LOG_ERR; + + default: + return NXT_LOG_CRIT; + } + + return NXT_LOG_INFO; +} + + +static void nxt_cdecl +nxt_openssl_log_error(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...) +{ + u_char *p, *end; + va_list args; + u_char msg[NXT_MAX_ERROR_STR]; + + end = msg + sizeof(msg); + + va_start(args, fmt); + p = nxt_vsprintf(msg, end, fmt, args); + va_end(args); + + p = nxt_openssl_copy_error(p, end); + + nxt_log_error(level, log, "%*s", p - msg, msg); +} + + +static u_char * +nxt_openssl_copy_error(u_char *p, u_char *end) +{ + int flags; + u_long err; + nxt_bool_t clear; + const char *data, *delimiter; + + err = ERR_peek_error(); + if (err == 0) { + return p; + } + + /* Log the most relevant error message ... */ + data = ERR_reason_error_string(err); + + p = nxt_sprintf(p, end, " (%d: %s) (OpenSSL: ", ERR_GET_REASON(err), data); + + /* + * ... followed by all queued cumbersome OpenSSL + * error messages and drain the error queue. + */ + delimiter = ""; + clear = 0; + + for ( ;; ) { + err = ERR_get_error_line_data(NULL, NULL, &data, &flags); + if (err == 0) { + break; + } + + p = nxt_sprintf(p, end, "%s", delimiter); + + ERR_error_string_n(err, (char *) p, end - p); + + while (p < end && *p != '\0') { + p++; + } + + if ((flags & ERR_TXT_STRING) != 0) { + p = nxt_sprintf(p, end, ":%s", data); + } + + clear |= ((flags & ERR_TXT_MALLOCED) != 0); + + delimiter = "; "; + } + + /* Deallocate additional data. */ + + if (clear) { + ERR_clear_error(); + } + + if (p < end) { + *p++ = ')'; + } + + return p; +} diff --git a/src/nxt_parse.c b/src/nxt_parse.c new file mode 100644 index 00000000..01dc8aa7 --- /dev/null +++ b/src/nxt_parse.c @@ -0,0 +1,344 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * nxt_int_parse() returns size_t value >= 0 on success, + * -1 on failure, and -2 on overflow. + */ + +nxt_int_t +nxt_int_parse(const u_char *p, size_t len) +{ + u_char c; + nxt_uint_t val; + + if (nxt_fast_path(len != 0)) { + + val = 0; + + do { + c = *p++; + + /* Values below '0' become >= 208. */ + c = c - '0'; + + if (nxt_slow_path(c > 9)) { + return -1; + } + + val = val * 10 + c; + + if (nxt_slow_path((nxt_int_t) val < 0)) { + /* An overflow. */ + return -2; + } + + len--; + + } while (len != 0); + + return val; + } + + return -1; +} + + +/* + * nxt_size_t_parse() returns size_t value >= 0 on success, + * -1 on failure, and -2 on overflow. + */ + +ssize_t +nxt_size_t_parse(const u_char *p, size_t len) +{ + u_char c; + size_t val; + + if (nxt_fast_path(len != 0)) { + + val = 0; + + do { + c = *p++; + + /* Values below '0' become >= 208. */ + c = c - '0'; + + if (nxt_slow_path(c > 9)) { + return -1; + } + + val = val * 10 + c; + + if (nxt_slow_path((ssize_t) val < 0)) { + /* An overflow. */ + return -2; + } + + len--; + + } while (len != 0); + + return val; + } + + return -1; +} + + +/* + * nxt_size_parse() parses size string with optional K or M units and + * returns size_t value >= 0 on success, -1 on failure, and -2 on overflow. + */ + +ssize_t +nxt_size_parse(const u_char *p, size_t len) +{ + u_char c, unit; + size_t val, max; + nxt_uint_t shift; + + if (nxt_fast_path(len != 0)) { + + len--; + + /* Upper case. */ + unit = p[len] & ~0x20; + + switch (unit) { + + case 'G': + max = NXT_SIZE_T_MAX >> 30; + shift = 30; + break; + + case 'M': + max = NXT_SIZE_T_MAX >> 20; + shift = 20; + break; + + case 'K': + max = NXT_SIZE_T_MAX >> 10; + shift = 10; + break; + + default: + max = NXT_SIZE_T_MAX; + shift = 0; + len++; + break; + } + + if (nxt_fast_path(len != 0)) { + + val = 0; + + do { + c = *p++; + + /* Values below '0' become >= 208. */ + c = c - '0'; + + if (nxt_slow_path(c > 9)) { + return -1; + } + + val = val * 10 + c; + + if (nxt_slow_path(val > max)) { + /* An overflow. */ + return -2; + } + + len--; + + } while (len != 0); + + return val << shift; + } + } + + return -1; +} + + +/* + * nxt_off_t_parse() returns nxt_off_t value >= 0 on success, + * -1 on failure, and -2 on overflow. + */ + +nxt_off_t +nxt_off_t_parse(const u_char *p, size_t len) +{ + u_char c; + nxt_uoff_t val; + + if (nxt_fast_path(len != 0)) { + + val = 0; + + do { + c = *p++; + + /* Values below '0' become >= 208. */ + c = c - '0'; + + if (nxt_slow_path(c > 9)) { + return -1; + } + + val = val * 10 + c; + + if (nxt_slow_path((nxt_off_t) val < 0)) { + /* An overflow. */ + return -2; + } + + len--; + + } while (len != 0); + + return val; + } + + return -1; +} + + +/* + * nxt_str_int_parse() returns nxt_int_t value >= 0 on success, + * -1 on failure, and -2 on overflow and also updates the 's' argument. + */ + +nxt_int_t +nxt_str_int_parse(nxt_str_t *s) +{ + u_char c, *p; + size_t len; + nxt_uint_t val; + + len = s->len; + + if (nxt_slow_path(len == 0)) { + return -1; + } + + p = s->data; + val = 0; + + do { + c = *p; + + /* Values below '0' become >= 208. */ + c = c - '0'; + + if (c > 9) { + break; + } + + val = val * 10 + c; + + if (nxt_slow_path((nxt_int_t) val < 0)) { + /* An overflow. */ + return -2; + } + + p++; + len--; + + } while (len != 0); + + s->len = len; + s->data = p; + + return val; +} + + +/* + * nxt_number_parse() returns a double value >= 0 and updates the start + * argument on success, or returns -1 on failure or -2 on overflow. + */ + +double +nxt_number_parse(const u_char **start, const u_char *end) +{ + u_char c; + nxt_uint_t integral, frac, power; + const u_char *p; + + p = *start; + integral = 0; + + while (p < end) { + c = *p; + + if (c == '.') { + goto dot; + } + + /* Values below '0' become >= 208. */ + c = c - '0'; + + if (c > 9) { + break; + } + + integral = integral * 10 + c; + + if (nxt_slow_path((nxt_int_t) integral < 0)) { + /* An overflow. */ + return -2; + } + + p++; + } + + if (nxt_fast_path(p != *start)) { + *start = p; + return integral; + } + + /* No value. */ + return -1; + +dot: + + if (nxt_slow_path(p == *start)) { + /* No leading digit before dot. */ + return -1; + } + + frac = 0; + power = 1; + + for (p++; p < end; p++) { + c = *p; + + /* Values below '0' become >= 208. */ + c = c - '0'; + + if (c > 9) { + break; + } + + frac = frac * 10 + c; + power *= 10; + + if (nxt_slow_path((nxt_int_t) frac < 0 || (nxt_int_t) power < 0)) { + /* An overflow. */ + return -2; + } + } + + *start = p; + + return integral + (double) frac / power; +} diff --git a/src/nxt_parse.h b/src/nxt_parse.h new file mode 100644 index 00000000..0643af1e --- /dev/null +++ b/src/nxt_parse.h @@ -0,0 +1,25 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_PARSE_H_INCLUDED_ +#define _NXT_PARSE_H_INCLUDED_ + + +NXT_EXPORT nxt_int_t nxt_int_parse(const u_char *p, size_t len); +NXT_EXPORT ssize_t nxt_size_t_parse(const u_char *p, size_t len); +NXT_EXPORT ssize_t nxt_size_parse(const u_char *p, size_t len); +NXT_EXPORT nxt_off_t nxt_off_t_parse(const u_char *p, size_t len); + +NXT_EXPORT nxt_int_t nxt_str_int_parse(nxt_str_t *s); + +NXT_EXPORT double nxt_number_parse(const u_char **start, const u_char *end); + +NXT_EXPORT nxt_time_t nxt_time_parse(const u_char *p, size_t len); +NXT_EXPORT nxt_int_t nxt_term_parse(const u_char *p, size_t len, + nxt_bool_t seconds); + + +#endif /* _NXT_PARSE_H_INCLUDED_ */ diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c new file mode 100644 index 00000000..aadd5648 --- /dev/null +++ b/src/nxt_php_sapi.c @@ -0,0 +1,611 @@ + +/* + * Copyright (C) Valentin V. Bartenev + * Copyright (C) NGINX, Inc. + */ + +#include "php.h" +#include "SAPI.h" +#include "php_main.h" +#include "php_variables.h" + +#include <nxt_main.h> +#include <nxt_application.h> + + +typedef struct { + size_t max_name; + + nxt_str_t *cookie; + nxt_str_t *content_type; + nxt_str_t *content_length; + + nxt_str_t script; + nxt_str_t query; + + size_t script_name_len; + + off_t content_length_n; +} nxt_php_ctx_t; + + +nxt_int_t nxt_php_init(nxt_thread_t *thr); +nxt_int_t nxt_php_request_init(nxt_app_request_t *r); +nxt_int_t nxt_php_request_header(nxt_app_request_t *r, + nxt_app_header_field_t *field); +nxt_int_t nxt_php_handler(nxt_app_request_t *r); + + +nxt_int_t nxt_python_init(); + + +static nxt_int_t nxt_php_opts(nxt_log_t *log); + + +static int nxt_php_startup(sapi_module_struct *sapi_module); +static int nxt_php_send_headers(sapi_headers_struct *sapi_headers); +static char *nxt_php_read_cookies(void); +static void nxt_php_register_variables(zval *track_vars_array); +static void nxt_php_log_message(char *message); + +#define NXT_PHP7 1 + +#ifdef NXT_PHP7 +static size_t nxt_php_unbuffered_write(const char *str, + size_t str_length TSRMLS_DC); +static size_t nxt_php_read_post(char *buffer, size_t count_bytes TSRMLS_DC); +#else +static int nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC); +static int nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC); +#endif + + +static sapi_module_struct nxt_php_sapi_module = +{ + (char *) "cli-server", + (char *) "nginman", + + nxt_php_startup, /* startup */ + php_module_shutdown_wrapper, /* shutdown */ + + NULL, /* activate */ + NULL, /* deactivate */ + + nxt_php_unbuffered_write, /* unbuffered write */ + NULL, /* flush */ + NULL, /* get uid */ + NULL, /* getenv */ + + php_error, /* error handler */ + + NULL, /* header handler */ + nxt_php_send_headers, /* send headers handler */ + NULL, /* send header handler */ + + nxt_php_read_post, /* read POST data */ + nxt_php_read_cookies, /* read Cookies */ + + nxt_php_register_variables, /* register server variables */ + nxt_php_log_message, /* log message */ + + NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, 0, 0, NULL, NULL, NULL, + NULL, NULL, NULL, 0, NULL, NULL, NULL +}; + + +static nxt_str_t nxt_php_path; +static nxt_str_t nxt_php_root; +static nxt_str_t nxt_php_script; + + +nxt_int_t +nxt_php_init(nxt_thread_t *thr) +{ + if (nxt_php_opts(thr->log)) { + return NXT_ERROR; + } + + sapi_startup(&nxt_php_sapi_module); + nxt_php_startup(&nxt_php_sapi_module); + + return NXT_OK; +} + + +static nxt_int_t +nxt_php_opts(nxt_log_t *log) +{ + char **argv; + u_char *p; + nxt_uint_t i; + + argv = nxt_process_argv; + + while (*argv != NULL) { + p = (u_char *) *argv++; + + if (nxt_strcmp(p, "--php") == 0) { + if (*argv == NULL) { + nxt_log_error(NXT_LOG_ERR, log, + "no argument for option \"--php\""); + return NXT_ERROR; + } + + p = (u_char *) *argv; + + nxt_php_root.data = p; + nxt_php_path.data = p; + + i = 0; + + for ( /* void */ ; p[i] != '\0'; i++) { + if (p[i] == '/') { + nxt_php_script.data = &p[i]; + nxt_php_root.len = i; + } + } + + nxt_php_path.len = i; + nxt_php_script.len = i - nxt_php_root.len; + + nxt_log_error(NXT_LOG_INFO, log, "php script \"%V\" root: \"%V\"", + &nxt_php_script, &nxt_php_root); + + return NXT_OK; + } + } + + nxt_log_error(NXT_LOG_ERR, log, "no option \"--php\" specified"); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_php_request_init(nxt_app_request_t *r) +{ + nxt_php_ctx_t *ctx; + + ctx = nxt_mem_zalloc(r->mem_pool, sizeof(nxt_php_ctx_t)); + if (nxt_slow_path(ctx == NULL)) { + return NXT_ERROR; + } + + r->ctx = ctx; + + return NXT_OK; +} + + + +nxt_int_t +nxt_php_request_header(nxt_app_request_t *r, nxt_app_header_field_t *field) +{ + nxt_php_ctx_t *ctx; + + static const u_char cookie[6] = "Cookie"; + static const u_char content_length[14] = "Content-Length"; + static const u_char content_type[12] = "Content-Type"; + + ctx = r->ctx; + + ctx->max_name = nxt_max(ctx->max_name, field->name.len); + + if (field->name.len == sizeof(cookie) + && nxt_memcasecmp(field->name.data, cookie, sizeof(cookie)) == 0) + { + ctx->cookie = &field->value; + + } else if (field->name.len == sizeof(content_length) + && nxt_memcasecmp(field->name.data, content_length, + sizeof(content_length)) == 0) + { + ctx->content_length = &field->value; + ctx->content_length_n = nxt_off_t_parse(field->value.data, + field->value.len); + + } else if (field->name.len == sizeof(content_type) + && nxt_memcasecmp(field->name.data, content_type, + sizeof(content_type)) == 0) + { + ctx->content_type = &field->value; + field->value.data[field->value.len] = '\0'; + } + + return NXT_OK; +} + + +#define ABS_MODE 1 + + +#if !ABS_MODE +static const u_char root[] = "/home/vbart/Development/tests/php/wordpress"; +#endif + + +nxt_int_t +nxt_php_handler(nxt_app_request_t *r) +{ + u_char *query; +#if !ABS_MODE + u_char *p; +#endif + nxt_php_ctx_t *ctx; + zend_file_handle file_handle; + +#if ABS_MODE + if (nxt_php_path.len == 0) { + return NXT_ERROR; + } +#endif + + r->header.path.data[r->header.path.len] = '\0'; + r->header.method.data[r->header.method.len] = '\0'; + + ctx = r->ctx; + + query = nxt_memchr(r->header.path.data, '?', r->header.path.len); + + if (query != NULL) { + ctx->script_name_len = query - r->header.path.data; + + ctx->query.data = query + 1; + ctx->query.len = r->header.path.data + r->header.path.len + - ctx->query.data; + + } else { + ctx->script_name_len = r->header.path.len; + } + +#if !ABS_MODE + ctx->script.len = sizeof(root) - 1 + ctx->script_name_len; + ctx->script.data = nxt_mem_nalloc(r->mem_pool, ctx->script.len + 1); + + if (nxt_slow_path(ctx->script.data == NULL)) { + return NXT_ERROR; + } + + p = nxt_cpymem(ctx->script.data, root, sizeof(root) - 1); + p = nxt_cpymem(p, r->header.path.data, ctx->script_name_len); + *p = '\0'; +#endif + + SG(server_context) = r; + SG(request_info).request_uri = (char *) r->header.path.data; + SG(request_info).request_method = (char *) r->header.method.data; + + SG(request_info).proto_num = 1001; + + SG(request_info).query_string = (char *) ctx->query.data; + SG(request_info).content_length = ctx->content_length_n; + + if (ctx->content_type != NULL) { + SG(request_info).content_type = (char *) ctx->content_type->data; + } + + SG(sapi_headers).http_response_code = 200; + + SG(request_info).path_translated = NULL; + + file_handle.type = ZEND_HANDLE_FILENAME; +#if ABS_MODE + file_handle.filename = (char *) nxt_php_path.data; +#else + file_handle.filename = (char *) ctx->script.data; +#endif + file_handle.free_filename = 0; + file_handle.opened_path = NULL; + +#if ABS_MODE + nxt_log_debug(r->log, "run script %V in absolute mode", &nxt_php_path); +#else + nxt_log_debug(r->log, "run script %V", &ctx->script); +#endif + + if (nxt_slow_path(php_request_startup() == FAILURE)) { + return NXT_ERROR; + } + + php_execute_script(&file_handle TSRMLS_CC); + php_request_shutdown(NULL); + + return NXT_OK; +} + + +static int +nxt_php_startup(sapi_module_struct *sapi_module) +{ + return php_module_startup(sapi_module, NULL, 0); +} + + +#ifdef NXT_PHP7 +static size_t +nxt_php_unbuffered_write(const char *str, size_t str_length TSRMLS_DC) +#else +static int +nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC) +#endif +{ + nxt_app_request_t *r; + + r = SG(server_context); + + nxt_app_write(r, (u_char *) str, str_length); + + return str_length; +} + + +static int +nxt_php_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC) +{ + size_t len; + nxt_app_request_t *r; + sapi_header_struct *h; + zend_llist_position zpos; + u_char *p, *status, buf[4096]; + + static const u_char default_repsonse[] + = "HTTP/1.1 200 OK\r\n" + "Server: nginman/0.1\r\n" + "Content-Type: text/html; charset=UTF-8\r\n" + "Connection: close\r\n" + "\r\n"; + + static const u_char default_headers[] + = "Server: nginman/0.1\r\n" + "Connection: close\r\n"; + + r = SG(server_context); + + if (SG(request_info).no_headers == 1) { + nxt_app_write(r, default_repsonse, sizeof(default_repsonse) - 1); + return SAPI_HEADER_SENT_SUCCESSFULLY; + } + + if (SG(sapi_headers).http_status_line) { + status = (u_char *) SG(sapi_headers).http_status_line + 9; + len = nxt_strlen(status); + + p = nxt_cpymem(buf, "HTTP/1.1 ", sizeof("HTTP/1.1 ") - 1); + p = nxt_cpymem(p, status, len); + *p++ = '\r'; *p++ = '\n'; + + } else if (SG(sapi_headers).http_response_code) { + p = nxt_cpymem(buf, "HTTP/1.1 ", sizeof("HTTP/1.1 ") - 1); + p = nxt_sprintf(p, buf + sizeof(buf), "%03d", + SG(sapi_headers).http_response_code); + *p++ = '\r'; *p++ = '\n'; + + } else { + p = nxt_cpymem(buf, "HTTP/1.1 200 OK\r\n", + sizeof("HTTP/1.1 200 OK\r\n") - 1); + } + + p = nxt_cpymem(p, default_headers, sizeof(default_headers) - 1); + + h = zend_llist_get_first_ex(&sapi_headers->headers, &zpos); + + while (h) { + p = nxt_cpymem(p, h->header, h->header_len); + *p++ = '\r'; *p++ = '\n'; + + h = zend_llist_get_next_ex(&sapi_headers->headers, &zpos); + } + + *p++ = '\r'; *p++ = '\n'; + + nxt_app_write(r, buf, p - buf); + + return SAPI_HEADER_SENT_SUCCESSFULLY; +} + + +#ifdef NXT_PHP7 +static size_t +nxt_php_read_post(char *buffer, size_t count_bytes TSRMLS_DC) +#else +static int +nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC) +#endif +{ + off_t rest; + size_t size; + ssize_t n; + nxt_err_t err; + nxt_php_ctx_t *ctx; + nxt_app_request_t *r; + + r = SG(server_context); + ctx = r->ctx; + + rest = ctx->content_length_n - SG(read_post_bytes); + + nxt_log_debug(r->log, "nxt_php_read_post %O", rest); + + if (rest == 0) { + return 0; + } + + size = 0; +#ifdef NXT_PHP7 + count_bytes = (size_t) nxt_min(rest, (off_t) count_bytes); +#else + count_bytes = (uint) nxt_min(rest, (off_t) count_bytes); +#endif + + if (r->body_preread.len != 0) { + size = nxt_min(r->body_preread.len, count_bytes); + + nxt_memcpy(buffer, r->body_preread.data, size); + + r->body_preread.len -= size; + r->body_preread.data += size; + + if (size == count_bytes) { + return size; + } + } + + nxt_log_debug(r->log, "recv %z", (size_t) count_bytes - size); + + n = recv(r->event_conn->socket.fd, buffer + size, count_bytes - size, 0); + + if (nxt_slow_path(n <= 0)) { + err = (n == 0) ? 0 : nxt_socket_errno; + + nxt_log_error(NXT_LOG_ERR, r->log, "recv(%d, %uz) failed %E", + r->event_conn->socket.fd, (size_t) count_bytes - size, + err); + + return size; + } + + return size + n; +} + + +static char * +nxt_php_read_cookies(TSRMLS_D) +{ + u_char *p; + nxt_php_ctx_t *ctx; + nxt_app_request_t *r; + + r = SG(server_context); + ctx = r->ctx; + + if (ctx->cookie == NULL) { + return NULL; + } + + p = ctx->cookie->data; + p[ctx->cookie->len] = '\0'; + + return (char *) p; +} + + +static void +nxt_php_register_variables(zval *track_vars_array TSRMLS_DC) +{ + u_char *var, *p, ch; + nxt_uint_t i, n; + nxt_php_ctx_t *ctx; + nxt_app_request_t *r; + nxt_app_header_field_t *fld; + + static const u_char prefix[5] = "HTTP_"; + + r = SG(server_context); + ctx = r->ctx; + + nxt_log_debug(r->log, "php register variables"); + + php_register_variable_safe((char *) "PHP_SELF", + (char *) r->header.path.data, + ctx->script_name_len, track_vars_array TSRMLS_CC); + + php_register_variable_safe((char *) "SERVER_PROTOCOL", + (char *) r->header.version.data, + r->header.version.len, track_vars_array TSRMLS_CC); + +#if ABS_MODE + php_register_variable_safe((char *) "SCRIPT_NAME", + (char *) nxt_php_script.data, + nxt_php_script.len, track_vars_array TSRMLS_CC); + + php_register_variable_safe((char *) "SCRIPT_FILENAME", + (char *) nxt_php_path.data, + nxt_php_path.len, track_vars_array TSRMLS_CC); + + php_register_variable_safe((char *) "DOCUMENT_ROOT", + (char *) nxt_php_root.data, + nxt_php_root.len, track_vars_array TSRMLS_CC); +#else + php_register_variable_safe((char *) "SCRIPT_NAME", + (char *) r->header.path.data, + ctx->script_name_len, track_vars_array TSRMLS_CC); + + php_register_variable_safe((char *) "SCRIPT_FILENAME", + (char *) ctx->script.data, ctx->script.len, + track_vars_array TSRMLS_CC); + + php_register_variable_safe((char *) "DOCUMENT_ROOT", (char *) root, + sizeof(root) - 1, track_vars_array TSRMLS_CC); +#endif + + php_register_variable_safe((char *) "REQUEST_METHOD", + (char *) r->header.method.data, + r->header.method.len, track_vars_array TSRMLS_CC); + + php_register_variable_safe((char *) "REQUEST_URI", + (char *) r->header.path.data, + r->header.path.len, track_vars_array TSRMLS_CC); + + if (ctx->query.data != NULL) { + php_register_variable_safe((char *) "QUERY_STRING", + (char *) ctx->query.data, + ctx->query.len, track_vars_array TSRMLS_CC); + } + + if (ctx->content_type != NULL) { + php_register_variable_safe((char *) "CONTENT_TYPE", + (char *) ctx->content_type->data, + ctx->content_type->len, track_vars_array TSRMLS_CC); + } + + if (ctx->content_length != NULL) { + php_register_variable_safe((char *) "CONTENT_LENGTH", + (char *) ctx->content_length->data, + ctx->content_length->len, track_vars_array TSRMLS_CC); + } + + var = nxt_mem_nalloc(r->mem_pool, sizeof(prefix) + ctx->max_name + 1); + + if (nxt_slow_path(var == NULL)) { + return; + } + + nxt_memcpy(var, prefix, sizeof(prefix)); + + for (i = 0; i < r->header.fields_num; i++) { + fld = &r->header.fields[i]; + p = var + sizeof(prefix); + + for (n = 0; n < fld->name.len; n++, p++) { + + ch = fld->name.data[n]; + + if (ch >= 'a' && ch <= 'z') { + *p = ch & ~0x20; + continue; + } + + if (ch == '-') { + *p = '_'; + continue; + } + + *p = ch; + } + + *p = '\0'; + + php_register_variable_safe((char *) var, (char *) fld->value.data, + fld->value.len, track_vars_array TSRMLS_CC); + } + + return; +} + + +static void +nxt_php_log_message(char *message TSRMLS_DC) +{ + return; +} diff --git a/src/nxt_polarssl.c b/src/nxt_polarssl.c new file mode 100644 index 00000000..fcee2da0 --- /dev/null +++ b/src/nxt_polarssl.c @@ -0,0 +1,118 @@ + +/* + * Copyright (C) NGINX, Inc. + * Copyright (C) Igor Sysoev + */ + +#include <nxt_main.h> +#include <polarssl/config.h> +#include <polarssl/ssl.h> +#include <polarssl/x509.h> +#include <polarssl/error.h> + + +typedef struct { + ssl_context ssl; + x509_cert certificate; + rsa_context key; +} nxt_polarssl_ctx_t; + + +static nxt_int_t nxt_polarssl_server_init(nxt_ssltls_conf_t *conf); +static void nxt_polarssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c); +static void nxt_polarssl_log_error(nxt_uint_t level, nxt_log_t *log, int err, + const char *fmt, ...); + + +nxt_ssltls_lib_t nxt_polarssl_lib = { + nxt_polarssl_server_init, + NULL, +}; + + +static nxt_int_t +nxt_polarssl_server_init(nxt_ssltls_conf_t *conf) +{ + int n; + nxt_thread_t *thr; + nxt_polarssl_ctx_t *ctx; + + thr = nxt_thread(); + + /* TODO: mem_pool */ + + ctx = nxt_zalloc(sizeof(nxt_polarssl_ctx_t)); + if (ctx == NULL) { + return NXT_ERROR; + } + + n = ssl_init(&ctx->ssl); + if (n != 0) { + nxt_polarssl_log_error(NXT_LOG_CRIT, thr->log, n, "ssl_init() failed"); + return NXT_ERROR; + } + + ssl_set_endpoint(&ctx->ssl, SSL_IS_SERVER ); + + conf->ctx = ctx; + conf->conn_init = nxt_polarssl_conn_init; + + n = x509parse_crtfile(&ctx->certificate, conf->certificate); + if (n != 0) { + nxt_polarssl_log_error(NXT_LOG_CRIT, thr->log, n, + "x509parse_crt(\"%V\") failed", + &conf->certificate); + goto fail; + } + + rsa_init(&ctx->key, RSA_PKCS_V15, 0); + + n = x509parse_keyfile(&ctx->key, conf->certificate_key, NULL); + if (n != 0) { + nxt_polarssl_log_error(NXT_LOG_CRIT, thr->log, n, + "x509parse_key(\"%V\") failed", + &conf->certificate_key); + goto fail; + } + + ssl_set_own_cert(&ctx->ssl, &ctx->certificate, &ctx->key); + + /* TODO: ciphers */ + + /* TODO: ca_certificate */ + + return NXT_OK; + +fail: + + return NXT_ERROR; +} + + +static void +nxt_polarssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c) +{ +} + + +static void +nxt_polarssl_log_error(nxt_uint_t level, nxt_log_t *log, int err, + const char *fmt, ...) +{ + va_list args; + u_char *p, *end, msg[NXT_MAX_ERROR_STR]; + + end = msg + NXT_MAX_ERROR_STR; + + va_start(args, fmt); + p = nxt_vsprintf(msg, end, fmt, args); + va_end(args); + + p = nxt_sprintf(p, end, " (%d: ", err); + + error_strerror(err, (char *) msg, p - msg); + + nxt_log_error(level, log, "%*s)", p - msg, msg); +} diff --git a/src/nxt_poll.c b/src/nxt_poll.c new file mode 100644 index 00000000..43cebe38 --- /dev/null +++ b/src/nxt_poll.c @@ -0,0 +1,750 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#define NXT_POLL_ADD 0 +#define NXT_POLL_CHANGE 1 +#define NXT_POLL_DELETE 2 + + +typedef struct { + /* + * A file descriptor is stored in hash entry to allow + * nxt_poll_fd_hash_test() to not dereference a pointer to + * nxt_event_fd_t which may be invalid if the file descriptor has + * been already closed and the nxt_event_fd_t's memory has been freed. + */ + nxt_socket_t fd; + + uint32_t index; + void *event; +} nxt_poll_hash_entry_t; + + +static nxt_event_set_t *nxt_poll_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +static void nxt_poll_free(nxt_event_set_t *event_set); +static void nxt_poll_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_poll_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_poll_drop_changes(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_enable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_enable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_disable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_disable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_poll_block_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_oneshot_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_oneshot_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_poll_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_uint_t op, nxt_uint_t events); +static nxt_int_t nxt_poll_commit_changes(nxt_thread_t *thr, + nxt_poll_event_set_t *ps); +static nxt_int_t nxt_poll_set_add(nxt_thread_t *thr, nxt_poll_event_set_t *ps, + nxt_poll_change_t *ch); +static nxt_int_t nxt_poll_set_change(nxt_thread_t *thr, + nxt_poll_event_set_t *ps, nxt_poll_change_t *ch); +static nxt_int_t nxt_poll_set_delete(nxt_thread_t *thr, + nxt_poll_event_set_t *ps, nxt_poll_change_t *ch); +static void nxt_poll_set_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout); +static nxt_poll_hash_entry_t *nxt_poll_fd_hash_get(nxt_poll_event_set_t *ps, + nxt_fd_t fd); +static nxt_int_t nxt_poll_fd_hash_test(nxt_lvlhsh_query_t *lhq, void *data); +static void nxt_poll_fd_hash_destroy(nxt_lvlhsh_t *lh); + + +const nxt_event_set_ops_t nxt_poll_event_set = { + "poll", + nxt_poll_create, + nxt_poll_free, + nxt_poll_enable, + nxt_poll_disable, + nxt_poll_disable, + nxt_poll_disable, + nxt_poll_enable_read, + nxt_poll_enable_write, + nxt_poll_disable_read, + nxt_poll_disable_write, + nxt_poll_block_read, + nxt_poll_block_write, + nxt_poll_oneshot_read, + nxt_poll_oneshot_write, + nxt_poll_enable_read, + NULL, + NULL, + NULL, + NULL, + nxt_poll_set_poll, + + &nxt_unix_event_conn_io, + + NXT_NO_FILE_EVENTS, + NXT_NO_SIGNAL_EVENTS, +}; + + +static const nxt_lvlhsh_proto_t nxt_poll_fd_hash_proto nxt_aligned(64) = +{ + NXT_LVLHSH_LARGE_MEMALIGN, + 0, + nxt_poll_fd_hash_test, + nxt_lvlhsh_alloc, + nxt_lvlhsh_free, +}; + + +static nxt_event_set_t * +nxt_poll_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + nxt_event_set_t *event_set; + nxt_poll_event_set_t *ps; + + event_set = nxt_zalloc(sizeof(nxt_poll_event_set_t)); + if (event_set == NULL) { + return NULL; + } + + ps = &event_set->poll; + + ps->mchanges = mchanges; + + ps->changes = nxt_malloc(sizeof(nxt_poll_change_t) * mchanges); + if (ps->changes == NULL) { + nxt_free(event_set); + return NULL; + } + + return event_set; +} + + +static void +nxt_poll_free(nxt_event_set_t *event_set) +{ + nxt_poll_event_set_t *ps; + + ps = &event_set->poll; + + nxt_main_log_debug("poll free"); + + nxt_free(ps->poll_set); + nxt_free(ps->changes); + nxt_poll_fd_hash_destroy(&ps->fd_hash); + nxt_free(ps); +} + + +static void +nxt_poll_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_DEFAULT; + ev->write = NXT_EVENT_DEFAULT; + + nxt_poll_change(event_set, ev, NXT_POLL_ADD, POLLIN | POLLOUT); +} + + +static void +nxt_poll_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_poll_drop_changes(event_set, ev); + /* + * A simple non-zero value POLLHUP is a flag to ignore error handling + * if the event is not present in poll set, because the event may be + * freed at the time when the NXT_POLL_DELETE change will be processed + * and correct event error_handler will not be available. + */ + nxt_poll_change(event_set, ev, NXT_POLL_DELETE, POLLHUP); +} + + +static void +nxt_poll_drop_changes(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_poll_change_t *dst, *src, *end; + nxt_poll_event_set_t *ps; + + ps = &event_set->poll; + + dst = ps->changes; + end = dst + ps->nchanges; + + for (src = dst; src < end; src++) { + + if (src->event == ev) { + continue; + } + + if (dst != src) { + *dst = *src; + } + + dst++; + } + + ps->nchanges -= end - dst; +} + + +static void +nxt_poll_enable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->read = NXT_EVENT_DEFAULT; + + if (ev->write == NXT_EVENT_INACTIVE) { + op = NXT_POLL_ADD; + events = POLLIN; + + } else { + op = NXT_POLL_CHANGE; + events = POLLIN | POLLOUT; + } + + nxt_poll_change(event_set, ev, op, events); +} + + +static void +nxt_poll_enable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->write = NXT_EVENT_DEFAULT; + + if (ev->read == NXT_EVENT_INACTIVE) { + op = NXT_POLL_ADD; + events = POLLOUT; + + } else { + op = NXT_POLL_CHANGE; + events = POLLIN | POLLOUT; + } + + nxt_poll_change(event_set, ev, op, events); +} + + +static void +nxt_poll_disable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->read = NXT_EVENT_INACTIVE; + + if (ev->write == NXT_EVENT_INACTIVE) { + op = NXT_POLL_DELETE; + events = 0; + + } else { + op = NXT_POLL_CHANGE; + events = POLLOUT; + } + + nxt_poll_change(event_set, ev, op, events); +} + + +static void +nxt_poll_disable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->write = NXT_EVENT_INACTIVE; + + if (ev->read == NXT_EVENT_INACTIVE) { + op = NXT_POLL_DELETE; + events = 0; + + } else { + op = NXT_POLL_CHANGE; + events = POLLIN; + } + + nxt_poll_change(event_set, ev, op, events); +} + + +static void +nxt_poll_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + nxt_poll_disable_read(event_set, ev); + } +} + + +static void +nxt_poll_block_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write != NXT_EVENT_INACTIVE) { + nxt_poll_disable_write(event_set, ev); + } +} + + +static void +nxt_poll_oneshot_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op; + + op = (ev->read == NXT_EVENT_INACTIVE && ev->write == NXT_EVENT_INACTIVE) ? + NXT_POLL_ADD : NXT_POLL_CHANGE; + + ev->read = NXT_EVENT_ONESHOT; + ev->write = NXT_EVENT_INACTIVE; + + nxt_poll_change(event_set, ev, op, POLLIN); +} + + +static void +nxt_poll_oneshot_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op; + + op = (ev->read == NXT_EVENT_INACTIVE && ev->write == NXT_EVENT_INACTIVE) ? + NXT_POLL_ADD : NXT_POLL_CHANGE; + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_ONESHOT; + + nxt_poll_change(event_set, ev, op, POLLOUT); +} + + +/* + * poll changes are batched to improve instruction and data cache + * locality of several lvlhsh operations followed by poll() call. + */ + +static void +nxt_poll_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, nxt_uint_t op, + nxt_uint_t events) +{ + nxt_poll_change_t *ch; + nxt_poll_event_set_t *ps; + + nxt_log_debug(ev->log, "poll change: fd:%d op:%d ev:%XD", + ev->fd, op, events); + + ps = &event_set->poll; + + if (ps->nchanges >= ps->mchanges) { + (void) nxt_poll_commit_changes(nxt_thread(), ps); + } + + ch = &ps->changes[ps->nchanges++]; + ch->op = op; + ch->fd = ev->fd; + ch->events = events; + ch->event = ev; +} + + +static nxt_int_t +nxt_poll_commit_changes(nxt_thread_t *thr, nxt_poll_event_set_t *ps) +{ + nxt_int_t ret; + nxt_event_fd_t *ev; + nxt_poll_change_t *ch, *end; + + nxt_log_debug(thr->log, "poll changes:%ui", ps->nchanges); + + ret = NXT_OK; + ch = ps->changes; + end = ch + ps->nchanges; + + do { + ev = ch->event; + + switch (ch->op) { + + case NXT_POLL_ADD: + if (nxt_fast_path(nxt_poll_set_add(thr, ps, ch) == NXT_OK)) { + goto next; + } + break; + + case NXT_POLL_CHANGE: + if (nxt_fast_path(nxt_poll_set_change(thr, ps, ch) == NXT_OK)) { + goto next; + } + break; + + case NXT_POLL_DELETE: + if (nxt_fast_path(nxt_poll_set_delete(thr, ps, ch) == NXT_OK)) { + goto next; + } + break; + } + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + ev->error_handler, ev, ev->data, ev->log); + + ret = NXT_ERROR; + + next: + + ch++; + + } while (ch < end); + + ps->nchanges = 0; + + return ret; +} + + +static nxt_int_t +nxt_poll_set_add(nxt_thread_t *thr, nxt_poll_event_set_t *ps, + nxt_poll_change_t *ch) +{ + nxt_uint_t max_nfds; + struct pollfd *pfd; + nxt_lvlhsh_query_t lhq; + nxt_poll_hash_entry_t *phe; + + nxt_log_debug(thr->log, "poll add event: fd:%d ev:%04Xi", + ch->fd, ch->events); + + if (ps->nfds >= ps->max_nfds) { + max_nfds = ps->max_nfds + 512; /* 4K */ + + pfd = nxt_realloc(ps->poll_set, sizeof(struct pollfd) * max_nfds); + if (nxt_slow_path(pfd == NULL)) { + return NXT_ERROR; + } + + ps->poll_set = pfd; + ps->max_nfds = max_nfds; + } + + phe = nxt_malloc(sizeof(nxt_poll_hash_entry_t)); + if (nxt_slow_path(phe == NULL)) { + return NXT_ERROR; + } + + phe->fd = ch->fd; + phe->index = ps->nfds; + phe->event = ch->event; + + pfd = &ps->poll_set[ps->nfds++]; + pfd->fd = ch->fd; + pfd->events = ch->events; + pfd->revents = 0; + + lhq.key_hash = nxt_murmur_hash2(&ch->fd, sizeof(nxt_fd_t)); + lhq.replace = 0; + lhq.key.len = sizeof(nxt_fd_t); + lhq.key.data = (u_char *) &ch->fd; + lhq.value = phe; + lhq.proto = &nxt_poll_fd_hash_proto; + lhq.data = ps->poll_set; + + if (nxt_fast_path(nxt_lvlhsh_insert(&ps->fd_hash, &lhq) == NXT_OK)) { + return NXT_OK; + } + + nxt_free(phe); + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_poll_set_change(nxt_thread_t *thr, nxt_poll_event_set_t *ps, + nxt_poll_change_t *ch) +{ + nxt_poll_hash_entry_t *phe; + + nxt_log_debug(thr->log, "poll change event: fd:%d ev:%04Xi", + ch->fd, ch->events); + + phe = nxt_poll_fd_hash_get(ps, ch->fd); + + if (nxt_fast_path(phe != NULL)) { + ps->poll_set[phe->index].events = ch->events; + return NXT_OK; + } + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_poll_set_delete(nxt_thread_t *thr, nxt_poll_event_set_t *ps, + nxt_poll_change_t *ch) +{ + nxt_uint_t index; + nxt_lvlhsh_query_t lhq; + nxt_poll_hash_entry_t *phe; + + nxt_log_debug(thr->log, "poll delete event: fd:%d", ch->fd); + + lhq.key_hash = nxt_murmur_hash2(&ch->fd, sizeof(nxt_fd_t)); + lhq.key.len = sizeof(nxt_fd_t); + lhq.key.data = (u_char *) &ch->fd; + lhq.proto = &nxt_poll_fd_hash_proto; + lhq.data = ps->poll_set; + + if (nxt_slow_path(nxt_lvlhsh_delete(&ps->fd_hash, &lhq) != NXT_OK)) { + /* + * Ignore NXT_DECLINED error if ch->events + * has the special value POLLHUP. + */ + return (ch->events != 0) ? NXT_OK : NXT_ERROR; + } + + phe = lhq.value; + + index = phe->index; + ps->nfds--; + + if (index != ps->nfds) { + ps->poll_set[index] = ps->poll_set[ps->nfds]; + + phe = nxt_poll_fd_hash_get(ps, ps->poll_set[ps->nfds].fd); + + phe->index = index; + } + + nxt_free(lhq.value); + + return NXT_OK; +} + + +static void +nxt_poll_set_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout) +{ + int nevents; + nxt_fd_t fd; + nxt_err_t err; + nxt_bool_t error; + nxt_uint_t i, events, level; + struct pollfd *pfd; + nxt_event_fd_t *ev; + nxt_poll_event_set_t *ps; + nxt_poll_hash_entry_t *phe; + + ps = &event_set->poll; + + if (ps->nchanges != 0) { + if (nxt_poll_commit_changes(nxt_thread(), ps) != NXT_OK) { + /* Error handlers have been enqueued on failure. */ + timeout = 0; + } + } + + nxt_log_debug(thr->log, "poll() events:%ui timeout:%M", ps->nfds, timeout); + + nevents = poll(ps->poll_set, ps->nfds, timeout); + + err = (nevents == -1) ? nxt_errno : 0; + + nxt_thread_time_update(thr); + + nxt_log_debug(thr->log, "poll(): %d", nevents); + + if (nevents == -1) { + level = (err == NXT_EINTR) ? NXT_LOG_INFO : NXT_LOG_ALERT; + nxt_log_error(level, thr->log, "poll() failed %E", err); + return; + } + + for (i = 0; i < ps->nfds && nevents != 0; i++) { + + pfd = &ps->poll_set[i]; + events = pfd->revents; + + if (events == 0) { + continue; + } + + fd = pfd->fd; + + phe = nxt_poll_fd_hash_get(ps, fd); + + if (nxt_slow_path(phe == NULL)) { + nxt_log_alert(thr->log, + "poll() returned invalid fd:%d ev:%04Xd rev:%04uXi", + fd, pfd->events, events); + + /* Mark the poll entry to ignore it by the kernel. */ + pfd->fd = -1; + goto next; + } + + ev = phe->event; + + nxt_log_debug(ev->log, "poll: fd:%d ev:%04uXi rd:%d %wr:%d", + fd, events, ev->read, ev->write); + + if (nxt_slow_path((events & POLLNVAL) != 0)) { + nxt_log_alert(ev->log, "poll() error fd:%d ev:%04Xd rev:%04uXi", + fd, pfd->events, events); + + /* Mark the poll entry to ignore it by the kernel. */ + pfd->fd = -1; + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + ev->error_handler, ev, ev->data, ev->log); + goto next; + } + + /* + * On a socket's remote end close: + * + * Linux, FreeBSD, and Solaris set POLLIN; + * MacOSX sets POLLIN and POLLHUP; + * NetBSD sets POLLIN, and poll(2) claims this explicitly: + * + * If the remote end of a socket is closed, poll() + * returns a POLLIN event, rather than a POLLHUP. + * + * On error: + * + * Linux sets POLLHUP and POLLERR only; + * FreeBSD adds POLLHUP to POLLIN or POLLOUT, although poll(2) + * claims the opposite: + * + * Note that POLLHUP and POLLOUT should never be + * present in the revents bitmask at the same time. + * + * Solaris and NetBSD do not add POLLHUP or POLLERR; + * MacOSX sets POLLHUP only. + * + * If an implementation sets POLLERR or POLLHUP only without POLLIN + * or POLLOUT, the "error" variable enqueues only one active handler. + */ + + error = (((events & (POLLERR | POLLHUP)) != 0) + && ((events & (POLLIN | POLLOUT)) == 0)); + + if ((events & POLLIN) || (error && ev->read_handler != NULL)) { + error = 0; + ev->read_ready = 1; + + if (ev->read == NXT_EVENT_ONESHOT) { + ev->read = NXT_EVENT_INACTIVE; + nxt_poll_change(event_set, ev, NXT_POLL_DELETE, 0); + } + + nxt_thread_work_queue_add(thr, ev->read_work_queue, + ev->read_handler, ev, ev->data, ev->log); + } + + if ((events & POLLOUT) || (error && ev->write_handler != NULL)) { + ev->write_ready = 1; + + if (ev->write == NXT_EVENT_ONESHOT) { + ev->write = NXT_EVENT_INACTIVE; + nxt_poll_change(event_set, ev, NXT_POLL_DELETE, 0); + } + + nxt_thread_work_queue_add(thr, ev->write_work_queue, + ev->write_handler, ev, ev->data, ev->log); + } + + next: + + nevents--; + } +} + + +static nxt_poll_hash_entry_t * +nxt_poll_fd_hash_get(nxt_poll_event_set_t *ps, nxt_fd_t fd) +{ + nxt_lvlhsh_query_t lhq; + nxt_poll_hash_entry_t *phe; + + lhq.key_hash = nxt_murmur_hash2(&fd, sizeof(nxt_fd_t)); + lhq.key.len = sizeof(nxt_fd_t); + lhq.key.data = (u_char *) &fd; + lhq.proto = &nxt_poll_fd_hash_proto; + lhq.data = ps->poll_set; + + if (nxt_lvlhsh_find(&ps->fd_hash, &lhq) == NXT_OK) { + phe = lhq.value; + return phe; + } + + nxt_thread_log_alert("fd %d not found in hash", fd); + + return NULL; +} + + +static nxt_int_t +nxt_poll_fd_hash_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + struct pollfd *poll_set; + nxt_poll_hash_entry_t *phe; + + phe = data; + + if (*(nxt_fd_t *) lhq->key.data == phe->fd) { + poll_set = lhq->data; + + if (nxt_fast_path(phe->fd == poll_set[phe->index].fd)) { + return NXT_OK; + } + + nxt_thread_log_alert("fd %d in hash mismatches fd %d in poll set", + phe->fd, poll_set[phe->index].fd); + } + + return NXT_DECLINED; +} + + +static void +nxt_poll_fd_hash_destroy(nxt_lvlhsh_t *lh) +{ + nxt_lvlhsh_each_t lhe; + nxt_lvlhsh_query_t lhq; + nxt_poll_hash_entry_t *phe; + + nxt_memzero(&lhe, sizeof(nxt_lvlhsh_each_t)); + lhe.proto = &nxt_poll_fd_hash_proto; + lhq.proto = &nxt_poll_fd_hash_proto; + + for ( ;; ) { + phe = nxt_lvlhsh_each(lh, &lhe); + + if (phe == NULL) { + return; + } + + lhq.key_hash = nxt_murmur_hash2(&phe->fd, sizeof(nxt_fd_t)); + lhq.key.len = sizeof(nxt_fd_t); + lhq.key.data = (u_char *) &phe->fd; + + if (nxt_lvlhsh_delete(lh, &lhq) != NXT_OK) { + nxt_thread_log_alert("event fd %d not found in hash", phe->fd); + } + + nxt_free(phe); + } +} diff --git a/src/nxt_pollset.c b/src/nxt_pollset.c new file mode 100644 index 00000000..8d05e560 --- /dev/null +++ b/src/nxt_pollset.c @@ -0,0 +1,627 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * pollset has been introduced in AIX 5L 5.3. + * + * pollset_create() returns a pollset_t descriptor which is not + * a file descriptor, so it cannot be added to another pollset. + * The first pollset_create() call returns 0. + */ + + +#define NXT_POLLSET_ADD 0 +#define NXT_POLLSET_UPDATE 1 +#define NXT_POLLSET_CHANGE 2 +#define NXT_POLLSET_DELETE 3 + + +static nxt_event_set_t *nxt_pollset_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +static void nxt_pollset_free(nxt_event_set_t *event_set); +static void nxt_pollset_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_pollset_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_pollset_enable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_enable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_disable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_disable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_block_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_block_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_oneshot_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_oneshot_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_pollset_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_uint_t op, nxt_uint_t events); +static nxt_int_t nxt_pollset_commit_changes(nxt_thread_t *thr, + nxt_pollset_event_set_t *ps); +static void nxt_pollset_change_error(nxt_thread_t *thr, + nxt_pollset_event_set_t *ps, nxt_event_fd_t *ev); +static void nxt_pollset_remove(nxt_thread_t *thr, nxt_pollset_event_set_t *ps, + nxt_fd_t fd); +static nxt_int_t nxt_pollset_write(nxt_thread_t *thr, int pollset, + struct poll_ctl *ctl, int n); +static void nxt_pollset_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout); + + +const nxt_event_set_ops_t nxt_pollset_event_set = { + "pollset", + nxt_pollset_create, + nxt_pollset_free, + nxt_pollset_enable, + nxt_pollset_disable, + nxt_pollset_disable, + nxt_pollset_disable, + nxt_pollset_enable_read, + nxt_pollset_enable_write, + nxt_pollset_disable_read, + nxt_pollset_disable_write, + nxt_pollset_block_read, + nxt_pollset_block_write, + nxt_pollset_oneshot_read, + nxt_pollset_oneshot_write, + nxt_pollset_enable_read, + NULL, + NULL, + NULL, + NULL, + nxt_pollset_poll, + + &nxt_unix_event_conn_io, + + NXT_NO_FILE_EVENTS, + NXT_NO_SIGNAL_EVENTS, +}; + + +static nxt_event_set_t * +nxt_pollset_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + nxt_event_set_t *event_set; + nxt_pollset_event_set_t *ps; + + event_set = nxt_zalloc(sizeof(nxt_pollset_event_set_t)); + if (event_set == NULL) { + return NULL; + } + + ps = &event_set->pollset; + + ps->pollset = -1; + ps->mchanges = mchanges; + ps->mevents = mevents; + + ps->pollset_changes = nxt_malloc(sizeof(nxt_pollset_change_t) * mchanges); + if (ps->pollset_changes == NULL) { + goto fail; + } + + /* + * NXT_POLLSET_CHANGE requires two struct poll_ctl's + * for PS_DELETE and subsequent PS_ADD. + */ + ps->changes = nxt_malloc(2 * sizeof(struct poll_ctl) * mchanges); + if (ps->changes == NULL) { + goto fail; + } + + ps->events = nxt_malloc(sizeof(struct pollfd) * mevents); + if (ps->events == NULL) { + goto fail; + } + + ps->pollset = pollset_create(-1); + if (ps->pollset == -1) { + nxt_main_log_emerg("pollset_create() failed %E", nxt_errno); + goto fail; + } + + nxt_main_log_debug("pollset_create(): %d", ps->pollset); + + return event_set; + +fail: + + nxt_pollset_free(event_set); + + return NULL; +} + + +static void +nxt_pollset_free(nxt_event_set_t *event_set) +{ + nxt_pollset_event_set_t *ps; + + ps = &event_set->pollset; + + nxt_main_log_debug("pollset %d free", ps->pollset); + + if (ps->pollset != -1) { + if (pollset_destroy(ps->pollset) != 0) { + nxt_main_log_emerg("pollset_destroy(%d) failed %E", + ps->pollset, nxt_errno); + } + } + + nxt_free(ps->events); + nxt_free(ps->changes); + nxt_free(ps->pollset_changes); + nxt_event_set_fd_hash_destroy(&ps->fd_hash); + nxt_free(ps); +} + + +static void +nxt_pollset_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_DEFAULT; + ev->write = NXT_EVENT_DEFAULT; + + nxt_pollset_change(event_set, ev, NXT_POLLSET_ADD, POLLIN | POLLOUT); +} + + +/* + * A closed descriptor must be deleted from a pollset, otherwise next + * pollset_poll() will return POLLNVAL on it. However, pollset_ctl() + * allows to delete the already closed file descriptor from the pollset + * using PS_DELETE, so the removal can be batched, pollset_ctl(2): + * + * After a file descriptor is added to a pollset, the file descriptor will + * not be removed until a pollset_ctl call with the cmd of PS_DELETE is + * executed. The file descriptor remains in the pollset even if the file + * descriptor is closed. A pollset_poll operation on a pollset containing + * a closed file descriptor returns a POLLNVAL event for that file + * descriptor. If the file descriptor is later allocated to a new object, + * the new object will be polled on future pollset_poll calls. + */ + +static void +nxt_pollset_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE || ev->write != NXT_EVENT_INACTIVE) { + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_pollset_change(event_set, ev, NXT_POLLSET_DELETE, 0); + } +} + + +static void +nxt_pollset_enable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + if (ev->read != NXT_EVENT_BLOCKED) { + + events = POLLIN; + + if (ev->write == NXT_EVENT_INACTIVE) { + op = NXT_POLLSET_ADD; + + } else if (ev->write == NXT_EVENT_BLOCKED) { + ev->write = NXT_EVENT_INACTIVE; + op = NXT_POLLSET_CHANGE; + + } else { + op = NXT_POLLSET_UPDATE; + events = POLLIN | POLLOUT; + } + + nxt_pollset_change(event_set, ev, op, events); + } + + ev->read = NXT_EVENT_DEFAULT; +} + + +static void +nxt_pollset_enable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + if (ev->write != NXT_EVENT_BLOCKED) { + + events = POLLOUT; + + if (ev->read == NXT_EVENT_INACTIVE) { + op = NXT_POLLSET_ADD; + + } else if (ev->read == NXT_EVENT_BLOCKED) { + ev->read = NXT_EVENT_INACTIVE; + op = NXT_POLLSET_CHANGE; + + } else { + op = NXT_POLLSET_UPDATE; + events = POLLIN | POLLOUT; + } + + nxt_pollset_change(event_set, ev, op, events); + } + + ev->write = NXT_EVENT_DEFAULT; +} + + +static void +nxt_pollset_disable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->read = NXT_EVENT_INACTIVE; + + if (ev->write <= NXT_EVENT_BLOCKED) { + ev->write = NXT_EVENT_INACTIVE; + op = NXT_POLLSET_DELETE; + events = POLLREMOVE; + + } else { + op = NXT_POLLSET_CHANGE; + events = POLLOUT; + } + + nxt_pollset_change(event_set, ev, op, events); +} + + +static void +nxt_pollset_disable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_uint_t op, events; + + ev->write = NXT_EVENT_INACTIVE; + + if (ev->read <= NXT_EVENT_BLOCKED) { + ev->read = NXT_EVENT_INACTIVE; + op = NXT_POLLSET_DELETE; + events = POLLREMOVE; + + } else { + op = NXT_POLLSET_CHANGE; + events = POLLIN; + } + + nxt_pollset_change(event_set, ev, op, events); +} + + +static void +nxt_pollset_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + ev->read = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_pollset_block_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write != NXT_EVENT_INACTIVE) { + ev->write = NXT_EVENT_BLOCKED; + } +} + + +static void +nxt_pollset_oneshot_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_pollset_enable_read(event_set, ev); + + ev->read = NXT_EVENT_ONESHOT; +} + + +static void +nxt_pollset_oneshot_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_pollset_enable_write(event_set, ev); + + ev->write = NXT_EVENT_ONESHOT; +} + + +/* + * PS_ADD adds only a new file descriptor to a pollset. + * PS_DELETE removes a file descriptor from a pollset. + * + * PS_MOD can add a new file descriptor or modify events for a file + * descriptor which is already in a pollset. However, modified events + * are always ORed, so to delete an event for a file descriptor, + * the file descriptor must be removed using PS_DELETE and then + * added again without the event. + */ + +static void +nxt_pollset_change(nxt_event_set_t *event_set, nxt_event_fd_t *ev, + nxt_uint_t op, nxt_uint_t events) +{ + nxt_pollset_change_t *ch; + nxt_pollset_event_set_t *ps; + + ps = &event_set->pollset; + + nxt_log_debug(ev->log, "pollset %d change fd:%d op:%ui ev:%04Xi", + ps->pollset, ev->fd, op, events); + + if (ps->nchanges >= ps->mchanges) { + (void) nxt_pollset_commit_changes(nxt_thread(), ps); + } + + ch = &ps->pollset_changes[ps->nchanges++]; + ch->op = op; + ch->cmd = (op == NXT_POLLSET_DELETE) ? PS_DELETE : PS_MOD; + ch->fd = ev->fd; + ch->events = events; + ch->event = ev; +} + + +static nxt_int_t +nxt_pollset_commit_changes(nxt_thread_t *thr, nxt_pollset_event_set_t *ps) +{ + size_t n; + nxt_int_t ret, retval; + struct poll_ctl *ctl; + nxt_pollset_change_t *ch, *end; + + nxt_log_debug(thr->log, "pollset %d changes:%ui", + ps->pollset, ps->nchanges); + + retval = NXT_OK; + n = 0; + ch = ps->pollset_changes; + end = ch + ps->nchanges; + + do { + nxt_log_debug(thr->log, "pollset fd:%d op:%d ev:%04Xd", + ch->fd, ch->op, ch->events); + + if (ch->op == NXT_POLLSET_CHANGE) { + ctl = &ps->changes[n++]; + ctl->cmd = PS_DELETE; + ctl->events = 0; + ctl->fd = ch->fd; + } + + ctl = &ps->changes[n++]; + ctl->cmd = ch->cmd; + ctl->events = ch->events; + ctl->fd = ch->fd; + + ch++; + + } while (ch < end); + + ch = ps->pollset_changes; + end = ch + ps->nchanges; + + ret = nxt_pollset_write(thr, ps->pollset, ps->changes, n); + + if (nxt_slow_path(ret != NXT_OK)) { + do { + nxt_pollset_change_error(thr, ps, ch->event); + ch++; + } while (ch < end); + + ps->nchanges = 0; + + return NXT_ERROR; + } + + do { + if (ch->op == NXT_POLLSET_ADD) { + ret = nxt_event_set_fd_hash_add(&ps->fd_hash, ch->fd, ch->event); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_pollset_change_error(thr, ps, ch->event); + retval = NXT_ERROR; + } + + } else if (ch->op == NXT_POLLSET_DELETE) { + nxt_event_set_fd_hash_delete(&ps->fd_hash, ch->fd, 0); + } + + /* Nothing to do for NXT_POLLSET_UPDATE and NXT_POLLSET_CHANGE. */ + + ch++; + + } while (ch < end); + + ps->nchanges = 0; + + return retval; +} + + +static void +nxt_pollset_change_error(nxt_thread_t *thr, nxt_pollset_event_set_t *ps, + nxt_event_fd_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + ev->error_handler, ev, ev->data, ev->log); + + nxt_event_set_fd_hash_delete(&ps->fd_hash, ev->fd, 1); + + nxt_pollset_remove(thr, ps, ev->fd); +} + + +static void +nxt_pollset_remove(nxt_thread_t *thr, nxt_pollset_event_set_t *ps, nxt_fd_t fd) +{ + int n; + struct pollfd pfd; + struct poll_ctl ctl; + + pfd.fd = fd; + pfd.events = 0; + pfd.revents = 0; + + n = pollset_query(ps->pollset, &pfd); + + nxt_thread_log_debug("pollset_query(%d, %d): %d", ps->pollset, fd, n); + + if (n == 0) { + /* The file descriptor is not in the pollset. */ + return; + } + + if (n == -1) { + nxt_thread_log_alert("pollset_query(%d, %d) failed %E", + ps->pollset, fd, nxt_errno); + /* Fall through. */ + } + + /* n == 1: The file descriptor is in the pollset. */ + + nxt_thread_log_debug("pollset %d remove fd:%d", ps->pollset, fd); + + ctl.cmd = PS_DELETE; + ctl.events = 0; + ctl.fd = fd; + + nxt_pollset_write(thr, ps->pollset, &ctl, 1); +} + + +static nxt_int_t +nxt_pollset_write(nxt_thread_t *thr, int pollset, struct poll_ctl *ctl, int n) +{ + nxt_thread_log_debug("pollset_ctl(%d) changes:%d", pollset, n); + + nxt_set_errno(0); + + n = pollset_ctl(pollset, ctl, n); + + if (nxt_fast_path(n == 0)) { + return NXT_OK; + } + + nxt_log_alert(thr->log, "pollset_ctl(%d) failed: %d %E", + pollset, n, nxt_errno); + + return NXT_ERROR; +} + + +static void +nxt_pollset_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout) +{ + int nevents; + nxt_fd_t fd; + nxt_int_t i; + nxt_err_t err; + nxt_uint_t events, level; + struct pollfd *pfd; + nxt_event_fd_t *ev; + nxt_pollset_event_set_t *ps; + + ps = &event_set->pollset; + + if (ps->nchanges != 0) { + if (nxt_pollset_commit_changes(thr, ps) != NXT_OK) { + /* Error handlers have been enqueued on failure. */ + timeout = 0; + } + } + + nxt_log_debug(thr->log, "pollset_poll(%d) timeout:%M", + ps->pollset, timeout); + + nevents = pollset_poll(ps->pollset, ps->events, ps->mevents, timeout); + + err = (nevents == -1) ? nxt_errno : 0; + + nxt_thread_time_update(thr); + + nxt_log_debug(thr->log, "pollset_poll(%d): %d", ps->pollset, nevents); + + if (nevents == -1) { + level = (err == NXT_EINTR) ? NXT_LOG_INFO : NXT_LOG_ALERT; + nxt_log_error(level, thr->log, "pollset_poll(%d) failed %E", + ps->pollset, err); + return; + } + + for (i = 0; i < nevents; i++) { + + pfd = &ps->events[i]; + fd = pfd->fd; + events = pfd->revents; + + ev = nxt_event_set_fd_hash_get(&ps->fd_hash, fd); + + if (nxt_slow_path(ev == NULL)) { + nxt_log_alert(thr->log, "pollset_poll(%d) returned invalid " + "fd:%d ev:%04Xd rev:%04uXi", + ps->pollset, fd, pfd->events, events); + + nxt_pollset_remove(thr, ps, fd); + continue; + } + + nxt_log_debug(ev->log, "pollset: fd:%d ev:%04uXi", fd, events); + + if (nxt_slow_path(events & (POLLERR | POLLHUP | POLLNVAL)) != 0) { + nxt_log_alert(ev->log, + "pollset_poll(%d) error fd:%d ev:%04Xd rev:%04uXi", + ps->pollset, fd, pfd->events, events); + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + ev->error_handler, ev, ev->data, ev->log); + continue; + } + + if (events & POLLIN) { + ev->read_ready = 1; + + if (ev->read != NXT_EVENT_BLOCKED) { + + if (ev->read == NXT_EVENT_ONESHOT) { + nxt_pollset_disable_read(event_set, ev); + } + + nxt_thread_work_queue_add(thr, ev->read_work_queue, + ev->read_handler, + ev, ev->data, ev->log); + } + } + + if (events & POLLOUT) { + ev->write_ready = 1; + + if (ev->write != NXT_EVENT_BLOCKED) { + + if (ev->write == NXT_EVENT_ONESHOT) { + nxt_pollset_disable_write(event_set, ev); + } + + nxt_thread_work_queue_add(thr, ev->write_work_queue, + ev->write_handler, + ev, ev->data, ev->log); + } + } + } +} diff --git a/src/nxt_process.c b/src/nxt_process.c new file mode 100644 index 00000000..b79e4388 --- /dev/null +++ b/src/nxt_process.c @@ -0,0 +1,436 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_user_groups_get(nxt_user_cred_t *uc); + + +/* A cached process pid. */ +nxt_pid_t nxt_pid; + +/* An original parent process pid. */ +nxt_pid_t nxt_ppid; + + +nxt_pid_t +nxt_process_create(nxt_process_start_t start, void *data, const char *name) +{ + nxt_pid_t pid; + nxt_thread_t *thr; + + thr = nxt_thread(); + + pid = fork(); + + switch (pid) { + + case -1: + nxt_log_alert(thr->log, "fork() failed while creating \"%s\" %E", + name, nxt_errno); + break; + + case 0: + /* A child. */ + nxt_pid = getpid(); + + /* Clean inherited cached thread tid. */ + thr->tid = 0; + + start(data); + break; + + default: + /* A parent. */ + nxt_log_debug(thr->log, "fork(): %PI", pid); + break; + } + + return pid; +} + + +#if (NXT_HAVE_POSIX_SPAWN) + +/* + * Linux glibc 2.2 posix_spawn() is implemented via fork()/execve(). + * Linux glibc 2.4 posix_spawn() without file actions and spawn + * attributes uses vfork()/execve(). + * + * On FreeBSD 8.0 posix_spawn() is implemented via vfork()/execve(). + * + * Solaris 10: + * In the Solaris 10 OS, posix_spawn() is currently implemented using + * private-to-libc vfork(), execve(), and exit() functions. They are + * identical to regular vfork(), execve(), and exit() in functionality, + * but they are not exported from libc and therefore don't cause the + * deadlock-in-the-dynamic-linker problem that any multithreaded code + * outside of libc that calls vfork() can cause. + * + * On MacOSX 10.5 (Leoprad) and NetBSD 6.0 posix_spawn() is implemented + * as syscall. + */ + +nxt_pid_t +nxt_process_execute(char *name, char **argv, char **envp) +{ + nxt_pid_t pid; + + nxt_thread_log_debug("posix_spawn(\"%s\")", name); + + if (posix_spawn(&pid, name, NULL, NULL, argv, envp) != 0) { + nxt_thread_log_alert("posix_spawn(\"%s\") failed %E", name, nxt_errno); + return -1; + } + + return pid; +} + +#else + +nxt_pid_t +nxt_process_execute(char *name, char **argv, char **envp) +{ + nxt_pid_t pid; + + /* + * vfork() is better than fork() because: + * it is faster several times; + * its execution time does not depend on private memory mapping size; + * it has lesser chances to fail due to the ENOMEM error. + */ + + pid = vfork(); + + switch (pid) { + + case -1: + nxt_thread_log_alert("vfork() failed while executing \"%s\" %E", + name, nxt_errno); + break; + + case 0: + /* A child. */ + nxt_thread_log_debug("execve(\"%s\")", name); + + (void) execve(name, argv, envp); + + nxt_thread_log_alert("execve(\"%s\") failed %E", name, nxt_errno); + + exit(1); + break; + + default: + /* A parent. */ + nxt_thread_log_debug("vfork(): %PI", pid); + break; + } + + return pid; +} + +#endif + + +nxt_int_t +nxt_process_daemon(void) +{ + nxt_fd_t fd; + nxt_pid_t pid; + const char *msg; + nxt_thread_t *thr; + + thr = nxt_thread(); + + /* + * fork() followed by a parent process's exit() detaches a child process + * from an init script or terminal shell process which has started the + * parent process and allows the child process to run in background. + */ + + pid = fork(); + + switch (pid) { + + case -1: + msg = "fork() failed %E"; + goto fail; + + case 0: + /* A child. */ + break; + + default: + /* A parent. */ + nxt_log_debug(thr->log, "fork(): %PI", pid); + exit(0); + nxt_unreachable(); + } + + nxt_pid = getpid(); + + /* Clean inherited cached thread tid. */ + thr->tid = 0; + + nxt_log_debug(thr->log, "daemon"); + + /* Detach from controlling terminal. */ + + if (setsid() == -1) { + nxt_log_emerg(thr->log, "setsid() failed %E", nxt_errno); + return NXT_ERROR; + } + + /* + * Reset file mode creation mask: any access + * rights can be set on file creation. + */ + umask(0); + + /* Redirect STDIN and STDOUT to the "/dev/null". */ + + fd = open("/dev/null", O_RDWR); + if (fd == -1) { + msg = "open(\"/dev/null\") failed %E"; + goto fail; + } + + if (dup2(fd, STDIN_FILENO) == -1) { + msg = "dup2(\"/dev/null\", STDIN) failed %E"; + goto fail; + } + + if (dup2(fd, STDOUT_FILENO) == -1) { + msg = "dup2(\"/dev/null\", STDOUT) failed %E"; + goto fail; + } + + if (fd > STDERR_FILENO) { + nxt_fd_close(fd); + } + + return NXT_OK; + +fail: + + nxt_log_emerg(thr->log, msg, nxt_errno); + + return NXT_ERROR; +} + + +void +nxt_nanosleep(nxt_nsec_t ns) +{ + struct timespec ts; + + ts.tv_sec = ns / 1000000000; + ts.tv_nsec = ns % 1000000000; + + (void) nanosleep(&ts, NULL); +} + + +nxt_int_t +nxt_user_cred_get(nxt_user_cred_t *uc, const char *group) +{ + struct group *grp; + struct passwd *pwd; + + pwd = getpwnam(uc->user); + + if (nxt_slow_path(pwd == NULL)) { + nxt_thread_log_emerg("getpwnam(%s) failed %E", uc->user, nxt_errno); + return NXT_ERROR; + } + + uc->uid = pwd->pw_uid; + uc->base_gid = pwd->pw_gid; + + if (group != NULL) { + grp = getgrnam(group); + + if (nxt_slow_path(grp == NULL)) { + nxt_thread_log_emerg("getgrnam(%s) failed %E", group, nxt_errno); + return NXT_ERROR; + } + + uc->base_gid = grp->gr_gid; + } + + if (getuid() == 0) { + return nxt_user_groups_get(uc); + } + + return NXT_OK; +} + + +/* + * nxt_user_groups_get() stores an array of groups IDs which should be + * set by the initgroups() function for a given user. The initgroups() + * may block a just forked worker process for some time if LDAP or NDIS+ + * is used, so nxt_user_groups_get() allows to get worker user groups in + * master process. In a nutshell the initgroups() calls getgrouplist() + * followed by setgroups(). However Solaris lacks the getgrouplist(). + * Besides getgrouplist() does not allow to query the exact number of + * groups while NGROUPS_MAX can be quite large (e.g. 65536 on Linux). + * So nxt_user_groups_get() emulates getgrouplist(): at first the function + * saves the super-user groups IDs, then calls initgroups() and saves the + * specified user groups IDs, and then restores the super-user groups IDs. + * This works at least on Linux, FreeBSD, and Solaris, but does not work + * on MacOSX, getgroups(2): + * + * To provide compatibility with applications that use getgroups() in + * environments where users may be in more than {NGROUPS_MAX} groups, + * a variant of getgroups(), obtained when compiling with either the + * macros _DARWIN_UNLIMITED_GETGROUPS or _DARWIN_C_SOURCE defined, can + * be used that is not limited to {NGROUPS_MAX} groups. However, this + * variant only returns the user's default group access list and not + * the group list modified by a call to setgroups(2). + * + * For such cases initgroups() is used in worker process as fallback. + */ + +static nxt_int_t +nxt_user_groups_get(nxt_user_cred_t *uc) +{ + int nsaved, ngroups; + nxt_int_t ret; + nxt_gid_t *saved; + + nsaved = getgroups(0, NULL); + + if (nsaved == -1) { + nxt_thread_log_emerg("getgroups(0, NULL) failed %E", nxt_errno); + return NXT_ERROR; + } + + nxt_thread_log_debug("getgroups(0, NULL): %d", nsaved); + + if (nsaved > NGROUPS_MAX) { + /* MacOSX case. */ + return NXT_OK; + } + + saved = nxt_malloc(nsaved * sizeof(nxt_gid_t)); + + if (saved == NULL) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + + nsaved = getgroups(nsaved, saved); + + if (nsaved == -1) { + nxt_thread_log_emerg("getgroups(%d) failed %E", nsaved, nxt_errno); + goto fail; + } + + nxt_thread_log_debug("getgroups(): %d", nsaved); + + if (initgroups(uc->user, uc->base_gid) != 0) { + nxt_thread_log_emerg("initgroups(%s, %d) failed", + uc->user, uc->base_gid); + goto restore; + } + + ngroups = getgroups(0, NULL); + + if (ngroups == -1) { + nxt_thread_log_emerg("getgroups(0, NULL) failed %E", nxt_errno); + goto restore; + } + + nxt_thread_log_debug("getgroups(0, NULL): %d", ngroups); + + uc->gids = nxt_malloc(ngroups * sizeof(nxt_gid_t)); + + if (uc->gids == NULL) { + goto restore; + } + + ngroups = getgroups(ngroups, uc->gids); + + if (ngroups == -1) { + nxt_thread_log_emerg("getgroups(%d) failed %E", ngroups, nxt_errno); + goto restore; + } + + uc->ngroups = ngroups; + +#if (NXT_DEBUG) + { + u_char *p, *end; + nxt_uint_t i; + u_char msg[NXT_MAX_ERROR_STR]; + + p = msg; + end = msg + NXT_MAX_ERROR_STR; + + for (i = 0; i < uc->ngroups; i++) { + p = nxt_sprintf(p, end, "%uL:", (uint64_t) uc->gids[i]); + } + + nxt_thread_log_debug("user \"%s\" cred: uid:%uL base gid:%uL, gids:%*s", + uc->user, (uint64_t) uc->uid, + (uint64_t) uc->base_gid, p - msg, msg); + } +#endif + + ret = NXT_OK; + +restore: + + if (setgroups(nsaved, saved) != 0) { + nxt_thread_log_emerg("setgroups(%d) failed %E", nsaved, nxt_errno); + ret = NXT_ERROR; + } + +fail: + + nxt_free(saved); + + return ret; +} + + +nxt_int_t +nxt_user_cred_set(nxt_user_cred_t *uc) +{ + nxt_thread_log_debug("user cred set: \"%s\" uid:%uL base gid:%uL", + uc->user, (uint64_t) uc->uid, uc->base_gid); + + if (setgid(uc->base_gid) != 0) { + nxt_thread_log_emerg("setgid(%d) failed %E", uc->base_gid, nxt_errno); + return NXT_ERROR; + } + + if (uc->gids != NULL) { + if (setgroups(uc->ngroups, uc->gids) != 0) { + nxt_thread_log_emerg("setgroups(%i) failed %E", + uc->ngroups, nxt_errno); + return NXT_ERROR; + } + + } else { + /* MacOSX fallback. */ + if (initgroups(uc->user, uc->base_gid) != 0) { + nxt_thread_log_emerg("initgroups(%s, %d) failed", + uc->user, uc->base_gid); + return NXT_ERROR; + } + } + + if (setuid(uc->uid) != 0) { + nxt_thread_log_emerg("setuid(%d) failed %E", uc->uid, nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} diff --git a/src/nxt_process.h b/src/nxt_process.h new file mode 100644 index 00000000..702d39f2 --- /dev/null +++ b/src/nxt_process.h @@ -0,0 +1,87 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_PROCESS_H_INCLUDED_ +#define _NXT_UNIX_PROCESS_H_INCLUDED_ + + +typedef pid_t nxt_pid_t; + + +#define \ +nxt_sched_yield() \ + sched_yield() + + +#define \ +nxt_process_id() \ + nxt_pid + + +/* + * Solaris declares abort() as __NORETURN, + * raise(SIGABRT) is mostly the same. + */ + +#define \ +nxt_abort() \ + (void) raise(SIGABRT) + + +typedef void (*nxt_process_start_t)(void *data); + +NXT_EXPORT nxt_pid_t nxt_process_create(nxt_process_start_t start, void *data, + const char *name); +NXT_EXPORT nxt_pid_t nxt_process_execute(char *name, char **argv, char **envp); +NXT_EXPORT nxt_int_t nxt_process_daemon(void); +NXT_EXPORT void nxt_nanosleep(nxt_nsec_t ns); + +NXT_EXPORT void nxt_process_arguments(char **orig_argv, char ***orig_envp); + + +#if (NXT_HAVE_SETPROCTITLE) + +#define \ +nxt_process_title(title) \ + setproctitle("%s", title) + +#elif (NXT_LINUX || NXT_SOLARIS || NXT_MACOSX) + +#define NXT_SETPROCTITLE_ARGV 1 +NXT_EXPORT void nxt_process_title(const char *title); + +#else + +#define \ +nxt_process_title(title) + +#endif + + +NXT_EXPORT extern nxt_pid_t nxt_pid; +NXT_EXPORT extern nxt_pid_t nxt_ppid; +NXT_EXPORT extern char **nxt_process_argv; +NXT_EXPORT extern char ***nxt_process_environ; + + +typedef uid_t nxt_uid_t; +typedef gid_t nxt_gid_t; + + +typedef struct { + const char *user; + nxt_uid_t uid; + nxt_gid_t base_gid; + nxt_uint_t ngroups; + nxt_gid_t *gids; +} nxt_user_cred_t; + + +NXT_EXPORT nxt_int_t nxt_user_cred_get(nxt_user_cred_t *uc, const char *group); +NXT_EXPORT nxt_int_t nxt_user_cred_set(nxt_user_cred_t *uc); + + +#endif /* _NXT_UNIX_PROCESS_H_INCLUDED_ */ diff --git a/src/nxt_process_chan.c b/src/nxt_process_chan.c new file mode 100644 index 00000000..4fcc1a15 --- /dev/null +++ b/src/nxt_process_chan.c @@ -0,0 +1,263 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_cycle.h> +#include <nxt_process_chan.h> + + +static void nxt_process_chan_handler(nxt_thread_t *thr, + nxt_chan_recv_msg_t *msg); +static void nxt_process_new_chan_buf_completion(nxt_thread_t *thr, void *obj, + void *data); + + +void +nxt_process_chan_create(nxt_thread_t *thr, nxt_process_chan_t *proc, + nxt_process_chan_handler_t *handlers) +{ + proc->pid = nxt_pid; + proc->engine = thr->engine->id; + proc->chan->handler = nxt_process_chan_handler; + proc->chan->data = handlers; + + nxt_chan_write_close(proc->chan); + nxt_chan_read_enable(thr, proc->chan); +} + + +void +nxt_process_chan_write(nxt_cycle_t *cycle, nxt_uint_t type, nxt_fd_t fd, + uint32_t stream, nxt_buf_t *b) +{ + nxt_uint_t i, n; + nxt_process_chan_t *proc; + + proc = cycle->processes->elts; + n = cycle->processes->nelts; + + for (i = 0; i < n; i++) { + if (nxt_pid != proc[i].pid) { + (void) nxt_chan_write(proc[i].chan, type, fd, stream, b); + } + } +} + + +static void +nxt_process_chan_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg) +{ + nxt_process_chan_handler_t *handlers; + + if (nxt_fast_path(msg->type <= NXT_CHAN_MSG_MAX)) { + + nxt_log_debug(thr->log, "chan %d: message type:%uD", + msg->chan->socket.fd, msg->type); + + handlers = msg->chan->data; + handlers[msg->type](thr, msg); + + return; + } + + nxt_log_alert(thr->log, "chan %d: unknown message type:%uD", + msg->chan->socket.fd, msg->type); +} + + +void +nxt_process_chan_quit_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg) +{ + nxt_cycle_quit(thr, NULL); +} + + +void +nxt_process_new_chan(nxt_cycle_t *cycle, nxt_process_chan_t *proc) +{ + nxt_buf_t *b; + nxt_uint_t i, n; + nxt_process_chan_t *p; + nxt_proc_msg_new_chan_t *new_chan; + + n = cycle->processes->nelts; + if (n == 0) { + return; + } + + nxt_thread_log_debug("new chan %d for process %PI engine %uD", + proc->chan->socket.fd, proc->pid, proc->engine); + + p = cycle->processes->elts; + + for (i = 0; i < n; i++) { + + if (proc->pid == p[i].pid || nxt_pid == p[i].pid || p[i].engine != 0) { + continue; + } + + b = nxt_buf_mem_alloc(p[i].chan->mem_pool, + sizeof(nxt_process_chan_data_t), 0); + + if (nxt_slow_path(b == NULL)) { + continue; + } + + b->data = p[i].chan; + b->completion_handler = nxt_process_new_chan_buf_completion; + b->mem.free += sizeof(nxt_proc_msg_new_chan_t); + new_chan = (nxt_proc_msg_new_chan_t *) b->mem.pos; + + new_chan->pid = proc->pid; + new_chan->engine = proc->engine; + new_chan->max_size = p[i].chan->max_size; + new_chan->max_share = p[i].chan->max_share; + + (void) nxt_chan_write(p[i].chan, NXT_CHAN_MSG_NEW_CHAN, + proc->chan->socket.fd, 0, b); + } +} + + +static void +nxt_process_new_chan_buf_completion(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_chan_t *chan; + + b = obj; + chan = b->data; + + /* TODO: b->mem.pos */ + + nxt_buf_free(chan->mem_pool, b); +} + + +void +nxt_process_chan_new_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg) +{ + nxt_chan_t *chan; + nxt_cycle_t *cycle; + nxt_process_chan_t *proc; + nxt_proc_msg_new_chan_t *new_chan; + + cycle = nxt_thread_cycle(); + + proc = nxt_array_add(cycle->processes); + if (nxt_slow_path(proc == NULL)) { + return; + } + + chan = nxt_chan_alloc(); + if (nxt_slow_path(chan == NULL)) { + return; + } + + proc->chan = chan; + + new_chan = (nxt_proc_msg_new_chan_t *) msg->buf->mem.pos; + msg->buf->mem.pos = msg->buf->mem.free; + + nxt_log_debug(thr->log, "new chan %d received for process %PI engine %uD", + msg->fd, new_chan->pid, new_chan->engine); + + proc->pid = new_chan->pid; + proc->engine = new_chan->engine; + chan->pair[1] = msg->fd; + chan->max_size = new_chan->max_size; + chan->max_share = new_chan->max_share; + + /* A read chan is not passed at all. */ + nxt_chan_write_enable(thr, chan); +} + + +void +nxt_process_chan_change_log_file(nxt_cycle_t *cycle, nxt_uint_t slot, + nxt_fd_t fd) +{ + nxt_buf_t *b; + nxt_uint_t i, n; + nxt_process_chan_t *p; + + n = cycle->processes->nelts; + if (n == 0) { + return; + } + + nxt_thread_log_debug("change log file #%ui fd:%FD", slot, fd); + + p = cycle->processes->elts; + + /* p[0] is master process. */ + + for (i = 1; i < n; i++) { + b = nxt_buf_mem_alloc(p[i].chan->mem_pool, + sizeof(nxt_process_chan_data_t), 0); + + if (nxt_slow_path(b == NULL)) { + continue; + } + + *(nxt_uint_t *) b->mem.pos = slot; + b->mem.free += sizeof(nxt_uint_t); + + (void) nxt_chan_write(p[i].chan, NXT_CHAN_MSG_CHANGE_FILE, fd, 0, b); + } +} + + +void +nxt_process_chan_change_log_file_handler(nxt_thread_t *thr, + nxt_chan_recv_msg_t *msg) +{ + nxt_buf_t *b; + nxt_uint_t slot; + nxt_file_t *log_file; + nxt_cycle_t *cycle; + + cycle = nxt_thread_cycle(); + + b = msg->buf; + slot = *(nxt_uint_t *) b->mem.pos; + + log_file = nxt_list_elt(cycle->log_files, slot); + + nxt_log_debug(thr->log, "change log file %FD:%FD", msg->fd, log_file->fd); + + /* + * The old log file descriptor must be closed at the moment when no + * other threads use it. dup2() allows to use the old file descriptor + * for new log file. This change is performed atomically in the kernel. + */ + if (nxt_file_redirect(log_file, msg->fd) == NXT_OK) { + + if (slot == 0) { + (void) nxt_file_stderr(log_file); + } + } +} + + +void +nxt_process_chan_data_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg) +{ + nxt_buf_t *b; + + b = msg->buf; + + nxt_log_debug(thr->log, "data: %*s", b->mem.free - b->mem.pos, b->mem.pos); + + b->mem.pos = b->mem.free; +} + + +void +nxt_process_chan_empty_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg) +{ + nxt_log_debug(thr->log, "chan empty handler"); +} diff --git a/src/nxt_process_chan.h b/src/nxt_process_chan.h new file mode 100644 index 00000000..a073e3dd --- /dev/null +++ b/src/nxt_process_chan.h @@ -0,0 +1,68 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_PROCESS_CHAN_H_INCLUDED_ +#define _NXT_PROCESS_CHAN_H_INCLUDED_ + + +#define NXT_CHAN_MSG_MAX NXT_CHAN_MSG_DATA + +typedef enum { + NXT_CHAN_MSG_QUIT = 0, + NXT_CHAN_MSG_NEW_CHAN, + NXT_CHAN_MSG_CHANGE_FILE, + NXT_CHAN_MSG_DATA, +} nxt_chan_msg_type_e; + + +typedef struct { + nxt_pid_t pid; + uint32_t engine; + uint32_t generation; + nxt_chan_t *chan; +} nxt_process_chan_t; + + +typedef struct { + nxt_pid_t pid; + uint32_t engine; + size_t max_size; + size_t max_share; +} nxt_proc_msg_new_chan_t; + + +/* + * nxt_process_chan_data_t is allocaiton size + * enabling effective reuse of memory pool cache. + */ +typedef union { + nxt_buf_t buf; + nxt_proc_msg_new_chan_t new_chan; +} nxt_process_chan_data_t; + + +typedef void (*nxt_process_chan_handler_t)(nxt_thread_t *thr, + nxt_chan_recv_msg_t *msg); + + +void nxt_process_chan_create(nxt_thread_t *thr, nxt_process_chan_t *proc, + nxt_process_chan_handler_t *handlers); +void nxt_process_chan_write(nxt_cycle_t *cycle, nxt_uint_t type, + nxt_fd_t fd, uint32_t stream, nxt_buf_t *b); +void nxt_process_new_chan(nxt_cycle_t *cycle, nxt_process_chan_t *proc); +void nxt_process_chan_change_log_file(nxt_cycle_t *cycle, nxt_uint_t slot, + nxt_fd_t fd); + +void nxt_process_chan_quit_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg); +void nxt_process_chan_new_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg); +void nxt_process_chan_change_log_file_handler(nxt_thread_t *thr, + nxt_chan_recv_msg_t *msg); +void nxt_process_chan_data_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg); +void nxt_process_chan_empty_handler(nxt_thread_t *thr, + nxt_chan_recv_msg_t *msg); + + +#endif /* _NXT_PROCESS_CHAN_H_INCLUDED_ */ diff --git a/src/nxt_process_title.c b/src/nxt_process_title.c new file mode 100644 index 00000000..76533a6e --- /dev/null +++ b/src/nxt_process_title.c @@ -0,0 +1,253 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* The arguments passed to main(). */ +char **nxt_process_argv; + +/* + * MacOSX environ(7): + * + * Shared libraries and bundles don't have direct access to environ, + * which is only available to the loader ld(1) when a complete program + * is being linked. + * + * So nxt_process_environ contains an address of environ to allow + * change environ[] placement. + */ +char ***nxt_process_environ; + + +#if (NXT_SETPROCTITLE_ARGV) + +/* + * A process title on Linux, Solaris, and MacOSX can be changed by + * copying a new title to a place where the program argument argv[0] + * points originally to. However, the argv[0] may be too small to hold + * the new title. Fortunately, these OSes place the program argument + * argv[] strings and the environment environ[] strings contiguously + * and their space can be used for the long new process title. + * + * Solaris "ps" command shows the new title only if it is run in + * UCB mode: either "/usr/ucb/ps -axwww" or "/usr/bin/ps axwww". + */ + + +static u_char *nxt_process_title_start; +static u_char *nxt_process_title_end; + + +void +nxt_process_arguments(char **orig_argv, char ***orig_envp) +{ + u_char *p, *end, *argv_end, **argv, **env; + size_t size, argv_size, environ_size, strings_size; + nxt_uint_t i; + + nxt_process_argv = orig_argv; + nxt_process_environ = orig_envp; + + if (orig_envp == NULL) { + return; + } + + /* + * Set a conservative title space for a case if program argument + * strings and environment strings are not contiguous. + */ + argv = (u_char **) orig_argv; + nxt_process_title_start = argv[0]; + nxt_process_title_end = argv[0] + nxt_strlen(argv[0]); + + end = argv[0]; + strings_size = 0; + argv_size = sizeof(void *); + + for (i = 0; argv[i] != NULL; i++) { + argv_size += sizeof(void *); + + if (argv[i] == end) { + /* Argument strings are contiguous. */ + size = nxt_strlen(argv[i]) + 1; + strings_size += size; + end = argv[i] + size; + } + } + + argv = nxt_malloc(argv_size); + if (argv == NULL) { + return; + } + + /* + * Copy the entire original argv[] array. The elements of this array + * can point to copied strings or if original argument strings are not + * contiguous, to the original argument strings. + */ + nxt_memcpy(argv, orig_argv, argv_size); + + /* + * The argv[1] must be set to NULL on Solaris otherwise the "ps" + * command outputs strings pointed by original argv[] elements. + * The original argv[] array has always at least two elements so + * it is safe to set argv[1]. + */ + orig_argv[1] = NULL; + + nxt_process_argv = (char **) argv; + + argv_end = end; + env = (u_char **) *orig_envp; + environ_size = sizeof(void *); + + for (i = 0; env[i] != NULL; i++) { + environ_size += sizeof(void *); + + if (env[i] == end) { + /* Environment strings are contiguous. */ + size = nxt_strlen(env[i]) + 1; + strings_size += size; + end = env[i] + size; + } + } + + p = nxt_malloc(strings_size); + if (p == NULL) { + return; + } + + if (argv_end == end) { + /* + * There is no reason to modify environ if arguments + * and environment are not contiguous. + */ + nxt_thread_log_debug("arguments and environment are not contiguous"); + goto done; + } + + end = argv[0]; + + for (i = 0; argv[i] != NULL; i++) { + + if (argv[i] != end) { + /* Argument strings are not contiguous. */ + goto done; + } + + size = nxt_strlen(argv[i]) + 1; + nxt_memcpy(p, argv[i], size); + + end = argv[i] + size; + argv[i] = p; + p += size; + } + + env = nxt_malloc(environ_size); + if (env == NULL) { + return; + } + + /* + * Copy the entire original environ[] array. The elements of + * this array can point to copied strings or if original environ + * strings are not contiguous, to the original environ strings. + */ + nxt_memcpy(env, *orig_envp, environ_size); + + /* Set the global environ variable to the new array. */ + *orig_envp = (char **) env; + + for (i = 0; env[i] != NULL; i++) { + + if (env[i] != end) { + /* Environment strings are not contiguous. */ + goto done; + } + + size = nxt_strlen(env[i]) + 1; + nxt_memcpy(p, env[i], size); + + end = env[i] + size; + env[i] = p; + p += size; + } + +done: + + /* Preserve space for the trailing zero. */ + end--; + + nxt_process_title_end = end; +} + + +void +nxt_process_title(const char *title) +{ + u_char *p, *start, *end; + + start = nxt_process_title_start; + + if (start == NULL) { + return; + } + + end = nxt_process_title_end; + + p = nxt_sprintf(start, end, "%s", title); + +#if (NXT_SOLARIS) + /* + * Solaris "ps" command shows a new process title only if it is + * longer than original command line. A simple workaround is just + * to append the original command line in parenthesis to the title. + */ + { + size_t size; + nxt_uint_t i; + + size = 0; + + for (i = 0; nxt_process_argv[i] != NULL; i++) { + size += nxt_strlen(nxt_process_argv[i]); + } + + if (size > (size_t) (p - start)) { + + p = nxt_sprintf(p, end, " ("); + + for (i = 0; nxt_process_argv[i] != NULL; i++) { + p = nxt_sprintf(p, end, "%s ", nxt_process_argv[i]); + } + + if (*(p - 1) == ' ') { + *(p - 1) = ')'; + } + } + } +#endif + + /* + * A process title must be padded with zeros on MacOSX. Otherwise + * the "ps" command may output parts of environment strings. + */ + nxt_memset(p, '\0', end - p); + + nxt_thread_log_debug("setproctitle: \"%s\"", start); +} + +#else /* !(NXT_SETPROCTITLE_ARGV) */ + +void +nxt_process_arguments(char **orig_argv, char ***orig_envp) +{ + nxt_process_argv = orig_argv; + nxt_process_environ = orig_envp; +} + +#endif diff --git a/src/nxt_python_wsgi.c b/src/nxt_python_wsgi.c new file mode 100644 index 00000000..c0b03efe --- /dev/null +++ b/src/nxt_python_wsgi.c @@ -0,0 +1,973 @@ + +/* + * Copyright (C) Valentin V. Bartenev + * Copyright (C) NGINX, Inc. + */ + + +#include <Python.h> + +#include <compile.h> +#include <node.h> + +#ifdef _DARWIN_C_SOURCE +#undef _DARWIN_C_SOURCE +#endif + +#include <nxt_main.h> +#include <nxt_cycle.h> +#include <nxt_application.h> + + +typedef struct { + PyObject_HEAD + //nxt_app_request_t *request; +} nxt_py_input_t; + + +typedef struct { + PyObject_HEAD + //nxt_app_request_t *request; +} nxt_py_error_t; + + +static nxt_int_t nxt_python_init(nxt_thread_t *thr); +static nxt_int_t nxt_python_run(nxt_app_request_t *r); + +static PyObject *nxt_python_create_environ(nxt_thread_t *thr); +static PyObject *nxt_python_get_environ(nxt_app_request_t *r); + +static PyObject *nxt_py_start_resp(PyObject *self, PyObject *args); + +static void nxt_py_input_dealloc(nxt_py_input_t *self); +static PyObject *nxt_py_input_read(nxt_py_input_t *self, PyObject *args); +static PyObject *nxt_py_input_readline(nxt_py_input_t *self, PyObject *args); +static PyObject *nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args); + + +extern nxt_int_t nxt_python_wsgi_init(nxt_thread_t *thr, nxt_cycle_t *cycle); + + +nxt_application_module_t nxt_python_module = { + nxt_python_init, + NULL, + NULL, + nxt_python_run, +}; + + +static PyMethodDef nxt_py_start_resp_method[] = { + {"nginman_start_response", nxt_py_start_resp, METH_VARARGS, ""} +}; + + +static PyMethodDef nxt_py_input_methods[] = { + { "read", (PyCFunction) nxt_py_input_read, METH_VARARGS, 0 }, + { "readline", (PyCFunction) nxt_py_input_readline, METH_VARARGS, 0 }, + { "readlines", (PyCFunction) nxt_py_input_readlines, METH_VARARGS, 0 }, + { NULL, NULL, 0, 0 } +}; + + +static PyTypeObject nxt_py_input_type = { + PyVarObject_HEAD_INIT(NULL, 0) + "nginman._input", /* tp_name */ + (int) sizeof(nxt_py_input_t), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor) nxt_py_input_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "nginman input object.", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + nxt_py_input_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro - method resolution order */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +}; + + +static char *nxt_py_module; + +static PyObject *nxt_py_application; +static PyObject *nxt_py_start_resp_obj; +static PyObject *nxt_py_environ_ptyp; + +static nxt_app_request_t *nxt_app_request; + + +nxt_int_t +nxt_python_wsgi_init(nxt_thread_t *thr, nxt_cycle_t *cycle) +{ + char **argv; + u_char *p, *dir; + + PyObject *obj, *pypath; + + argv = nxt_process_argv; + + while (*argv != NULL) { + p = (u_char *) *argv++; + + if (nxt_strcmp(p, "--py-module") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, + "no argument for option \"--py-module\""); + return NXT_ERROR; + } + + nxt_py_module = *argv++; + + nxt_log_error(NXT_LOG_INFO, thr->log, "python module: \"%s\"", + nxt_py_module); + + break; + } + } + + if (nxt_py_module == NULL) { + return NXT_OK; + } + + Py_InitializeEx(0); + + obj = NULL; + argv = nxt_process_argv; + + while (*argv != NULL) { + p = (u_char *) *argv++; + + if (nxt_strcmp(p, "--py-path") == 0) { + if (*argv == NULL) { + nxt_log_emerg(thr->log, "no argument for option \"--py-path\""); + goto fail; + } + + dir = (u_char *) *argv++; + + nxt_log_error(NXT_LOG_INFO, thr->log, "python path \"%s\"", dir); + + obj = PyString_FromString((char *) dir); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_alert(thr->log, + "Python failed create string object \"%s\"", dir); + goto fail; + } + + pypath = PySys_GetObject((char *) "path"); + + if (nxt_slow_path(pypath == NULL)) { + nxt_log_alert(thr->log, + "Python failed to get \"sys.path\" list"); + goto fail; + } + + if (nxt_slow_path(PyList_Insert(pypath, 0, obj) != 0)) { + nxt_log_alert(thr->log, + "Python failed to insert \"%s\" into \"sys.path\"", dir); + goto fail; + } + + Py_DECREF(obj); + obj = NULL; + + continue; + } + } + + obj = PyCFunction_New(nxt_py_start_resp_method, NULL); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_alert(thr->log, + "Python failed to initialize the \"start_response\" function"); + goto fail; + } + + nxt_py_start_resp_obj = obj; + + obj = nxt_python_create_environ(thr); + + if (obj == NULL) { + goto fail; + } + + nxt_py_environ_ptyp = obj; + + + obj = Py_BuildValue("[s]", "nginman"); + if (obj == NULL) { + nxt_log_alert(thr->log, + "Python failed to create the \"sys.argv\" list"); + goto fail; + } + + if (PySys_SetObject((char *) "argv", obj) != 0) { + nxt_log_alert(thr->log, "Python failed to set the \"sys.argv\" list"); + goto fail; + } + + Py_DECREF(obj); + + return NXT_OK; + +fail: + + Py_XDECREF(obj); + Py_XDECREF(nxt_py_start_resp_obj); + + Py_Finalize(); + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_python_init(nxt_thread_t *thr) +{ + PyObject *module, *obj; + +#if 0 + FILE *fp; + PyObject *co; + struct _node *node; + + chdir((char *) dir); + fp = fopen((char *) script, "r"); + + if (fp == NULL) { + nxt_log_debug(thr->log, "fopen failed"); + return NXT_ERROR; + } + + + Py_SetProgramName((char *) "python mysite/wsgi.py"); + Py_InitializeEx(0); + + node = PyParser_SimpleParseFile(fp, (char *) script, Py_file_input); + + fclose(fp); + + if (node == NULL) { + nxt_log_debug(thr->log, "BAD node"); + return NXT_ERROR; + } + + co = (PyObject *) PyNode_Compile(node, (char *) script); + + PyNode_Free(node); + + if (co == NULL) { + nxt_log_debug(thr->log, "BAD co"); + return NXT_ERROR; + } + + pModule = PyImport_ExecCodeModuleEx((char *) "_wsgi_nginman", co, (char *) script); + + Py_XDECREF(co); +#endif + + PyOS_AfterFork(); + + module = PyImport_ImportModule(nxt_py_module); + + if (nxt_slow_path(module == NULL)) { + nxt_log_emerg(thr->log, "Python failed to import module \"%s\"", + nxt_py_module); + return NXT_ERROR; + } + + obj = PyDict_GetItemString(PyModule_GetDict(module), "application"); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_emerg(thr->log, "Python failed to get \"application\" " + "from module \"%s\"", nxt_py_module); + goto fail; + } + + if (nxt_slow_path(PyCallable_Check(obj) == 0)) { + nxt_log_emerg(thr->log, "\"application\" in module \"%s\" " + "is not a callable object", nxt_py_module); + goto fail; + } + + Py_INCREF(obj); + Py_DECREF(module); + + nxt_py_application = obj; + + return NXT_OK; + +fail: + + Py_DECREF(module); + + return NXT_ERROR; +} + + +static nxt_int_t +nxt_python_run(nxt_app_request_t *r) +{ + u_char *buf; + size_t size; + PyObject *result, *iterator, *item, *args, *environ; + + nxt_app_request = r; + + environ = nxt_python_get_environ(r); + + if (nxt_slow_path(environ == NULL)) { + return NXT_ERROR; + } + + args = PyTuple_New(2); + + if (nxt_slow_path(args == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create arguments tuple"); + return NXT_ERROR; + } + + PyTuple_SET_ITEM(args, 0, environ); + + Py_INCREF(nxt_py_start_resp_obj); + PyTuple_SET_ITEM(args, 1, nxt_py_start_resp_obj); + + result = PyObject_CallObject(nxt_py_application, args); + + Py_DECREF(args); + + if (nxt_slow_path(result == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to call the application"); + PyErr_Print(); + return NXT_ERROR; + } + + iterator = PyObject_GetIter(result); + + Py_DECREF(result); + + if (nxt_slow_path(iterator == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "the application returned not an iterable object"); + return NXT_ERROR; + } + + while((item = PyIter_Next(iterator))) { + + if (nxt_slow_path(PyString_Check(item) == 0)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "the application returned not a string object"); + + Py_DECREF(item); + Py_DECREF(iterator); + + return NXT_ERROR; + } + + size = PyString_GET_SIZE(item); + buf = (u_char *) PyString_AS_STRING(item); + + nxt_app_write(r, buf, size); + + Py_DECREF(item); + } + + Py_DECREF(iterator); + + if (nxt_slow_path(PyErr_Occurred() != NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, "an application error occurred"); + PyErr_Print(); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static PyObject * +nxt_python_create_environ(nxt_thread_t *thr) +{ + PyObject *obj, *stderr, *environ; + + environ = PyDict_New(); + + if (nxt_slow_path(environ == NULL)) { + nxt_log_alert(thr->log, + "Python failed to create the \"environ\" dictionary"); + return NULL; + } + + obj = Py_BuildValue("(ii)", 1, 0); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_alert(thr->log, + "Python failed to build the \"wsgi.version\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.version", obj) != 0)) + { + nxt_log_alert(thr->log, + "Python failed to set the \"wsgi.version\" environ value"); + goto fail; + } + + Py_DECREF(obj); + obj = NULL; + + + if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.multithread", + Py_False) + != 0)) + { + nxt_log_alert(thr->log, + "Python failed to set the \"wsgi.multithread\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.multiprocess", + Py_True) + != 0)) + { + nxt_log_alert(thr->log, + "Python failed to set the \"wsgi.multiprocess\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.run_once", + Py_False) + != 0)) + { + nxt_log_alert(thr->log, + "Python failed to set the \"wsgi.run_once\" environ value"); + goto fail; + } + + + obj = PyString_FromString("http"); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_alert(thr->log, + "Python failed to create the \"wsgi.url_scheme\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.url_scheme", obj) + != 0)) + { + nxt_log_alert(thr->log, + "Python failed to set the \"wsgi.url_scheme\" environ value"); + goto fail; + } + + Py_DECREF(obj); + obj = NULL; + + + if (nxt_slow_path(PyType_Ready(&nxt_py_input_type) != 0)) { + nxt_log_alert(thr->log, + "Python failed to initialize the \"wsgi.input\" type object"); + goto fail; + } + + obj = (PyObject *) PyObject_New(nxt_py_input_t, &nxt_py_input_type); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_alert(thr->log, + "Python failed to create the \"wsgi.input\" object"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.input", obj) != 0)) { + nxt_log_alert(thr->log, + "Python failed to set the \"wsgi.input\" environ value"); + goto fail; + } + + Py_DECREF(obj); + obj = NULL; + + + stderr = PySys_GetObject((char *) "stderr"); + + if (nxt_slow_path(stderr == NULL)) { + nxt_log_alert(thr->log, "Python failed to get \"sys.stderr\" object"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.error", stderr) != 0)) + { + nxt_log_alert(thr->log, + "Python failed to set the \"wsgi.error\" environ value"); + goto fail; + } + + + obj = PyString_FromString("localhost"); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_alert(thr->log, + "Python failed to create the \"SERVER_NAME\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "SERVER_NAME", obj) != 0)) { + nxt_log_alert(thr->log, + "Python failed to set the \"SERVER_NAME\" environ value"); + goto fail; + } + + Py_DECREF(obj); + + + obj = PyString_FromString("80"); + + if (nxt_slow_path(obj == NULL)) { + nxt_log_alert(thr->log, + "Python failed to create the \"SERVER_PORT\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "SERVER_PORT", obj) != 0)) { + nxt_log_alert(thr->log, + "Python failed to set the \"SERVER_PORT\" environ value"); + goto fail; + } + + Py_DECREF(obj); + + return environ; + +fail: + + Py_XDECREF(obj); + Py_DECREF(environ); + + return NULL; +} + + +static PyObject * +nxt_python_get_environ(nxt_app_request_t *r) +{ + u_char *p, ch, *query; + nxt_str_t *str; + nxt_uint_t i, n; + nxt_app_header_field_t *fld; + + PyObject *environ, *value; + + static const u_char prefix[5] = "HTTP_"; + + static u_char key[256]; + + environ = PyDict_Copy(nxt_py_environ_ptyp); + + if (nxt_slow_path(environ == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"environ\" dictionary"); + return NULL; + } + + value = PyString_FromStringAndSize((char *) r->header.version.data, + r->header.version.len); + + if (nxt_slow_path(value == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"SERVER_PROTOCOL\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "SERVER_PROTOCOL", value) + != 0)) + { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"SERVER_PROTOCOL\" environ value"); + goto fail; + } + + Py_DECREF(value); + + value = PyString_FromStringAndSize((char *) r->header.method.data, + r->header.method.len); + + if (nxt_slow_path(value == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"REQUEST_METHOD\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "REQUEST_METHOD", value) + != 0)) + { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"REQUEST_METHOD\" environ value"); + goto fail; + } + + Py_DECREF(value); + + value = PyString_FromStringAndSize((char *) r->header.path.data, + r->header.path.len); + + if (nxt_slow_path(value == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"REQUEST_URI\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "REQUEST_URI", value) + != 0)) + { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"REQUEST_URI\" environ value"); + goto fail; + } + + Py_DECREF(value); + + query = nxt_memchr(r->header.path.data, '?', r->header.path.len); + + if (query != NULL) { + value = PyString_FromStringAndSize((char *) r->header.path.data, + query - r->header.path.data); + + query++; + + } else { + value = PyString_FromStringAndSize((char *) r->header.path.data, + r->header.path.len); + } + + if (nxt_slow_path(value == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"PATH_INFO\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "PATH_INFO", value) != 0)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"PATH_INFO\" environ value"); + goto fail; + } + + Py_DECREF(value); + + if (query != NULL) { + value = PyString_FromStringAndSize((char *) query, + r->header.path.data + + r->header.path.len - query); + + if (nxt_slow_path(value == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"QUERY_STRING\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "QUERY_STRING", value) + != 0)) + { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"QUERY_STRING\" environ value"); + goto fail; + } + + Py_DECREF(value); + } + + if (r->header.content_length != NULL) { + str = r->header.content_length; + + value = PyString_FromStringAndSize((char *) str->data, str->len); + + if (nxt_slow_path(value == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"CONTENT_LENGTH\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "CONTENT_LENGTH", value) + != 0)) + { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"CONTENT_LENGTH\" environ value"); + goto fail; + } + + Py_DECREF(value); + } + + if (r->header.content_type != NULL) { + str = r->header.content_type; + + value = PyString_FromStringAndSize((char *) str->data, str->len); + + if (nxt_slow_path(value == NULL)) { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to create the \"CONTENT_TYPE\" environ value"); + goto fail; + } + + if (nxt_slow_path(PyDict_SetItemString(environ, "CONTENT_TYPE", value) + != 0)) + { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"CONTENT_TYPE\" environ value"); + goto fail; + } + + Py_DECREF(value); + } + + nxt_memcpy(key, prefix, sizeof(prefix)); + + for (i = 0; i < r->header.fields_num; i++) { + fld = &r->header.fields[i]; + p = key + sizeof(prefix); + + for (n = 0; n < fld->name.len; n++, p++) { + + ch = fld->name.data[n]; + + if (ch >= 'a' && ch <= 'z') { + *p = ch & ~0x20; + continue; + } + + if (ch == '-') { + *p = '_'; + continue; + } + + *p = ch; + } + + *p = '\0'; + + value = PyString_FromStringAndSize((char *) fld->value.data, + fld->value.len); + + if (nxt_slow_path(PyDict_SetItemString(environ, (char *) key, value) + != 0)) + { + nxt_log_error(NXT_LOG_ERR, r->log, + "Python failed to set the \"%s\" environ value", key); + goto fail; + } + + Py_DECREF(value); + } + + return environ; + +fail: + + Py_XDECREF(value); + Py_DECREF(environ); + + return NULL; +} + + +static PyObject * +nxt_py_start_resp(PyObject *self, PyObject *args) +{ + u_char *p, buf[4096]; + PyObject *headers, *tuple, *string; + nxt_str_t str; + nxt_uint_t i, n; + + static const u_char resp[] = "HTTP/1.1 "; + + static const u_char default_headers[] + = "Server: nginman/0.1\r\n" + "Connection: close\r\n"; + + n = PyTuple_GET_SIZE(args); + + if (n < 2 || n > 3) { + return PyErr_Format(PyExc_TypeError, "invalid number of arguments"); + } + + string = PyTuple_GET_ITEM(args, 0); + + if (!PyString_Check(string)) { + return PyErr_Format(PyExc_TypeError, + "the first argument is not a string"); + } + + str.len = PyString_GET_SIZE(string); + str.data = (u_char *) PyString_AS_STRING(string); + + p = nxt_cpymem(buf, resp, sizeof(resp) - 1); + p = nxt_cpymem(p, str.data, str.len); + + *p++ = '\r'; *p++ = '\n'; + + p = nxt_cpymem(p, default_headers, sizeof(default_headers) - 1); + + headers = PyTuple_GET_ITEM(args, 1); + + if (!PyList_Check(headers)) { + return PyErr_Format(PyExc_TypeError, + "the second argument is not a response headers list"); + } + + for (i = 0; i < (nxt_uint_t) PyList_GET_SIZE(headers); i++) { + tuple = PyList_GET_ITEM(headers, i); + + if (!PyTuple_Check(tuple)) { + return PyErr_Format(PyExc_TypeError, + "the response headers must be a list of tuples"); + } + + if (PyTuple_GET_SIZE(tuple) != 2) { + return PyErr_Format(PyExc_TypeError, + "each header must be a tuple of two items"); + } + + string = PyTuple_GET_ITEM(tuple, 0); + + if (!PyString_Check(string)) { + return PyErr_Format(PyExc_TypeError, + "all response headers names must be strings"); + } + + str.len = PyString_GET_SIZE(string); + str.data = (u_char *) PyString_AS_STRING(string); + + p = nxt_cpymem(p, str.data, str.len); + + *p++ = ':'; *p++ = ' '; + + string = PyTuple_GET_ITEM(tuple, 1); + + if (!PyString_Check(string)) { + return PyErr_Format(PyExc_TypeError, + "all response headers values must be strings"); + } + + str.len = PyString_GET_SIZE(string); + str.data = (u_char *) PyString_AS_STRING(string); + + p = nxt_cpymem(p, str.data, str.len); + + *p++ = '\r'; *p++ = '\n'; + } + + *p++ = '\r'; *p++ = '\n'; + + nxt_app_write(nxt_app_request, buf, p - buf); + + return args; +} + + +static void +nxt_py_input_dealloc(nxt_py_input_t *self) +{ + PyObject_Del(self); +} + + +static PyObject * +nxt_py_input_read(nxt_py_input_t *self, PyObject *args) +{ + u_char *buf; + PyObject *body, *obj; + Py_ssize_t size; + nxt_uint_t n; + + nxt_app_request_t *r = nxt_app_request; + + size = r->body_rest; + + n = PyTuple_GET_SIZE(args); + + if (n > 0) { + if (n != 1) { + return PyErr_Format(PyExc_TypeError, "invalid number of arguments"); + } + + obj = PyTuple_GET_ITEM(args, 0); + + size = PyNumber_AsSsize_t(obj, PyExc_OverflowError); + + if (nxt_slow_path(size < 0)) { + if (size == -1 && PyErr_Occurred()) { + return NULL; + } + + return PyErr_Format(PyExc_ValueError, + "the read body size cannot be zero or less"); + } + + if (size == 0 || size > r->body_rest) { + size = r->body_rest; + } + } + + body = PyString_FromStringAndSize(NULL, size); + + if (nxt_slow_path(body == NULL)) { + return NULL; + } + + buf = (u_char *) PyString_AS_STRING(body); + + if (nxt_app_http_read_body(r, buf, size) != NXT_OK) { + return PyErr_Format(PyExc_IOError, "failed to read body"); + } + + return body; +} + + +static PyObject * +nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) +{ + return PyString_FromString(""); +} + + +static PyObject * +nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args) +{ + return PyList_New(0); +} diff --git a/src/nxt_queue.c b/src/nxt_queue.c new file mode 100644 index 00000000..c81356db --- /dev/null +++ b/src/nxt_queue.c @@ -0,0 +1,85 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#include <nxt_main.h> + + +/* + * Find the middle queue element if the queue has odd number of elements, + * or the first element of the queue's second part otherwise. + */ + +nxt_queue_link_t * +nxt_queue_middle(nxt_queue_t *queue) +{ + nxt_queue_link_t *middle, *next; + + middle = nxt_queue_first(queue); + + if (middle == nxt_queue_last(queue)) { + return middle; + } + + next = middle; + + for ( ;; ) { + middle = nxt_queue_next(middle); + + next = nxt_queue_next(next); + + if (next == nxt_queue_last(queue)) { + return middle; + } + + next = nxt_queue_next(next); + + if (next == nxt_queue_last(queue)) { + return middle; + } + } +} + + +/* + * nxt_queue_sort() provides a stable sort because it uses the insertion + * sort algorithm. Its worst and average computational complexity is O^2. + */ + +void +nxt_queue_sort(nxt_queue_t *queue, + nxt_int_t (*cmp)(const void *data, const nxt_queue_link_t *, + const nxt_queue_link_t *), const void *data) +{ + nxt_queue_link_t *link, *prev, *next; + + link = nxt_queue_first(queue); + + if (link == nxt_queue_last(queue)) { + return; + } + + for (link = nxt_queue_next(link); + link != nxt_queue_tail(queue); + link = next) + { + prev = nxt_queue_prev(link); + next = nxt_queue_next(link); + + nxt_queue_remove(link); + + do { + if (cmp(data, prev, link) <= 0) { + break; + } + + prev = nxt_queue_prev(prev); + + } while (prev != nxt_queue_head(queue)); + + nxt_queue_insert_after(prev, link); + } +} diff --git a/src/nxt_queue.h b/src/nxt_queue.h new file mode 100644 index 00000000..e5506630 --- /dev/null +++ b/src/nxt_queue.h @@ -0,0 +1,219 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_QUEUE_H_INCLUDED_ +#define _NXT_QUEUE_H_INCLUDED_ + + +typedef struct nxt_queue_link_s nxt_queue_link_t; + +struct nxt_queue_link_s { + nxt_queue_link_t *prev; + nxt_queue_link_t *next; +}; + + +typedef struct { + nxt_queue_link_t head; +} nxt_queue_t; + + +#define \ +nxt_queue_init(queue) \ + do { \ + (queue)->head.prev = &(queue)->head; \ + (queue)->head.next = &(queue)->head; \ + } while (0) + + +#define \ +nxt_queue_sentinel(link) \ + do { \ + (link)->prev = (link); \ + (link)->next = (link); \ + } while (0) + + +/* + * Short-circuit a queue link to itself to allow once remove safely it + * using nxt_queue_remove(). + */ + +#define \ +nxt_queue_self(link) \ + nxt_queue_sentinel(link) + + +#define \ +nxt_queue_is_empty(queue) \ + (&(queue)->head == (queue)->head.prev) + +/* + * A loop to iterate all queue links starting from head: + * + * nxt_queue_link_t link; + * } nxt_type_t *tp; + * + * + * for (lnk = nxt_queue_first(queue); + * lnk != nxt_queue_tail(queue); + * lnk = nxt_queue_next(lnk)) + * { + * tp = nxt_queue_link_data(lnk, nxt_type_t, link); + * + * or starting from tail: + * + * for (lnk = nxt_queue_last(queue); + * lnk != nxt_queue_head(queue); + * lnk = nxt_queue_next(lnk)) + * { + * tp = nxt_queue_link_data(lnk, nxt_type_t, link); + */ + +#define \ +nxt_queue_first(queue) \ + (queue)->head.next + + +#define \ +nxt_queue_last(queue) \ + (queue)->head.prev + + +#define \ +nxt_queue_head(queue) \ + (&(queue)->head) + + +#define \ +nxt_queue_tail(queue) \ + (&(queue)->head) + + +#define \ +nxt_queue_next(link) \ + (link)->next + + +#define \ +nxt_queue_prev(link) \ + (link)->prev + + +#define \ +nxt_queue_insert_head(queue, link) \ + do { \ + (link)->next = (queue)->head.next; \ + (link)->next->prev = (link); \ + (link)->prev = &(queue)->head; \ + (queue)->head.next = (link); \ + } while (0) + + +#define \ +nxt_queue_insert_tail(queue, link) \ + do { \ + (link)->prev = (queue)->head.prev; \ + (link)->prev->next = (link); \ + (link)->next = &(queue)->head; \ + (queue)->head.prev = (link); \ + } while (0) + + +#define \ +nxt_queue_insert_after(target, link) \ + do { \ + (link)->next = (target)->next; \ + (link)->next->prev = (link); \ + (link)->prev = (target); \ + (target)->next = (link); \ + } while (0) + + +#define \ +nxt_queue_insert_before(target, link) \ + do { \ + (link)->next = (target); \ + (link)->prev = (target)->prev; \ + (target)->prev = (link); \ + (link)->prev->next = (link); \ + } while (0) + + +#if (NXT_DEBUG) + +#define \ +nxt_queue_remove(link) \ + do { \ + (link)->next->prev = (link)->prev; \ + (link)->prev->next = (link)->next; \ + (link)->prev = NULL; \ + (link)->next = NULL; \ + } while (0) + +#else + +#define \ +nxt_queue_remove(link) \ + do { \ + (link)->next->prev = (link)->prev; \ + (link)->prev->next = (link)->next; \ + } while (0) + +#endif + + +/* + * Split the queue "queue" starting at the element "link", + * the "tail" is the new tail queue. + */ + +#define \ +nxt_queue_split(queue, link, tail) \ + do { \ + (tail)->head.prev = (queue)->head.prev; \ + (tail)->head.prev->next = &(tail)->head; \ + (tail)->head.next = (link); \ + (queue)->head.prev = (link)->prev; \ + (queue)->head.prev->next = &(queue)->head; \ + (link)->prev = &(tail)->head; \ + } while (0) + + +/* Truncate the queue "queue" starting at element "link". */ + +#define \ +nxt_queue_truncate(queue, link) \ + do { \ + (queue)->head.prev = (link)->prev; \ + (queue)->head.prev->next = &(queue)->head; \ + } while (0) + + +/* Add the queue "tail" to the queue "queue". */ + +#define \ +nxt_queue_add(queue, tail) \ + do { \ + (queue)->head.prev->next = (tail)->head.next; \ + (tail)->head.next->prev = (queue)->head.prev; \ + (queue)->head.prev = (tail)->head.prev; \ + (queue)->head.prev->next = &(queue)->head; \ + } while (0) + + +#define \ +nxt_queue_link_data(lnk, type, link) \ + nxt_container_of(lnk, type, link) + + +NXT_EXPORT nxt_queue_link_t *nxt_queue_middle(nxt_queue_t *queue); +NXT_EXPORT void nxt_queue_sort(nxt_queue_t *queue, + nxt_int_t (*cmp)(const void *, const nxt_queue_link_t *, + const nxt_queue_link_t *), const void *data); + + +#endif /* _NXT_QUEUE_H_INCLUDED_ */ diff --git a/src/nxt_random.c b/src/nxt_random.c new file mode 100644 index 00000000..9af40079 --- /dev/null +++ b/src/nxt_random.c @@ -0,0 +1,206 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#include <nxt_main.h> + + +#if !(NXT_HAVE_ARC4RANDOM) + +/* + * The pseudorandom generator based on OpenBSD arc4random. Although it is + * usually stated that arc4random uses RC4 pseudorandom generation algorithm + * they are actually different in nxt_random_add(). + */ + + +#define NXT_RANDOM_KEY_SIZE 128 + + +nxt_inline void nxt_random_start_schedule(nxt_random_t *r); +static void nxt_random_stir(nxt_random_t *r); +static void nxt_random_add(nxt_random_t *r, const u_char *key, uint32_t len); +nxt_inline uint8_t nxt_random_byte(nxt_random_t *r); + + +void +nxt_random_init(nxt_random_t *r) +{ + nxt_random_start_schedule(r); + + nxt_random_stir(r); +} + + +nxt_inline void +nxt_random_start_schedule(nxt_random_t *r) +{ + nxt_uint_t i; + + r->i = 0; + r->j = 0; + + for (i = 0; i < 256; i++) { + r->s[i] = i; + } +} + + +static void +nxt_random_stir(nxt_random_t *r) +{ + int fd; + ssize_t n; + struct timeval tv; + union { + uint32_t value[3]; + u_char bytes[NXT_RANDOM_KEY_SIZE]; + } key; + + n = 0; + +#if (NXT_HAVE_GETRANDOM) + + /* Linux 3.17 getrandom(). */ + + n = getrandom(key, NXT_RANDOM_KEY_SIZE, 0); + +#endif + + if (n != NXT_RANDOM_KEY_SIZE) { + fd = open("/dev/urandom", O_RDONLY); + + if (fd >= 0) { + n = read(fd, &key, NXT_RANDOM_KEY_SIZE); + (void) close(fd); + } + } + + if (n != NXT_RANDOM_KEY_SIZE) { + (void) gettimeofday(&tv, NULL); + + /* XOR with stack garbage. */ + + key.value[0] ^= tv.tv_usec; + key.value[1] ^= tv.tv_sec; + key.value[2] ^= nxt_pid; + } + + nxt_random_add(r, key.bytes, NXT_RANDOM_KEY_SIZE); + + /* Drop the first 3072 bytes. */ + for (n = 3072; n != 0; n--) { + (void) nxt_random_byte(r); + } + + /* Stir again after 1,600,000 bytes. */ + r->count = 400000; +} + + +static void +nxt_random_add(nxt_random_t *r, const u_char *key, uint32_t len) +{ + uint8_t val; + uint32_t n; + + for (n = 0; n < 256; n++) { + val = r->s[r->i]; + r->j += val + key[n % len]; + + r->s[r->i] = r->s[r->j]; + r->s[r->j] = val; + + r->i++; + } + + /* This index is not decremented in RC4 algorithm. */ + r->i--; + + r->j = r->i; +} + + +uint32_t +nxt_random(nxt_random_t *r) +{ + uint32_t val; + + r->count--; + + if (r->count <= 0) { + nxt_random_stir(r); + } + + val = nxt_random_byte(r) << 24; + val |= nxt_random_byte(r) << 16; + val |= nxt_random_byte(r) << 8; + val |= nxt_random_byte(r); + + return val; +} + + +nxt_inline uint8_t +nxt_random_byte(nxt_random_t *r) +{ + uint8_t si, sj; + + r->i++; + si = r->s[r->i]; + r->j += si; + + sj = r->s[r->j]; + r->s[r->i] = sj; + r->s[r->j] = si; + + si += sj; + + return r->s[si]; +} + + +#if (NXT_LIB_UNIT_TEST) + +nxt_int_t +nxt_random_unit_test(nxt_thread_t *thr) +{ + nxt_uint_t n; + nxt_random_t r; + + nxt_random_start_schedule(&r); + + r.count = 400000; + + nxt_random_add(&r, (u_char *) "arc4random", sizeof("arc4random") - 1); + + /* + * Test arc4random() numbers. + * RC4 pseudorandom numbers would be 0x4642AFC3 and 0xBAF0FFF0. + */ + + if (nxt_random(&r) == 0xD6270B27) { + + for (n = 100000; n != 0; n--) { + (void) nxt_random(&r); + } + + if (nxt_random(&r) == 0x6FCAE186) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "arc4random unit test passed"); + + return NXT_OK; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "arc4random unit test failed"); + + return NXT_ERROR; +} + +#endif + +#endif diff --git a/src/nxt_random.h b/src/nxt_random.h new file mode 100644 index 00000000..bfc4e3b3 --- /dev/null +++ b/src/nxt_random.h @@ -0,0 +1,47 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_RANDOM_H_INCLUDED_ +#define _NXT_RANDOM_H_INCLUDED_ + + +#if (NXT_HAVE_ARC4RANDOM) + +/* + * arc4random() has been introduced in OpenBSD 2.1 and then was ported + * to FreeBSD 2.2.6, NetBSD 2.0, MacOSX and SmartOS. + * + * arc4random() automatically initializes itself in the first call and + * then reinitializes itself in the first call in every forked processes. + */ + +typedef void *nxt_random_t; + + +#define nxt_random_init(r) +#define nxt_random(r) arc4random() + +#else + +typedef struct { + uint8_t i; + uint8_t j; + uint8_t s[256]; + int32_t count; +} nxt_random_t; + + +void nxt_random_init(nxt_random_t *r); +uint32_t nxt_random(nxt_random_t *r); + +#if (NXT_LIB_UNIT_TEST) +nxt_int_t nxt_random_unit_test(nxt_thread_t *thr); +#endif + +#endif + + +#endif /* _NXT_RANDOM_H_INCLUDED_ */ diff --git a/src/nxt_rbtree.c b/src/nxt_rbtree.c new file mode 100644 index 00000000..ff043d59 --- /dev/null +++ b/src/nxt_rbtree.c @@ -0,0 +1,515 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * The red-black tree code is based on the algorithm described in + * the "Introduction to Algorithms" by Cormen, Leiserson and Rivest. + */ + + +static void nxt_rbtree_insert_fixup(nxt_rbtree_node_t *node); +static void nxt_rbtree_delete_fixup(nxt_rbtree_t *tree, + nxt_rbtree_node_t *node); +nxt_inline void nxt_rbtree_left_rotate(nxt_rbtree_node_t *node); +nxt_inline void nxt_rbtree_right_rotate(nxt_rbtree_node_t *node); +nxt_inline void nxt_rbtree_parent_relink(nxt_rbtree_node_t *subst, + nxt_rbtree_node_t *node); + + +#define NXT_RBTREE_BLACK 0 +#define NXT_RBTREE_RED 1 + + +#define nxt_rbtree_set_callback_type(tree, type) \ + (tree)->sentinel.spare = type + +#define nxt_rbtree_has_insertion_callback(tree) \ + ((tree)->sentinel.spare != 0) + +#define nxt_rbtree_comparison_callback(tree) \ + ((nxt_rbtree_compare_t) (tree)->sentinel.right) + + +void +nxt_rbtree_init(nxt_rbtree_t *tree, nxt_rbtree_compare_t compare, + nxt_rbtree_insert_t insert) +{ + void *callback; + nxt_bool_t insertion; + + /* + * The sentinel is used as a leaf node sentinel and as a tree root + * sentinel: it is a parent of a root node and the root node is + * the left child of the sentinel. Combining two sentinels in one + * entry and the fact that the sentinel's left child is a root node + * simplifies nxt_rbtree_node_successor() and eliminates explicit + * root node test before or inside nxt_rbtree_min(). + */ + + /* The root is empty. */ + tree->sentinel.left = &tree->sentinel; + + /* + * The sentinel's right child is never used so either insertion + * or comparison callback can be safely stored here. + */ + insertion = (insert != NULL); + nxt_rbtree_set_callback_type(tree, insertion); + callback = insertion ? (void *) insert : (void *) compare; + tree->sentinel.right = callback; + + /* The root and leaf sentinel must be black. */ + tree->sentinel.color = NXT_RBTREE_BLACK; +} + + +void +nxt_rbtree_insert(nxt_rbtree_t *tree, nxt_rbtree_part_t *part) +{ + void *callback; + nxt_rbtree_node_t *node, *new_node, *sentinel, **child; + nxt_rbtree_insert_t insert; + nxt_rbtree_compare_t compare; + + new_node = (nxt_rbtree_node_t *) part; + + node = nxt_rbtree_root(tree); + sentinel = nxt_rbtree_sentinel(tree); + + new_node->left = sentinel; + new_node->right = sentinel; + new_node->color = NXT_RBTREE_RED; + + callback = tree->sentinel.right; + + if (nxt_rbtree_has_insertion_callback(tree)) { + insert = (nxt_rbtree_insert_t) callback; + + insert(node, new_node, sentinel); + + } else { + compare = (nxt_rbtree_compare_t) callback; + + child = &nxt_rbtree_root(tree); + + while (*child != sentinel) { + node = *child; + + nxt_prefetch(node->left); + nxt_prefetch(node->right); + + child = (compare(new_node, node) < 0) ? &node->left : &node->right; + } + + *child = new_node; + + new_node->parent = node; + } + + nxt_rbtree_insert_fixup(new_node); + + node = nxt_rbtree_root(tree); + node->color = NXT_RBTREE_BLACK; +} + + +static void +nxt_rbtree_insert_fixup(nxt_rbtree_node_t *node) +{ + nxt_rbtree_node_t *parent, *grandparent, *uncle; + + /* + * Prefetching parent nodes does not help here because they are + * already traversed during insertion. + */ + + for ( ;; ) { + parent = node->parent; + + /* + * Testing whether a node is a tree root is not required here since + * a root node's parent is the sentinel and it is always black. + */ + if (parent->color == NXT_RBTREE_BLACK) { + return; + } + + grandparent = parent->parent; + + if (parent == grandparent->left) { + uncle = grandparent->right; + + if (uncle->color == NXT_RBTREE_BLACK) { + + if (node == parent->right) { + node = parent; + nxt_rbtree_left_rotate(node); + } + + parent = node->parent; + parent->color = NXT_RBTREE_BLACK; + + grandparent = parent->parent; + grandparent->color = NXT_RBTREE_RED; + nxt_rbtree_right_rotate(grandparent); + + continue; + } + + } else { + uncle = grandparent->left; + + if (uncle->color == NXT_RBTREE_BLACK) { + + if (node == parent->left) { + node = parent; + nxt_rbtree_right_rotate(node); + } + + parent = node->parent; + parent->color = NXT_RBTREE_BLACK; + + grandparent = parent->parent; + grandparent->color = NXT_RBTREE_RED; + nxt_rbtree_left_rotate(grandparent); + + continue; + } + } + + uncle->color = NXT_RBTREE_BLACK; + parent->color = NXT_RBTREE_BLACK; + grandparent->color = NXT_RBTREE_RED; + + node = grandparent; + } +} + + +nxt_rbtree_node_t * +nxt_rbtree_find(nxt_rbtree_t *tree, nxt_rbtree_part_t *part) +{ + nxt_int_t n; + nxt_rbtree_node_t *node, *next, *sentinel; + nxt_rbtree_compare_t compare; + + node = (nxt_rbtree_node_t *) part; + + next = nxt_rbtree_root(tree); + sentinel = nxt_rbtree_sentinel(tree); + compare = nxt_rbtree_comparison_callback(tree); + + while (next != sentinel) { + nxt_prefetch(next->left); + nxt_prefetch(next->right); + + n = compare(node, next); + + if (n < 0) { + next = next->left; + + } else if (n > 0) { + next = next->right; + + } else { + return next; + } + } + + return NULL; +} + + +nxt_rbtree_node_t * +nxt_rbtree_find_less_or_equal(nxt_rbtree_t *tree, nxt_rbtree_part_t *part) +{ + nxt_int_t n; + nxt_rbtree_node_t *node, *retval, *next, *sentinel; + nxt_rbtree_compare_t compare; + + node = (nxt_rbtree_node_t *) part; + + retval = NULL; + next = nxt_rbtree_root(tree); + sentinel = nxt_rbtree_sentinel(tree); + compare = nxt_rbtree_comparison_callback(tree); + + while (next != sentinel) { + nxt_prefetch(next->left); + nxt_prefetch(next->right); + + n = compare(node, next); + + if (n < 0) { + next = next->left; + + } else if (n > 0) { + retval = next; + next = next->right; + + } else { + /* Exact match. */ + return next; + } + } + + return retval; +} + + +nxt_rbtree_node_t * +nxt_rbtree_find_greater_or_equal(nxt_rbtree_t *tree, nxt_rbtree_part_t *part) +{ + nxt_int_t n; + nxt_rbtree_node_t *node, *retval, *next, *sentinel; + nxt_rbtree_compare_t compare; + + node = (nxt_rbtree_node_t *) part; + + retval = NULL; + next = nxt_rbtree_root(tree); + sentinel = nxt_rbtree_sentinel(tree); + compare = nxt_rbtree_comparison_callback(tree); + + while (next != sentinel) { + nxt_prefetch(next->left); + nxt_prefetch(next->right); + + n = compare(node, next); + + if (n < 0) { + retval = next; + next = next->left; + + } else if (n > 0) { + next = next->right; + + } else { + /* Exact match. */ + return next; + } + } + + return retval; +} + + +void +nxt_rbtree_delete(nxt_rbtree_t *tree, nxt_rbtree_part_t *part) +{ + nxt_uint_t color; + nxt_rbtree_node_t *node, *sentinel, *subst, *child; + + node = (nxt_rbtree_node_t *) part; + + subst = node; + sentinel = nxt_rbtree_sentinel(tree); + + if (node->left == sentinel) { + child = node->right; + + } else if (node->right == sentinel) { + child = node->left; + + } else { + subst = nxt_rbtree_branch_min(tree, node->right); + child = subst->right; + } + + nxt_rbtree_parent_relink(child, subst); + + color = subst->color; + + if (subst != node) { + /* Move the subst node to the deleted node position in the tree. */ + + subst->color = node->color; + + subst->left = node->left; + subst->left->parent = subst; + + subst->right = node->right; + subst->right->parent = subst; + + nxt_rbtree_parent_relink(subst, node); + } + +#if (NXT_DEBUG) + node->left = NULL; + node->right = NULL; + node->parent = NULL; +#endif + + if (color == NXT_RBTREE_BLACK) { + nxt_rbtree_delete_fixup(tree, child); + } +} + + +static void +nxt_rbtree_delete_fixup(nxt_rbtree_t *tree, nxt_rbtree_node_t *node) +{ + nxt_rbtree_node_t *parent, *sibling; + + while (node != nxt_rbtree_root(tree) && node->color == NXT_RBTREE_BLACK) { + /* + * Prefetching parent nodes does not help here according + * to microbenchmarks. + */ + + parent = node->parent; + + if (node == parent->left) { + sibling = parent->right; + + if (sibling->color != NXT_RBTREE_BLACK) { + + sibling->color = NXT_RBTREE_BLACK; + parent->color = NXT_RBTREE_RED; + + nxt_rbtree_left_rotate(parent); + + sibling = parent->right; + } + + if (sibling->right->color == NXT_RBTREE_BLACK) { + + sibling->color = NXT_RBTREE_RED; + + if (sibling->left->color == NXT_RBTREE_BLACK) { + node = parent; + continue; + } + + sibling->left->color = NXT_RBTREE_BLACK; + + nxt_rbtree_right_rotate(sibling); + /* + * If the node is the leaf sentinel then the right + * rotate above changes its parent so a sibling below + * becames the leaf sentinel as well and this causes + * segmentation fault. This is the reason why usual + * red-black tree implementations with a leaf sentinel + * which does not require to test leaf nodes at all + * nevertheless test the leaf sentinel in the left and + * right rotate procedures. Since according to the + * algorithm node->parent must not be changed by both + * the left and right rotates above, it can be cached + * in a local variable. This not only eliminates the + * sentinel test in nxt_rbtree_parent_relink() but also + * decreases the code size because C forces to reload + * non-restrict pointers. + */ + sibling = parent->right; + } + + sibling->color = parent->color; + parent->color = NXT_RBTREE_BLACK; + sibling->right->color = NXT_RBTREE_BLACK; + + nxt_rbtree_left_rotate(parent); + + break; + + } else { + sibling = parent->left; + + if (sibling->color != NXT_RBTREE_BLACK) { + + sibling->color = NXT_RBTREE_BLACK; + parent->color = NXT_RBTREE_RED; + + nxt_rbtree_right_rotate(parent); + + sibling = parent->left; + } + + if (sibling->left->color == NXT_RBTREE_BLACK) { + + sibling->color = NXT_RBTREE_RED; + + if (sibling->right->color == NXT_RBTREE_BLACK) { + node = parent; + continue; + } + + sibling->right->color = NXT_RBTREE_BLACK; + + nxt_rbtree_left_rotate(sibling); + + /* See the comment in the symmetric branch above. */ + sibling = parent->left; + } + + sibling->color = parent->color; + parent->color = NXT_RBTREE_BLACK; + sibling->left->color = NXT_RBTREE_BLACK; + + nxt_rbtree_right_rotate(parent); + + break; + } + } + + node->color = NXT_RBTREE_BLACK; +} + + +nxt_inline void +nxt_rbtree_left_rotate(nxt_rbtree_node_t *node) +{ + nxt_rbtree_node_t *child; + + child = node->right; + node->right = child->left; + child->left->parent = node; + child->left = node; + + nxt_rbtree_parent_relink(child, node); + + node->parent = child; +} + + +nxt_inline void +nxt_rbtree_right_rotate(nxt_rbtree_node_t *node) +{ + nxt_rbtree_node_t *child; + + child = node->left; + node->left = child->right; + child->right->parent = node; + child->right = node; + + nxt_rbtree_parent_relink(child, node); + + node->parent = child; +} + + +/* Relink a parent from the node to the subst node. */ + +nxt_inline void +nxt_rbtree_parent_relink(nxt_rbtree_node_t *subst, nxt_rbtree_node_t *node) +{ + nxt_rbtree_node_t *parent, **link; + + parent = node->parent; + /* + * The leaf sentinel's parent can be safely changed here. + * See the comment in nxt_rbtree_delete_fixup() for details. + */ + subst->parent = parent; + /* + * If the node's parent is the root sentinel it is safely changed + * because the root sentinel's left child is the tree root. + */ + link = (node == parent->left) ? &parent->left : &parent->right; + *link = subst; +} diff --git a/src/nxt_rbtree.h b/src/nxt_rbtree.h new file mode 100644 index 00000000..63b759fb --- /dev/null +++ b/src/nxt_rbtree.h @@ -0,0 +1,120 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_RBTREE_H_INCLUDED_ +#define _NXT_RBTREE_H_INCLUDED_ + + +typedef struct nxt_rbtree_node_s nxt_rbtree_node_t; + +struct nxt_rbtree_node_s { + nxt_rbtree_node_t *left; + nxt_rbtree_node_t *right; + nxt_rbtree_node_t *parent; + + uint8_t color; + uint8_t spare; +}; + + +typedef struct { + nxt_rbtree_node_t *left; + nxt_rbtree_node_t *right; + nxt_rbtree_node_t *parent; +} nxt_rbtree_part_t; + + +#define NXT_RBTREE_NODE(node) \ + nxt_rbtree_part_t node; \ + uint8_t color + + +#define NXT_RBTREE_NODE_INIT { NULL, NULL, NULL }, 0 + + +typedef struct { + nxt_rbtree_node_t sentinel; +} nxt_rbtree_t; + + +typedef void (*nxt_rbtree_insert_t)(nxt_rbtree_node_t *root, + nxt_rbtree_node_t *node, nxt_rbtree_node_t *sentinel); + +typedef nxt_int_t (*nxt_rbtree_compare_t)(nxt_rbtree_node_t *node1, + nxt_rbtree_node_t *node2); + + +#define \ +nxt_rbtree_root(tree) \ + ((tree)->sentinel.left) + + +#define nxt_rbtree_sentinel(tree) \ + (&(tree)->sentinel) + + +#define nxt_rbtree_is_empty(tree) \ + (nxt_rbtree_root(tree) == nxt_rbtree_sentinel(tree)) + + +#define nxt_rbtree_min(tree) \ + nxt_rbtree_branch_min(tree, &(tree)->sentinel) + + +nxt_inline nxt_rbtree_node_t * +nxt_rbtree_branch_min(nxt_rbtree_t *tree, nxt_rbtree_node_t *node) +{ + while (node->left != nxt_rbtree_sentinel(tree)) { + node = node->left; + } + + return node; +} + + +#define nxt_rbtree_is_there_successor(tree, node) \ + ((node) != nxt_rbtree_sentinel(tree)) + + +nxt_inline nxt_rbtree_node_t * +nxt_rbtree_node_successor(nxt_rbtree_t *tree, nxt_rbtree_node_t *node) +{ + nxt_rbtree_node_t *parent; + + if (node->right != nxt_rbtree_sentinel(tree)) { + return nxt_rbtree_branch_min(tree, node->right); + } + + for ( ;; ) { + parent = node->parent; + + /* + * Explicit test for a root node is not required here, because + * the root node is always the left child of the sentinel. + */ + if (node == parent->left) { + return parent; + } + + node = parent; + } +} + + +NXT_EXPORT void nxt_rbtree_init(nxt_rbtree_t *tree, + nxt_rbtree_compare_t compare, nxt_rbtree_insert_t insert); +NXT_EXPORT void nxt_rbtree_insert(nxt_rbtree_t *tree, nxt_rbtree_part_t *node); +NXT_EXPORT nxt_rbtree_node_t *nxt_rbtree_find(nxt_rbtree_t *tree, + nxt_rbtree_part_t *node); +NXT_EXPORT nxt_rbtree_node_t *nxt_rbtree_find_less_or_equal(nxt_rbtree_t *tree, + nxt_rbtree_part_t *node); +NXT_EXPORT nxt_rbtree_node_t * + nxt_rbtree_find_greater_or_equal(nxt_rbtree_t *tree, + nxt_rbtree_part_t *node); +NXT_EXPORT void nxt_rbtree_delete(nxt_rbtree_t *tree, nxt_rbtree_part_t *node); + + +#endif /* _NXT_RBTREE_H_INCLUDED_ */ diff --git a/src/nxt_recvbuf.c b/src/nxt_recvbuf.c new file mode 100644 index 00000000..29a2b65d --- /dev/null +++ b/src/nxt_recvbuf.c @@ -0,0 +1,82 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_uint_t +nxt_recvbuf_mem_coalesce(nxt_recvbuf_coalesce_t *rb) +{ + u_char *last; + size_t size, total; + nxt_int_t n; + nxt_buf_t *b; + + total = 0; + last = NULL; + n = -1; + + for (b = rb->buf; b != NULL; b = b->next) { + + nxt_prefetch(b->next); + + size = b->mem.end - b->mem.free; + + if (b->mem.free != last) { + + if (++n >= rb->nmax) { + goto done; + } + + nxt_iobuf_set(&rb->iobuf[n], b->mem.free, size); + + } else { + nxt_iobuf_add(&rb->iobuf[n], size); + } + + nxt_thread_log_debug("recvbuf: %ui, %p, %uz", n, + nxt_iobuf_data(&rb->iobuf[n]), + nxt_iobuf_size(&rb->iobuf[n])); + + total += size; + last = b->mem.end; + } + + n++; + +done: + + rb->size = total; + + return n; +} + + +void +nxt_recvbuf_update(nxt_buf_t *b, size_t sent) +{ + size_t size; + + while (b != NULL && sent != 0) { + + nxt_prefetch(b->next); + + if (!nxt_buf_is_sync(b)) { + + size = b->mem.end - b->mem.free; + + if (sent < size) { + b->mem.free += sent; + return; + } + + b->mem.free = b->mem.end; + sent -= size; + } + + b = b->next; + } +} diff --git a/src/nxt_recvbuf.h b/src/nxt_recvbuf.h new file mode 100644 index 00000000..69b51498 --- /dev/null +++ b/src/nxt_recvbuf.h @@ -0,0 +1,24 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_RECVBUF_H_INCLUDED_ +#define _NXT_RECVBUF_H_INCLUDED_ + + +typedef struct { + nxt_buf_t *buf; + nxt_iobuf_t *iobuf; + + int32_t nmax; + size_t size; +} nxt_recvbuf_coalesce_t; + + +nxt_uint_t nxt_recvbuf_mem_coalesce(nxt_recvbuf_coalesce_t *rb); +void nxt_recvbuf_update(nxt_buf_t *b, size_t sent); + + +#endif /* _NXT_RECVBUF_H_INCLUDED_ */ diff --git a/src/nxt_select.c b/src/nxt_select.c new file mode 100644 index 00000000..a9713ac0 --- /dev/null +++ b/src/nxt_select.c @@ -0,0 +1,393 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_event_set_t *nxt_select_create(nxt_event_signals_t *signals, + nxt_uint_t mchanges, nxt_uint_t mevents); +static void nxt_select_free(nxt_event_set_t *event_set); +static void nxt_select_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_select_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev); +static void nxt_select_enable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_enable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_error_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_select_disable_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_disable_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_block_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_block_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_oneshot_read(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_oneshot_write(nxt_event_set_t *event_set, + nxt_event_fd_t *ev); +static void nxt_select_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout); + + +const nxt_event_set_ops_t nxt_select_event_set = { + "select", + nxt_select_create, + nxt_select_free, + nxt_select_enable, + nxt_select_disable, + nxt_select_disable, + nxt_select_disable, + nxt_select_enable_read, + nxt_select_enable_write, + nxt_select_disable_read, + nxt_select_disable_write, + nxt_select_block_read, + nxt_select_block_write, + nxt_select_oneshot_read, + nxt_select_oneshot_write, + nxt_select_enable_read, + NULL, + NULL, + NULL, + NULL, + nxt_select_poll, + + &nxt_unix_event_conn_io, + + NXT_NO_FILE_EVENTS, + NXT_NO_SIGNAL_EVENTS, +}; + + +static nxt_event_set_t * +nxt_select_create(nxt_event_signals_t *signals, nxt_uint_t mchanges, + nxt_uint_t mevents) +{ + nxt_event_set_t *event_set; + nxt_select_event_set_t *ss; + + event_set = nxt_zalloc(sizeof(nxt_select_event_set_t)); + if (event_set == NULL) { + return NULL; + } + + ss = &event_set->select; + + ss->nfds = -1; + ss->update_nfds = 0; + + ss->events = nxt_zalloc(FD_SETSIZE * sizeof(nxt_event_fd_t *)); + if (ss->events != NULL) { + return event_set; + } + + nxt_select_free(event_set); + + return NULL; +} + + +static void +nxt_select_free(nxt_event_set_t *event_set) +{ + nxt_select_event_set_t *ss; + + nxt_main_log_debug("select free"); + + ss = &event_set->select; + + nxt_free(ss->events); + nxt_free(ss); +} + + +static void +nxt_select_enable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_select_enable_read(event_set, ev); + nxt_select_enable_write(event_set, ev); +} + + +static void +nxt_select_disable(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + nxt_select_disable_read(event_set, ev); + } + + if (ev->write != NXT_EVENT_INACTIVE) { + nxt_select_disable_write(event_set, ev); + } +} + + +static void +nxt_select_enable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_fd_t fd; + nxt_thread_t *thr; + nxt_select_event_set_t *ss; + + fd = ev->fd; + + nxt_log_debug(ev->log, "select enable read: fd:%d", fd); + + ss = &event_set->select; + + if (fd < 0 || fd >= (nxt_fd_t) FD_SETSIZE) { + thr = nxt_thread(); + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_select_error_handler, + ev, ev->data, ev->log); + return; + } + + ev->read = NXT_EVENT_DEFAULT; + + FD_SET(fd, &ss->main_read_fd_set); + ss->events[fd] = ev; + + if (ss->nfds < fd) { + ss->nfds = fd; + ss->update_nfds = 0; + } +} + + +static void +nxt_select_enable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_fd_t fd; + nxt_thread_t *thr; + nxt_select_event_set_t *ss; + + fd = ev->fd; + + nxt_log_debug(ev->log, "select enable write: fd:%d", fd); + + ss = &event_set->select; + + if (fd < 0 || fd >= (nxt_fd_t) FD_SETSIZE) { + thr = nxt_thread(); + nxt_thread_work_queue_add(thr, &thr->work_queue.main, + nxt_select_error_handler, + ev, ev->data, ev->log); + return; + } + + ev->write = NXT_EVENT_DEFAULT; + + FD_SET(fd, &ss->main_write_fd_set); + ss->events[fd] = ev; + + if (ss->nfds < fd) { + ss->nfds = fd; + ss->update_nfds = 0; + } +} + + +static void +nxt_select_error_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_fd_t *ev; + + ev = obj; + + ev->read = NXT_EVENT_INACTIVE; + ev->write = NXT_EVENT_INACTIVE; + + ev->error_handler(thr, ev, data); +} + + +static void +nxt_select_disable_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_fd_t fd; + nxt_select_event_set_t *ss; + + fd = ev->fd; + + nxt_log_debug(ev->log, "select disable read: fd:%d", fd); + + if (fd < 0 || fd >= (nxt_fd_t) FD_SETSIZE) { + return; + } + + ss = &event_set->select; + FD_CLR(fd, &ss->main_read_fd_set); + + ev->read = NXT_EVENT_INACTIVE; + + if (ev->write == NXT_EVENT_INACTIVE) { + ss->events[fd] = NULL; + ss->update_nfds = 1; + } +} + + +static void +nxt_select_disable_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_fd_t fd; + nxt_select_event_set_t *ss; + + fd = ev->fd; + + nxt_log_debug(ev->log, "select disable write: fd:%d", fd); + + if (fd < 0 || fd >= (nxt_fd_t) FD_SETSIZE) { + return; + } + + ss = &event_set->select; + FD_CLR(fd, &ss->main_write_fd_set); + + ev->write = NXT_EVENT_INACTIVE; + + if (ev->read == NXT_EVENT_INACTIVE) { + ss->events[fd] = NULL; + ss->update_nfds = 1; + } +} + + +static void +nxt_select_block_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->read != NXT_EVENT_INACTIVE) { + nxt_select_disable_read(event_set, ev); + } +} + + +static void +nxt_select_block_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + if (ev->write != NXT_EVENT_INACTIVE) { + nxt_select_disable_write(event_set, ev); + } +} + + +static void +nxt_select_oneshot_read(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_select_enable_read(event_set, ev); + + ev->read = NXT_EVENT_ONESHOT; +} + + +static void +nxt_select_oneshot_write(nxt_event_set_t *event_set, nxt_event_fd_t *ev) +{ + nxt_select_enable_write(event_set, ev); + + ev->write = NXT_EVENT_ONESHOT; +} + + +static void +nxt_select_poll(nxt_thread_t *thr, nxt_event_set_t *event_set, + nxt_msec_t timeout) +{ + int nevents, nfds, found; + nxt_err_t err; + nxt_int_t i; + nxt_uint_t fd, level; + nxt_event_fd_t *ev; + struct timeval tv, *tp; + nxt_select_event_set_t *ss; + + if (timeout == NXT_INFINITE_MSEC) { + tp = NULL; + + } else { + tv.tv_sec = (long) (timeout / 1000); + tv.tv_usec = (long) ((timeout % 1000) * 1000); + tp = &tv; + } + + ss = &event_set->select; + + if (ss->update_nfds) { + for (i = ss->nfds; i >= 0; i--) { + if (ss->events[i] != NULL) { + ss->nfds = i; + ss->update_nfds = 0; + break; + } + } + } + + ss->work_read_fd_set = ss->main_read_fd_set; + ss->work_write_fd_set = ss->main_write_fd_set; + + nfds = ss->nfds + 1; + + nxt_log_debug(thr->log, "select() nfds:%d timeout:%M", nfds, timeout); + + nevents = select(nfds, &ss->work_read_fd_set, &ss->work_write_fd_set, + NULL, tp); + + err = (nevents == -1) ? nxt_errno : 0; + + nxt_thread_time_update(thr); + + nxt_log_debug(thr->log, "select(): %d", nevents); + + if (nevents == -1) { + level = (err == NXT_EINTR) ? NXT_LOG_INFO : NXT_LOG_ALERT; + nxt_log_error(level, thr->log, "select() failed %E", err); + return; + } + + for (fd = 0; fd < (nxt_uint_t) nfds && nevents != 0; fd++) { + + found = 0; + + if (FD_ISSET(fd, &ss->work_read_fd_set)) { + ev = ss->events[fd]; + + nxt_log_debug(ev->log, "select() fd:%ui read rd:%d wr:%d", + fd, ev->read, ev->write); + + ev->read_ready = 1; + + if (ev->read == NXT_EVENT_ONESHOT) { + nxt_select_disable_read(event_set, ev); + } + + nxt_thread_work_queue_add(thr, ev->read_work_queue, + ev->read_handler, ev, ev->data, ev->log); + found = 1; + } + + if (FD_ISSET(fd, &ss->work_write_fd_set)) { + ev = ss->events[fd]; + + nxt_log_debug(ev->log, "select() fd:%ui write rd:%d wr:%d", + fd, ev->read, ev->write); + + ev->write_ready = 1; + + if (ev->write == NXT_EVENT_ONESHOT) { + nxt_select_disable_write(event_set, ev); + } + + nxt_thread_work_queue_add(thr, ev->write_work_queue, + ev->write_handler, ev, ev->data, ev->log); + found = 1; + } + + nevents -= found; + } +} diff --git a/src/nxt_semaphore.c b/src/nxt_semaphore.c new file mode 100644 index 00000000..ad05d8b9 --- /dev/null +++ b/src/nxt_semaphore.c @@ -0,0 +1,244 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#if (NXT_HAVE_SEM_TIMEDWAIT) + +/* + * Linux POSIX semaphores use atomic/futex operations in since glibc 2.3. + * + * FreeBSD has two POSIX semaphore implementations. The first implementation + * has been introduced in FreeBSD 5.0 but it has some drawbacks: + * 1) it had a bug (http://bugs.freebsd.org/127545) fixed in FreeBSD 7.2; + * 2) it does not use atomic operations and always calls ksem syscalls; + * 3) a number of semaphores is just 30 by default and until FreeBSD 8.1 + * the number cannot be changed after boot time. + * + * The second implementation has been introduced in FreeBSD 6.1 in libthr + * and uses atomic operations and umtx syscall. However, until FreeBSD 9.0 + * a choice of implementation depended on linking order of libthr and libc. + * In FreeBSD 9.0 the umtx implementation has been moved to libc. + * + * Solaris have POSIX semaphores. + * + * MacOSX has limited POSIX semaphore implementation: + * 1) sem_init() exists but returns ENOSYS; + * 2) no sem_timedwait(). + */ + +nxt_int_t +nxt_sem_init(nxt_sem_t *sem, nxt_uint_t count) +{ + if (sem_init(sem, 0, count) == 0) { + nxt_thread_log_debug("sem_init(%p)", sem); + return NXT_OK; + } + + nxt_thread_log_alert("sem_init(%p) failed %E", sem, nxt_errno); + return NXT_ERROR; +} + + +void +nxt_sem_destroy(nxt_sem_t *sem) +{ + if (sem_destroy(sem) == 0) { + nxt_thread_log_debug("sem_destroy(%p)", sem); + return; + } + + nxt_thread_log_alert("sem_destroy(%p) failed %E", sem, nxt_errno); +} + + +nxt_int_t +nxt_sem_post(nxt_sem_t *sem) +{ + nxt_thread_log_debug("sem_post(%p)", sem); + + if (nxt_fast_path(sem_post(sem) == 0)) { + return NXT_OK; + } + + nxt_thread_log_alert("sem_post(%p) failed %E", sem, nxt_errno); + + return NXT_ERROR; +} + + +nxt_err_t +nxt_sem_wait(nxt_sem_t *sem, nxt_nsec_t timeout) +{ + int n; + nxt_err_t err; + nxt_nsec_t ns; + nxt_thread_t *thr; + nxt_realtime_t *now; + struct timespec ts; + + thr = nxt_thread(); + + if (timeout == NXT_INFINITE_NSEC) { + nxt_log_debug(thr->log, "sem_wait(%p) enter", sem); + + for ( ;; ) { + n = sem_wait(sem); + + err = nxt_errno; + + nxt_thread_time_update(thr); + + if (nxt_fast_path(n == 0)) { + nxt_thread_log_debug("sem_wait(%p) exit", sem); + return 0; + } + + switch (err) { + + case NXT_EINTR: + nxt_log_error(NXT_LOG_INFO, thr->log, "sem_wait(%p) failed %E", + sem, err); + continue; + + default: + nxt_log_alert(thr->log, "sem_wait(%p) failed %E", sem, err); + return err; + } + } + } + +#if (NXT_HAVE_SEM_TRYWAIT_FAST) + + nxt_log_debug(thr->log, "sem_trywait(%p) enter", sem); + + /* + * Fast sem_trywait() using atomic operations may eliminate + * timeout processing. + */ + + if (nxt_fast_path(sem_trywait(sem) == 0)) { + return 0; + } + +#endif + + nxt_log_debug(thr->log, "sem_timedwait(%p, %N) enter", sem, timeout); + + now = nxt_thread_realtime(thr); + ns = now->nsec + timeout; + ts.tv_sec = now->sec + ns / 1000000000; + ts.tv_nsec = ns % 1000000000; + + for ( ;; ) { + n = sem_timedwait(sem, &ts); + + err = nxt_errno; + + nxt_thread_time_update(thr); + + if (nxt_fast_path(n == 0)) { + nxt_thread_log_debug("sem_timedwait(%p) exit", sem); + return 0; + } + + switch (err) { + + case NXT_ETIMEDOUT: + nxt_log_debug(thr->log, "sem_timedwait(%p) exit: %d", sem, err); + return err; + + case NXT_EINTR: + nxt_log_error(NXT_LOG_INFO, thr->log, "sem_timedwait(%p) failed %E", + sem, err); + continue; + + default: + nxt_log_alert(thr->log, "sem_timedwait(%p) failed %E", sem, err); + return err; + } + } +} + +#else + +/* Semaphore implementation using pthread conditional variable. */ + +nxt_int_t +nxt_sem_init(nxt_sem_t *sem, nxt_uint_t count) +{ + if (nxt_thread_mutex_create(&sem->mutex) == NXT_OK) { + + if (nxt_thread_cond_create(&sem->cond) == NXT_OK) { + sem->count = count; + return NXT_OK; + } + + nxt_thread_mutex_destroy(&sem->mutex); + } + + return NXT_ERROR; +} + + +void +nxt_sem_destroy(nxt_sem_t *sem) +{ + nxt_thread_cond_destroy(&sem->cond); + nxt_thread_mutex_destroy(&sem->mutex); +} + + +nxt_int_t +nxt_sem_post(nxt_sem_t *sem) +{ + nxt_int_t ret; + + if (nxt_slow_path(nxt_thread_mutex_lock(&sem->mutex) != NXT_OK)) { + return NXT_ERROR; + } + + ret = nxt_thread_cond_signal(&sem->cond); + + sem->count++; + + /* NXT_ERROR overrides NXT_OK. */ + + return (nxt_thread_mutex_unlock(&sem->mutex) | ret); +} + + +nxt_err_t +nxt_sem_wait(nxt_sem_t *sem, nxt_nsec_t timeout) +{ + nxt_err_t err; + + err = 0; + + if (nxt_slow_path(nxt_thread_mutex_lock(&sem->mutex) != NXT_OK)) { + return NXT_ERROR; + } + + while (sem->count == 0) { + + err = nxt_thread_cond_wait(&sem->cond, &sem->mutex, timeout); + + if (err != 0) { + goto error; + } + } + + sem->count--; + +error: + + /* NXT_ERROR overrides NXT_OK and NXT_ETIMEDOUT. */ + + return (nxt_thread_mutex_unlock(&sem->mutex) | err); +} + +#endif diff --git a/src/nxt_semaphore.h b/src/nxt_semaphore.h new file mode 100644 index 00000000..d1985342 --- /dev/null +++ b/src/nxt_semaphore.h @@ -0,0 +1,32 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_SEMAPHORE_H_INCLUDED_ +#define _NXT_UNIX_SEMAPHORE_H_INCLUDED_ + + +#if (NXT_HAVE_SEM_TIMEDWAIT) + +typedef sem_t nxt_sem_t; + +#else + +typedef struct { + nxt_atomic_t count; + nxt_thread_mutex_t mutex; + nxt_thread_cond_t cond; +} nxt_sem_t; + +#endif + + +NXT_EXPORT nxt_int_t nxt_sem_init(nxt_sem_t *sem, nxt_uint_t count); +NXT_EXPORT void nxt_sem_destroy(nxt_sem_t *sem); +NXT_EXPORT nxt_int_t nxt_sem_post(nxt_sem_t *sem); +NXT_EXPORT nxt_err_t nxt_sem_wait(nxt_sem_t *sem, nxt_nsec_t timeout); + + +#endif /* _NXT_UNIX_SEMAPHORE_H_INCLUDED_ */ diff --git a/src/nxt_sendbuf.c b/src/nxt_sendbuf.c new file mode 100644 index 00000000..d473a29b --- /dev/null +++ b/src/nxt_sendbuf.c @@ -0,0 +1,353 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_bool_t nxt_sendbuf_copy(nxt_buf_mem_t *bm, nxt_buf_t *b, + size_t *copied); + + +nxt_uint_t +nxt_sendbuf_mem_coalesce(nxt_sendbuf_coalesce_t *sb) +{ + u_char *last; + size_t size, total; + nxt_buf_t *b; + nxt_uint_t n; + + total = sb->size; + last = NULL; + n = (nxt_uint_t) -1; + + for (b = sb->buf; b != NULL && total < sb->limit; b = b->next) { + + nxt_prefetch(b->next); + + if (nxt_buf_is_file(b)) { + break; + } + + if (nxt_buf_is_mem(b)) { + + size = b->mem.free - b->mem.pos; + + if (size != 0) { + + if (total + size > sb->limit) { + size = sb->limit - total; + + if (size == 0) { + break; + } + } + + if (b->mem.pos != last) { + + if (++n >= sb->nmax) { + goto done; + } + + nxt_iobuf_set(&sb->iobuf[n], b->mem.pos, size); + + } else { + nxt_iobuf_add(&sb->iobuf[n], size); + } + + nxt_thread_log_debug("sendbuf: %ui, %p, %uz", n, + nxt_iobuf_data(&sb->iobuf[n]), + nxt_iobuf_size(&sb->iobuf[n])); + + total += size; + last = b->mem.pos + size; + } + + } else { + sb->sync = 1; + sb->last |= nxt_buf_is_last(b); + } + } + + n++; + +done: + + sb->buf = b; + sb->size = total; + + return n; +} + + +size_t +nxt_sendbuf_file_coalesce(nxt_sendbuf_coalesce_t *sb) +{ + size_t file_start, total; + nxt_fd_t fd; + nxt_off_t size, last; + nxt_buf_t *b; + + b = sb->buf; + fd = b->file->fd; + + total = sb->size; + + for ( ;; ) { + + nxt_prefetch(b->next); + + size = b->file_end - b->file_pos; + + if (total + size >= sb->limit) { + total = sb->limit; + break; + } + + total += size; + last = b->file_pos + size; + + b = b->next; + + if (b == NULL || !nxt_buf_is_file(b)) { + break; + } + + if (b->file_pos != last || b->file->fd != fd) { + break; + } + } + + sb->buf = b; + + file_start = sb->size; + sb->size = total; + + return total - file_start; +} + + +ssize_t +nxt_sendbuf_copy_coalesce(nxt_event_conn_t *c, nxt_buf_mem_t *bm, + nxt_buf_t *b, size_t limit) +{ + size_t size, bsize, copied; + ssize_t n; + nxt_bool_t flush; + + size = nxt_buf_mem_used_size(&b->mem); + bsize = nxt_buf_mem_size(bm); + + if (bsize != 0) { + + if (size > bsize && bm->pos == bm->free) { + /* + * A data buffer size is larger than the internal + * buffer size and the internal buffer is empty. + */ + goto no_buffer; + } + + if (bm->pos == NULL) { + bm->pos = nxt_malloc(bsize); + if (nxt_slow_path(bm->pos == NULL)) { + return NXT_ERROR; + } + + bm->start = bm->pos; + bm->free = bm->pos; + bm->end += (uintptr_t) bm->pos; + } + + copied = 0; + + flush = nxt_sendbuf_copy(bm, b, &copied); + + nxt_log_debug(c->socket.log, "sendbuf copy:%uz fl:%b", copied, flush); + + if (flush == 0) { + return copied; + } + + size = nxt_buf_mem_used_size(bm); + + if (size == 0 && nxt_buf_is_sync(b)) { + goto done; + } + + n = c->io->send(c, bm->pos, nxt_min(size, limit)); + + nxt_log_debug(c->socket.log, "sendbuf sent:%z", n); + + if (n > 0) { + bm->pos += n; + + if (bm->pos == bm->free) { + bm->pos = bm->start; + bm->free = bm->start; + } + + n = 0; + } + + return (copied != 0) ? (ssize_t) copied : n; + } + + /* No internal buffering. */ + + if (size == 0 && nxt_buf_is_sync(b)) { + goto done; + } + +no_buffer: + + return c->io->send(c, b->mem.pos, nxt_min(size, limit)); + +done: + + nxt_log_debug(c->socket.log, "sendbuf done"); + + return 0; +} + + +static nxt_bool_t +nxt_sendbuf_copy(nxt_buf_mem_t *bm, nxt_buf_t *b, size_t *copied) +{ + size_t size, bsize; + nxt_bool_t flush; + + flush = 0; + + do { + nxt_prefetch(b->next); + + if (nxt_buf_is_mem(b)) { + bsize = bm->end - bm->free; + size = b->mem.free - b->mem.pos; + size = nxt_min(size, bsize); + + nxt_memcpy(bm->free, b->mem.pos, size); + + *copied += size; + bm->free += size; + + if (bm->free == bm->end) { + return 1; + } + } + + flush |= nxt_buf_is_flush(b) || nxt_buf_is_last(b); + + b = b->next; + + } while (b != NULL); + + return flush; +} + + +nxt_buf_t * +nxt_sendbuf_update(nxt_buf_t *b, size_t sent) +{ + size_t size; + + while (b != NULL) { + + nxt_prefetch(b->next); + + if (!nxt_buf_is_sync(b)) { + + size = nxt_buf_used_size(b); + + if (size != 0) { + + if (sent == 0) { + break; + } + + if (sent < size) { + + if (nxt_buf_is_mem(b)) { + b->mem.pos += sent; + } + + if (nxt_buf_is_file(b)) { + b->file_pos += sent; + } + + break; + } + + /* b->mem.free is NULL in file-only buffer. */ + b->mem.pos = b->mem.free; + + if (nxt_buf_is_file(b)) { + b->file_pos = b->file_end; + } + + sent -= size; + } + } + + b = b->next; + } + + return b; +} + + +nxt_buf_t * +nxt_sendbuf_completion(nxt_thread_t *thr, nxt_work_queue_t *wq, nxt_buf_t *b, + size_t sent) +{ + size_t size; + + while (b != NULL) { + + nxt_prefetch(b->next); + + if (!nxt_buf_is_sync(b)) { + + size = nxt_buf_used_size(b); + + if (size != 0) { + + if (sent == 0) { + break; + } + + if (sent < size) { + + if (nxt_buf_is_mem(b)) { + b->mem.pos += sent; + } + + if (nxt_buf_is_file(b)) { + b->file_pos += sent; + } + + break; + } + + /* b->mem.free is NULL in file-only buffer. */ + b->mem.pos = b->mem.free; + + if (nxt_buf_is_file(b)) { + b->file_pos = b->file_end; + } + + sent -= size; + } + } + + nxt_thread_work_queue_add(thr, wq, b->completion_handler, + b, b->parent, thr->log); + + b = b->next; + } + + return b; +} diff --git a/src/nxt_sendbuf.h b/src/nxt_sendbuf.h new file mode 100644 index 00000000..338bb84f --- /dev/null +++ b/src/nxt_sendbuf.h @@ -0,0 +1,108 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_SENDBUF_H_INCLUDED_ +#define _NXT_SENDBUF_H_INCLUDED_ + + +/* + * The sendbuf interface is intended to send a buffer chain to a connection. + * It uses sendfile interface if available. Otherwise it can send only + * memory buffers, so file buffers must be read in memory in advance. + * + * The sendbuf interface sets c->socket.write_ready to appropriate state + * and returns: + * + * N > 0 if sendbuf sent N bytes. + * + * 0 if sendbuf was interrupted (EINTR and so on), + * or sendbuf sent previously buffered data, + * or single sync buffer has been encountered. + * In all these cases sendbuf is ready to continue + * operation, unless c->socket.write_ready is cleared. + * + * NXT_AGAIN if sendbuf did not send any bytes. + * + * NXT_ERROR if there was erorr. + * + * The sendbuf limit is size_t type since size_t is large enough and many + * sendfile implementations do not support anyway sending more than size_t + * at once. The limit support is located at the sendbuf level otherwise + * an additional limited chain must be created on each sendbuf call. + */ + + +typedef struct { + nxt_buf_t *buf; + nxt_iobuf_t *iobuf; + + uint32_t nmax; + uint8_t sync; /* 1 bit */ + uint8_t last; /* 1 bit */ + + size_t size; + size_t limit; +} nxt_sendbuf_coalesce_t; + + +#if (NXT_HAVE_LINUX_SENDFILE) +#define NXT_HAVE_SENDFILE 1 +ssize_t nxt_linux_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); +#endif + +#if (NXT_HAVE_FREEBSD_SENDFILE) +#define NXT_HAVE_SENDFILE 1 +ssize_t nxt_freebsd_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); +#endif + +#if (NXT_HAVE_SOLARIS_SENDFILEV) +#define NXT_HAVE_SENDFILE 1 +ssize_t nxt_solaris_event_conn_io_sendfilev(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); +#endif + +#if (NXT_HAVE_MACOSX_SENDFILE) +#define NXT_HAVE_SENDFILE 1 +ssize_t nxt_macosx_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); +#endif + +#if (NXT_HAVE_AIX_SEND_FILE) +#define NXT_HAVE_SENDFILE 1 +ssize_t nxt_aix_event_conn_io_send_file(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); +#endif + +#if (NXT_HAVE_HPUX_SENDFILE) +#define NXT_HAVE_SENDFILE 1 +ssize_t nxt_hpux_event_conn_io_sendfile(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); +#endif + +ssize_t nxt_event_conn_io_sendbuf(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); + + +nxt_uint_t nxt_sendbuf_mem_coalesce(nxt_sendbuf_coalesce_t *sb); +size_t nxt_sendbuf_file_coalesce(nxt_sendbuf_coalesce_t *sb); + +/* + * Auxiliary nxt_sendbuf_copy_coalesce() interface copies small memory + * buffers into internal buffer before output. It is intended for + * SSL/TLS libraries which lack vector I/O interface yet add noticeable + * overhead to each SSL/TLS record. + */ +ssize_t nxt_sendbuf_copy_coalesce(nxt_event_conn_t *c, nxt_buf_mem_t *bm, + nxt_buf_t *b, size_t limit); + +nxt_buf_t *nxt_sendbuf_update(nxt_buf_t *b, size_t sent); +nxt_buf_t *nxt_sendbuf_completion(nxt_thread_t *thr, nxt_work_queue_t *wq, + nxt_buf_t *b, size_t sent); + + +#endif /* _NXT_SENDBUF_H_INCLUDED_ */ diff --git a/src/nxt_service.c b/src/nxt_service.c new file mode 100644 index 00000000..a911f53e --- /dev/null +++ b/src/nxt_service.c @@ -0,0 +1,165 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static const nxt_service_t nxt_services[] = { + +#if (NXT_HAVE_KQUEUE) + { "engine", "kqueue", &nxt_kqueue_event_set }, +#endif + +#if (NXT_HAVE_EPOLL_EDGE) + { "engine", "epoll", &nxt_epoll_edge_event_set }, + { "engine", "epoll_edge", &nxt_epoll_edge_event_set }, + { "engine", "epoll_level", &nxt_epoll_level_event_set }, + +#elif (NXT_HAVE_EPOLL) + { "engine", "epoll", &nxt_epoll_level_event_set }, + { "engine", "epoll_level", &nxt_epoll_level_event_set }, +#endif + +#if (NXT_HAVE_EVENTPORT) + { "engine", "eventport", &nxt_eventport_event_set }, +#endif + +#if (NXT_HAVE_DEVPOLL) + { "engine", "devpoll", &nxt_devpoll_event_set }, + { "engine", "/dev/poll", &nxt_devpoll_event_set }, +#endif + +#if (NXT_HAVE_POLLSET) + { "engine", "pollset", &nxt_pollset_event_set }, +#endif + + { "engine", "poll", &nxt_poll_event_set }, + { "engine", "select", &nxt_select_event_set }, + +#if (NXT_HAVE_OPENSSL) + { "SSL/TLS", "OpenSSL", &nxt_openssl_lib }, + { "SSL/TLS", "openssl", &nxt_openssl_lib }, +#endif + +#if (NXT_HAVE_GNUTLS) + { "SSL/TLS", "GnuTLS", &nxt_gnutls_lib }, + { "SSL/TLS", "gnutls", &nxt_gnutls_lib }, +#endif + +#if (NXT_HAVE_CYASSL) + { "SSL/TLS", "CyaSSL", &nxt_cyassl_lib }, + { "SSL/TLS", "cyassl", &nxt_cyassl_lib }, +#endif + +}; + + +nxt_array_t * +nxt_services_init(nxt_mem_pool_t *mp) +{ + nxt_uint_t n; + nxt_array_t *services; + nxt_service_t *s; + const nxt_service_t *service; + + services = nxt_array_create(mp, 32, sizeof(nxt_service_t)); + + if (nxt_fast_path(services != NULL)) { + + service = nxt_services; + n = nxt_nitems(nxt_services); + + while (n != 0) { + s = nxt_array_add(services); + if (nxt_slow_path(s == NULL)) { + return NULL; + } + + *s = *service; + + service++; + n--; + } + } + + return services; +} + + +nxt_int_t +nxt_service_add(nxt_array_t *services, const nxt_service_t *service) +{ + nxt_uint_t n; + nxt_service_t *s; + + s = services->elts; + n = services->nelts; + + while (n != 0) { + if (nxt_strcmp(s->type, service->type) != 0) { + goto next; + } + + if (nxt_strcmp(s->name, service->name) != 0) { + goto next; + } + + nxt_thread_log_emerg("service \"%s:%s\" is duplicate", + service->type, service->name); + return NXT_ERROR; + + next: + + s++; + n--; + } + + s = nxt_array_add(services); + if (nxt_fast_path(s != NULL)) { + *s = *service; + return NXT_OK; + } + + return NXT_ERROR; +} + + +const void * +nxt_service_get(nxt_array_t *services, const char *type, const char *name) +{ + nxt_uint_t n; + const nxt_service_t *s; + + if (services != NULL) { + s = services->elts; + n = services->nelts; + + } else { + s = nxt_services; + n = nxt_nitems(nxt_services); + } + + while (n != 0) { + if (nxt_strcmp(s->type, type) == 0) { + + if (name == NULL) { + return s->service; + } + + if (nxt_strcmp(s->name, name) == 0) { + return s->service; + } + } + + s++; + n--; + } + + nxt_thread_log_emerg("service \"%s%s%s\" not found", + type, (name != NULL) ? ":" : "", name); + + return NULL; +} diff --git a/src/nxt_service.h b/src/nxt_service.h new file mode 100644 index 00000000..1484f73e --- /dev/null +++ b/src/nxt_service.h @@ -0,0 +1,30 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_SERVICE_H_INCLUDED_ +#define _NXT_SERVICE_H_INCLUDED_ + + +typedef struct { + const char *type; + const char *name; + const void *service; +} nxt_service_t; + + +#define \ +nxt_service_is_module(s) \ + ((s)->type == NULL) + + +NXT_EXPORT nxt_array_t *nxt_services_init(nxt_mem_pool_t *mp); +NXT_EXPORT nxt_int_t nxt_service_add(nxt_array_t *services, + const nxt_service_t *service); +NXT_EXPORT const void *nxt_service_get(nxt_array_t *services, const char *type, + const char *name); + + +#endif /* _NXT_SERVICE_H_INCLUDED_ */ diff --git a/src/nxt_signal.c b/src/nxt_signal.c new file mode 100644 index 00000000..e267e3eb --- /dev/null +++ b/src/nxt_signal.c @@ -0,0 +1,230 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * Signals are handled only via a main thread event engine work queue. + * There are three ways to route signals to the work queue: + * + * 1) Using signal event notifications if an event facility supports it: + * kqueue and epoll/signalfd. This method is used regardless of thread mode. + * + * 2) Multi-threaded mode: a dedicated signal thread which waits in sigwait() + * and post a signal number to the main thread event engine. + * + * 3) Single-threaded mode: a signal handler which posts a signal number + * to the event engine. + */ + + +static nxt_int_t nxt_signal_action(int signo, void (*handler)(int)); + + +nxt_event_signals_t * +nxt_event_engine_signals(const nxt_event_sig_t *sigev) +{ + nxt_event_signals_t *signals; + + signals = nxt_zalloc(sizeof(nxt_event_signals_t)); + if (signals == NULL) { + return NULL; + } + + signals->sigev = sigev; + + if (nxt_signal_action(SIGSYS, SIG_IGN) != NXT_OK) { + goto fail; + } + + if (nxt_signal_action(SIGPIPE, SIG_IGN) != NXT_OK) { + goto fail; + } + + sigemptyset(&signals->sigmask); + + while (sigev->signo != 0) { + sigaddset(&signals->sigmask, sigev->signo); + sigev++; + } + + if (sigprocmask(SIG_BLOCK, &signals->sigmask, NULL) != 0) { + nxt_main_log_alert("sigprocmask(SIG_BLOCK) failed %E", nxt_errno); + goto fail; + } + + return signals; + +fail: + + nxt_free(signals); + + return NULL; +} + + +static nxt_int_t +nxt_signal_action(int signo, void (*handler)(int)) +{ + struct sigaction sa; + + nxt_memzero(&sa, sizeof(struct sigaction)); + sigemptyset(&sa.sa_mask); + sa.sa_handler = handler; + + if (sigaction(signo, &sa, NULL) == 0) { + return NXT_OK; + } + + nxt_main_log_alert("sigaction(%d) failed %E", signo, nxt_errno); + + return NXT_ERROR; +} + + +static void +nxt_signal_handler(int signo) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + + /* Thread is running in a single context now. */ + thr->time.signal++; + + nxt_thread_time_update(thr); + + nxt_main_log_error(NXT_LOG_INFO, "signal handler: %d", signo); + + nxt_event_engine_signal(thr->engine, signo); + + thr->time.signal--; +} + + +#if (NXT_THREADS) + +static void nxt_signal_thread(void *data); + + +nxt_int_t +nxt_signal_thread_start(nxt_event_engine_t *engine) +{ + nxt_thread_link_t *link; + const nxt_event_sig_t *sigev; + + if (engine->signals->process == nxt_pid) { + return NXT_OK; + } + + if (sigprocmask(SIG_BLOCK, &engine->signals->sigmask, NULL) != 0) { + nxt_main_log_alert("sigprocmask(SIG_BLOCK) failed %E", nxt_errno); + return NXT_ERROR; + } + + /* + * kqueue sets signal handlers to SIG_IGN and sigwait() ignores + * them after the switch of event facility from "kqueue" to "select". + */ + + for (sigev = engine->signals->sigev; sigev->signo != 0; sigev++) { + if (nxt_signal_action(sigev->signo, nxt_signal_handler) != NXT_OK) { + return NXT_ERROR; + } + } + + link = nxt_zalloc(sizeof(nxt_thread_link_t)); + + if (nxt_fast_path(link != NULL)) { + link->start = nxt_signal_thread; + link->data = engine; + + if (nxt_thread_create(&engine->signals->thread, link) == NXT_OK) { + engine->signals->process = nxt_pid; + return NXT_OK; + } + } + + return NXT_ERROR; +} + + +static void +nxt_signal_thread(void *data) +{ + int signo; + nxt_err_t err; + nxt_thread_t *thr; + nxt_event_engine_t *engine; + + engine = data; + + thr = nxt_thread(); + + nxt_main_log_debug("signal thread"); + + for ( ;; ) { + err = sigwait(&engine->signals->sigmask, &signo); + + nxt_thread_time_update(thr); + + if (nxt_fast_path(err == 0)) { + nxt_main_log_error(NXT_LOG_INFO, "signo: %d", signo); + + nxt_event_engine_signal(engine, signo); + + } else { + nxt_main_log_alert("sigwait() failed %E", err); + } + } +} + + +void +nxt_signal_thread_stop(nxt_event_engine_t *engine) +{ + nxt_thread_handle_t thread; + + thread = engine->signals->thread; + + nxt_thread_cancel(thread); + nxt_thread_wait(thread); +} + + +#else /* !(NXT_THREADS) */ + + +nxt_int_t +nxt_signal_handlers_start(nxt_event_engine_t *engine) +{ + const nxt_event_sig_t *sigev; + + for (sigev = engine->signals->sigev; sigev->signo != 0; sigev++) { + if (nxt_signal_action(sigev->signo, nxt_signal_handler) != NXT_OK) { + return NXT_ERROR; + } + } + + if (sigprocmask(SIG_UNBLOCK, &engine->signals->sigmask, NULL) != 0) { + nxt_main_log_alert("sigprocmask(SIG_UNBLOCK) failed %E", nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + + +void +nxt_signal_handlers_stop(nxt_event_engine_t *engine) +{ + if (sigprocmask(SIG_BLOCK, &engine->signals->sigmask, NULL) != 0) { + nxt_main_log_alert("sigprocmask(SIG_BLOCK) failed %E", nxt_errno); + } +} + +#endif diff --git a/src/nxt_signal.h b/src/nxt_signal.h new file mode 100644 index 00000000..4141f39f --- /dev/null +++ b/src/nxt_signal.h @@ -0,0 +1,74 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_SIGNAL_H_INCLUDED_ +#define _NXT_UNIX_SIGNAL_H_INCLUDED_ + + +typedef struct nxt_event_sig_s nxt_event_sig_t; + +struct nxt_event_sig_s { + int signo; + nxt_work_handler_t handler; + const char *name; +}; + +#define nxt_event_signal(sig, handler) \ + { sig, handler, #sig } + +#define nxt_event_signal_end \ + { 0, NULL, NULL } + + +typedef struct { + /* Used by epoll and eventport. */ + nxt_work_handler_t handler; + + const nxt_event_sig_t *sigev; + sigset_t sigmask; + +#if (NXT_THREADS) + /* Used by the signal thread. */ + nxt_pid_t process; + nxt_thread_handle_t thread; +#endif +} nxt_event_signals_t; + + +nxt_event_signals_t *nxt_event_engine_signals(const nxt_event_sig_t *sigev); + +#if (NXT_THREADS) + +#define \ +nxt_event_engine_signals_start(engine) \ + nxt_signal_thread_start(engine) + +#define \ +nxt_event_engine_signals_stop(engine) \ + nxt_signal_thread_stop(engine) + + +NXT_EXPORT nxt_int_t nxt_signal_thread_start(nxt_event_engine_t *engine); +NXT_EXPORT void nxt_signal_thread_stop(nxt_event_engine_t *engine); + +#else /* !(NXT_THREADS) */ + +#define \ +nxt_event_engine_signals_start(engine) \ + nxt_signal_handlers_start(engine) + +#define \ +nxt_event_engine_signals_stop(engine) \ + nxt_signal_handlers_stop(engine) + + +NXT_EXPORT nxt_int_t nxt_signal_handlers_start(nxt_event_engine_t *engine); +NXT_EXPORT void nxt_signal_handlers_stop(nxt_event_engine_t *engine); + +#endif + + +#endif /* _NXT_UNIX_SIGNAL_H_INCLUDED_ */ diff --git a/src/nxt_sockaddr.c b/src/nxt_sockaddr.c new file mode 100644 index 00000000..b2515ef9 --- /dev/null +++ b/src/nxt_sockaddr.c @@ -0,0 +1,973 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#if (NXT_INET6) +static u_char *nxt_inet6_ntop(u_char *addr, u_char *buf, u_char *end); +#endif + +static nxt_int_t nxt_job_sockaddr_unix_parse(nxt_job_sockaddr_parse_t *jbs); +static nxt_int_t nxt_job_sockaddr_inet6_parse(nxt_job_sockaddr_parse_t *jbs); +static nxt_int_t nxt_job_sockaddr_inet_parse(nxt_job_sockaddr_parse_t *jbs); + + +nxt_sockaddr_t * +nxt_sockaddr_alloc(nxt_mem_pool_t *mp, socklen_t len) +{ + nxt_sockaddr_t *sa; + + /* + * The current struct sockaddr's define 32-bit fields at maximum + * and may define 64-bit AF_INET6 fields in the future. Alignment + * of memory allocated by nxt_mem_zalloc() is enough for these fields. + * If 128-bit alignment will be required then nxt_mem_malloc() and + * nxt_memzero() should be used instead. + */ + sa = nxt_mem_zalloc(mp, offsetof(nxt_sockaddr_t, u) + len); + + if (nxt_fast_path(sa != NULL)) { + nxt_socklen_set(sa, len); + } + + return sa; +} + + +nxt_sockaddr_t * +nxt_sockaddr_create(nxt_mem_pool_t *mp, struct sockaddr *sockaddr, + socklen_t len) +{ + size_t size, copy; + nxt_sockaddr_t *sa; + + size = len; + copy = len; + +#if (NXT_HAVE_UNIX_DOMAIN) + + /* + * Unspecified Unix domain sockaddr_un form and length are very + * platform depended (see comment in unix/socket.h). Here they are + * normalized to the sockaddr_un with single zero byte sun_path[]. + */ + + if (size <= offsetof(struct sockaddr_un, sun_path)) { + /* + * Small socket length means a short unspecified Unix domain + * socket address: + * + * getsockname() and getpeername() on OpenBSD prior to 5.3 + * return zero length and does not update a passed sockaddr + * buffer at all. + * + * Linux returns length equal to 2, i.e. sockaddr_un without + * sun_path[], unix(7): + * + * unnamed: A stream socket that has not been bound + * to a pathname using bind(2) has no name. Likewise, + * the two sockets created by socketpair(2) are unnamed. + * When the address of an unnamed socket is returned by + * getsockname(2), getpeername(2), and accept(2), its + * length is sizeof(sa_family_t), and sun_path should + * not be inspected. + */ + size = offsetof(struct sockaddr_un, sun_path) + 1; + +#if !(NXT_LINUX) + + } else if (sockaddr->sa_family == AF_UNIX && sockaddr->sa_data[0] == '\0') { + /* + * Omit nonsignificant zeros of the unspecified Unix domain socket + * address. This test is disabled for Linux since Linux abstract + * socket address also starts with zero. However Linux unspecified + * Unix domain socket address is short and is handled above. + */ + size = offsetof(struct sockaddr_un, sun_path) + 1; + copy = size; + +#endif + } + +#endif /* NXT_HAVE_UNIX_DOMAIN */ + + sa = nxt_sockaddr_alloc(mp, size); + + if (nxt_fast_path(sa != NULL)) { + + nxt_memcpy(&sa->u.sockaddr, sockaddr, copy); + +#if (NXT_SOCKADDR_SA_LEN) + + /* Update shortcut sockaddr length overwritten by nxt_memcpy(). */ + nxt_socklen_set(sa, size); + +#endif + +#if (NXT_HAVE_UNIX_DOMAIN && NXT_OPENBSD) + + if (len == 0) { + sa->u.sockaddr.sa_family = AF_UNIX; + } + +#endif + } + + return sa; +} + + +nxt_sockaddr_t * +nxt_sockaddr_copy(nxt_mem_pool_t *mp, nxt_sockaddr_t *src) +{ + size_t len; + nxt_sockaddr_t *dst; + + len = offsetof(nxt_sockaddr_t, u) + nxt_socklen(src); + + dst = nxt_mem_alloc(mp, len); + + if (nxt_fast_path(dst != NULL)) { + nxt_memcpy(dst, src, len); + } + + return dst; +} + + +nxt_sockaddr_t * +nxt_getsockname(nxt_mem_pool_t *mp, nxt_socket_t s) +{ + int ret; + socklen_t socklen; + nxt_sockaddr_buf_t sockaddr; + + socklen = NXT_SOCKADDR_LEN; + + ret = getsockname(s, &sockaddr.buf, &socklen); + + if (nxt_fast_path(ret == 0)) { + return nxt_sockaddr_create(mp, &sockaddr.buf, socklen); + } + + nxt_thread_log_error(NXT_LOG_ERR, "getsockname(%d) failed %E", + s, nxt_errno); + + return NULL; +} + + +nxt_int_t +nxt_sockaddr_text(nxt_mem_pool_t *mp, nxt_sockaddr_t *sa, nxt_bool_t port) +{ + size_t len; + u_char *p; + u_char buf[NXT_SOCKADDR_STR_LEN + NXT_SOCKPORT_STR_LEN]; + + len = NXT_SOCKADDR_STR_LEN + NXT_SOCKPORT_STR_LEN; + + len = nxt_sockaddr_ntop(sa, buf, buf + len, port); + + p = nxt_mem_alloc(mp, len); + + if (nxt_fast_path(p != NULL)) { + + sa->text = p; + sa->text_len = len; + nxt_memcpy(p, buf, len); + + return NXT_OK; + } + + return NXT_ERROR; +} + + +uint32_t +nxt_sockaddr_port(nxt_sockaddr_t *sa) +{ + uint32_t port; + + switch (sa->u.sockaddr.sa_family) { + +#if (NXT_INET6) + + case AF_INET6: + port = sa->u.sockaddr_in6.sin6_port; + break; + +#endif + +#if (NXT_HAVE_UNIX_DOMAIN) + + case AF_UNIX: + return 0; + +#endif + + default: + port = sa->u.sockaddr_in.sin_port; + break; + } + + return ntohs((uint16_t) port); +} + + +nxt_bool_t +nxt_sockaddr_cmp(nxt_sockaddr_t *sa1, nxt_sockaddr_t *sa2) +{ + if (nxt_socklen(sa1) != nxt_socklen(sa2)) { + return 0; + } + + if (sa1->type != sa2->type) { + return 0; + } + + if (sa1->u.sockaddr.sa_family != sa2->u.sockaddr.sa_family) { + return 0; + } + + /* + * sockaddr struct's cannot be compared in whole since kernel + * may fill some fields in inherited sockaddr struct's. + */ + + switch (sa1->u.sockaddr.sa_family) { + +#if (NXT_INET6) + + case AF_INET6: + if (sa1->u.sockaddr_in6.sin6_port != sa2->u.sockaddr_in6.sin6_port) { + return 0; + } + + if (nxt_memcmp(&sa1->u.sockaddr_in6.sin6_addr, + &sa2->u.sockaddr_in6.sin6_addr, 16) + != 0) + { + return 0; + } + + return 1; + +#endif + +#if (NXT_HAVE_UNIX_DOMAIN) + + case AF_UNIX: + { + size_t len; + + len = nxt_socklen(sa1) - offsetof(struct sockaddr_un, sun_path); + + if (nxt_memcmp(&sa1->u.sockaddr_un.sun_path, + &sa2->u.sockaddr_un.sun_path, len) + != 0) + { + return 0; + } + + return 1; + } + +#endif + + default: /* AF_INET */ + if (sa1->u.sockaddr_in.sin_port != sa2->u.sockaddr_in.sin_port) { + return 0; + } + + if (sa1->u.sockaddr_in.sin_addr.s_addr + != sa2->u.sockaddr_in.sin_addr.s_addr) + { + return 0; + } + + return 1; + } +} + + +size_t +nxt_sockaddr_ntop(nxt_sockaddr_t *sa, u_char *buf, u_char *end, nxt_bool_t port) +{ + u_char *p; + + switch (sa->u.sockaddr.sa_family) { + + case AF_INET: + p = (u_char *) &sa->u.sockaddr_in.sin_addr; + + if (port) { + p = nxt_sprintf(buf, end, "%ud.%ud.%ud.%ud:%d", + p[0], p[1], p[2], p[3], + ntohs(sa->u.sockaddr_in.sin_port)); + } else { + p = nxt_sprintf(buf, end, "%ud.%ud.%ud.%ud", + p[0], p[1], p[2], p[3]); + } + + return p - buf; + +#if (NXT_INET6) + + case AF_INET6: + p = buf; + + if (port) { + *p++ = '['; + } + + p = nxt_inet6_ntop(sa->u.sockaddr_in6.sin6_addr.s6_addr, p, end); + + if (port) { + p = nxt_sprintf(p, end, "]:%d", + ntohs(sa->u.sockaddr_in6.sin6_port)); + } + + return p - buf; +#endif + +#if (NXT_HAVE_UNIX_DOMAIN) + + case AF_UNIX: + +#if (NXT_LINUX) + + p = (u_char *) sa->u.sockaddr_un.sun_path; + + if (p[0] == '\0') { + int len; + + /* Linux abstract socket address has no trailing zero. */ + + len = nxt_socklen(sa) - offsetof(struct sockaddr_un, sun_path) - 1; + p = nxt_sprintf(buf, end, "unix:\\0%*s", len, p + 1); + + } else { + p = nxt_sprintf(buf, end, "unix:%s", p); + } + +#else /* !(NXT_LINUX) */ + + p = nxt_sprintf(buf, end, "unix:%s", sa->u.sockaddr_un.sun_path); + +#endif + + return p - buf; + +#endif /* NXT_HAVE_UNIX_DOMAIN */ + + default: + return 0; + } +} + + +#if (NXT_INET6) + +static u_char * +nxt_inet6_ntop(u_char *addr, u_char *buf, u_char *end) +{ + u_char *p; + size_t zero_groups, last_zero_groups, ipv6_bytes; + nxt_uint_t i, zero_start, last_zero_start; + + if (buf + NXT_INET6_ADDR_STR_LEN > end) { + return buf; + } + + zero_start = 8; + zero_groups = 0; + last_zero_start = 8; + last_zero_groups = 0; + + for (i = 0; i < 16; i += 2) { + + if (addr[i] == 0 && addr[i + 1] == 0) { + + if (last_zero_groups == 0) { + last_zero_start = i; + } + + last_zero_groups++; + + } else { + if (zero_groups < last_zero_groups) { + zero_groups = last_zero_groups; + zero_start = last_zero_start; + } + + last_zero_groups = 0; + } + } + + if (zero_groups < last_zero_groups) { + zero_groups = last_zero_groups; + zero_start = last_zero_start; + } + + ipv6_bytes = 16; + p = buf; + + if (zero_start == 0) { + + /* IPv4-mapped address */ + if ((zero_groups == 5 && addr[10] == 0xff && addr[11] == 0xff) + /* IPv4-compatible address */ + || (zero_groups == 6) + /* not IPv6 loopback address */ + || (zero_groups == 7 && addr[14] != 0 && addr[15] != 1)) + { + ipv6_bytes = 12; + } + + *p++ = ':'; + } + + for (i = 0; i < ipv6_bytes; i += 2) { + + if (i == zero_start) { + /* Output maximum number of consecutive zero groups as "::". */ + i += (zero_groups - 1) * 2; + *p++ = ':'; + continue; + } + + p = nxt_sprintf(p, end, "%uxd", (addr[i] << 8) + addr[i + 1]); + + if (i < 14) { + *p++ = ':'; + } + } + + if (ipv6_bytes == 12) { + p = nxt_sprintf(p, end, "%ud.%ud.%ud.%ud", + addr[12], addr[13], addr[14], addr[15]); + } + + return p; +} + +#endif + + +void +nxt_job_sockaddr_parse(nxt_job_sockaddr_parse_t *jbs) +{ + u_char *p; + size_t len; + nxt_int_t ret; + nxt_work_handler_t handler; + + nxt_job_set_name(&jbs->resolve.job, "job sockaddr parse"); + + len = jbs->addr.len; + p = jbs->addr.data; + + if (len > 6 && nxt_memcmp(p, (u_char *) "unix:", 5) == 0) { + ret = nxt_job_sockaddr_unix_parse(jbs); + + } else if (len != 0 && *p == '[') { + ret = nxt_job_sockaddr_inet6_parse(jbs); + + } else { + ret = nxt_job_sockaddr_inet_parse(jbs); + } + + switch (ret) { + + case NXT_OK: + handler = jbs->resolve.ready_handler; + break; + + case NXT_ERROR: + handler = jbs->resolve.error_handler; + break; + + default: /* NXT_AGAIN */ + return; + } + + nxt_job_return(nxt_thread(), &jbs->resolve.job, handler); +} + + +static nxt_int_t +nxt_job_sockaddr_unix_parse(nxt_job_sockaddr_parse_t *jbs) +{ +#if (NXT_HAVE_UNIX_DOMAIN) + size_t len, socklen; + u_char *path; + nxt_mem_pool_t *mp; + nxt_sockaddr_t *sa; + + /* + * Actual sockaddr_un length can be lesser or even larger than defined + * struct sockaddr_un length (see comment in unix/nxt_socket.h). So + * limit maximum Unix domain socket address length by defined sun_path[] + * length because some OSes accept addresses twice larger than defined + * struct sockaddr_un. Also reserve space for a trailing zero to avoid + * ambiguity, since many OSes accept Unix domain socket addresses + * without a trailing zero. + */ + const size_t max_len = sizeof(struct sockaddr_un) + - offsetof(struct sockaddr_un, sun_path) - 1; + + /* cutting "unix:" */ + len = jbs->addr.len - 5; + path = jbs->addr.data + 5; + + if (len > max_len) { + nxt_thread_log_error(jbs->resolve.log_level, + "unix domain socket \"%V\" name is too long", + &jbs->addr); + return NXT_ERROR; + } + + socklen = offsetof(struct sockaddr_un, sun_path) + len + 1; + +#if (NXT_LINUX) + + /* + * Linux unix(7): + * + * abstract: an abstract socket address is distinguished by the fact + * that sun_path[0] is a null byte ('\0'). The socket's address in + * this namespace is given by the additional bytes in sun_path that + * are covered by the specified length of the address structure. + * (Null bytes in the name have no special significance.) + */ + if (path[0] == '\0') { + socklen--; + } + +#endif + + mp = jbs->resolve.job.mem_pool; + + jbs->resolve.sockaddrs = nxt_mem_alloc(mp, sizeof(void *)); + + if (nxt_fast_path(jbs->resolve.sockaddrs != NULL)) { + sa = nxt_sockaddr_alloc(mp, socklen); + + if (nxt_fast_path(sa != NULL)) { + jbs->resolve.count = 1; + jbs->resolve.sockaddrs[0] = sa; + + sa->u.sockaddr_un.sun_family = AF_UNIX; + nxt_memcpy(sa->u.sockaddr_un.sun_path, path, len); + + return NXT_OK; + } + } + + return NXT_ERROR; + +#else /* !(NXT_HAVE_UNIX_DOMAIN) */ + + nxt_thread_log_error(jbs->resolve.log_level, + "unix domain socket \"%V\" is not supported", + &jbs->addr); + return NXT_ERROR; + +#endif +} + + +static nxt_int_t +nxt_job_sockaddr_inet6_parse(nxt_job_sockaddr_parse_t *jbs) +{ +#if (NXT_INET6) + u_char *p, *addr, *addr_end; + size_t len; + nxt_int_t port; + nxt_mem_pool_t *mp; + nxt_sockaddr_t *sa; + struct in6_addr *in6_addr; + + len = jbs->addr.len - 1; + addr = jbs->addr.data + 1; + + addr_end = nxt_memchr(addr, ']', len); + + if (addr_end == NULL) { + goto invalid_address; + } + + mp = jbs->resolve.job.mem_pool; + + jbs->resolve.sockaddrs = nxt_mem_alloc(mp, sizeof(void *)); + + if (nxt_slow_path(jbs->resolve.sockaddrs == NULL)) { + return NXT_ERROR; + } + + sa = nxt_sockaddr_alloc(mp, sizeof(struct sockaddr_in6)); + + if (nxt_slow_path(sa == NULL)) { + return NXT_ERROR; + } + + jbs->resolve.count = 1; + jbs->resolve.sockaddrs[0] = sa; + + in6_addr = &sa->u.sockaddr_in6.sin6_addr; + + if (nxt_inet6_addr(in6_addr, addr, addr_end - addr) != NXT_OK) { + goto invalid_address; + } + + p = addr_end + 1; + len = (addr + len) - p; + + if (len == 0) { + jbs->no_port = 1; + port = jbs->resolve.port; + goto found; + } + + if (*p == ':') { + port = nxt_int_parse(p + 1, len - 1); + + if (port >= 1 && port <= 65535) { + port = htons((in_port_t) port); + goto found; + } + } + + nxt_thread_log_error(jbs->resolve.log_level, + "invalid port in \"%V\"", &jbs->addr); + + return NXT_ERROR; + +found: + + sa->u.sockaddr_in6.sin6_family = AF_INET6; + sa->u.sockaddr_in6.sin6_port = (in_port_t) port; + + if (IN6_IS_ADDR_UNSPECIFIED(in6_addr)) { + jbs->wildcard = 1; + } + + return NXT_OK; + +invalid_address: + + nxt_thread_log_error(jbs->resolve.log_level, + "invalid IPv6 address in \"%V\"", &jbs->addr); + return NXT_ERROR; + +#else + + nxt_thread_log_error(jbs->resolve.log_level, + "IPv6 socket \"%V\" is not supported", &jbs->addr); + return NXT_ERROR; + +#endif +} + + +static nxt_int_t +nxt_job_sockaddr_inet_parse(nxt_job_sockaddr_parse_t *jbs) +{ + u_char *p, *host; + size_t len; + in_addr_t addr; + nxt_int_t port; + nxt_mem_pool_t *mp; + nxt_sockaddr_t *sa; + + addr = INADDR_ANY; + + len = jbs->addr.len; + host = jbs->addr.data; + + p = nxt_memchr(host, ':', len); + + if (p == NULL) { + + /* single value port, address, or host name */ + + port = nxt_int_parse(host, len); + + if (port > 0) { + if (port < 1 || port > 65535) { + goto invalid_port; + } + + /* "*:XX" */ + port = htons((in_port_t) port); + jbs->resolve.port = (in_port_t) port; + + } else { + jbs->no_port = 1; + + addr = nxt_inet_addr(host, len); + + if (addr == INADDR_NONE) { + jbs->resolve.name.len = len; + jbs->resolve.name.data = host; + + nxt_job_resolve(&jbs->resolve); + return NXT_AGAIN; + } + + /* "x.x.x.x" */ + port = jbs->resolve.port; + } + + } else { + + /* x.x.x.x:XX or host:XX */ + + p++; + len = (host + len) - p; + port = nxt_int_parse(p, len); + + if (port < 1 || port > 65535) { + goto invalid_port; + } + + port = htons((in_port_t) port); + + len = (p - 1) - host; + + if (len != 1 || host[0] != '*') { + addr = nxt_inet_addr(host, len); + + if (addr == INADDR_NONE) { + jbs->resolve.name.len = len; + jbs->resolve.name.data = host; + jbs->resolve.port = (in_port_t) port; + + nxt_job_resolve(&jbs->resolve); + return NXT_AGAIN; + } + + /* "x.x.x.x:XX" */ + } + } + + mp = jbs->resolve.job.mem_pool; + + jbs->resolve.sockaddrs = nxt_mem_alloc(mp, sizeof(void *)); + if (nxt_slow_path(jbs->resolve.sockaddrs == NULL)) { + return NXT_ERROR; + } + + sa = nxt_sockaddr_alloc(mp, sizeof(struct sockaddr_in)); + + if (nxt_fast_path(sa != NULL)) { + jbs->resolve.count = 1; + jbs->resolve.sockaddrs[0] = sa; + + jbs->wildcard = (addr == INADDR_ANY); + + sa->u.sockaddr_in.sin_family = AF_INET; + sa->u.sockaddr_in.sin_port = (in_port_t) port; + sa->u.sockaddr_in.sin_addr.s_addr = addr; + + return NXT_OK; + } + + return NXT_ERROR; + +invalid_port: + + nxt_thread_log_error(jbs->resolve.log_level, + "invalid port in \"%V\"", &jbs->addr); + + return NXT_ERROR; +} + + +in_addr_t +nxt_inet_addr(u_char *buf, size_t len) +{ + u_char c, *end; + in_addr_t addr; + nxt_uint_t digit, octet, dots; + + addr = 0; + octet = 0; + dots = 0; + + end = buf + len; + + while (buf < end) { + + c = *buf++; + + digit = c - '0'; + /* values below '0' become large unsigned integers */ + + if (digit < 10) { + octet = octet * 10 + digit; + continue; + } + + if (c == '.' && octet < 256) { + addr = (addr << 8) + octet; + octet = 0; + dots++; + continue; + } + + return INADDR_NONE; + } + + if (dots == 3 && octet < 256) { + addr = (addr << 8) + octet; + return htonl(addr); + } + + return INADDR_NONE; +} + + +#if (NXT_INET6) + +nxt_int_t +nxt_inet6_addr(struct in6_addr *in6_addr, u_char *buf, size_t len) +{ + u_char c, *addr, *zero_start, *ipv4, *dst, *src, *end; + nxt_uint_t digit, group, nibbles, groups_left; + + if (len == 0) { + return NXT_ERROR; + } + + end = buf + len; + + if (buf[0] == ':') { + buf++; + } + + addr = in6_addr->s6_addr; + zero_start = NULL; + groups_left = 8; + nibbles = 0; + group = 0; + ipv4 = NULL; + + while (buf < end) { + c = *buf++; + + if (c == ':') { + if (nibbles != 0) { + ipv4 = buf; + + *addr++ = (u_char) (group >> 8); + *addr++ = (u_char) (group & 0xff); + groups_left--; + + if (groups_left != 0) { + nibbles = 0; + group = 0; + continue; + } + + } else { + if (zero_start == NULL) { + ipv4 = buf; + zero_start = addr; + continue; + } + } + + return NXT_ERROR; + } + + if (c == '.' && nibbles != 0) { + + if (groups_left < 2 || ipv4 == NULL) { + return NXT_ERROR; + } + + group = nxt_inet_addr(ipv4, end - ipv4); + if (group == INADDR_NONE) { + return NXT_ERROR; + } + + group = ntohl(group); + + *addr++ = (u_char) ((group >> 24) & 0xff); + *addr++ = (u_char) ((group >> 16) & 0xff); + groups_left--; + + /* the low 16-bit are copied below */ + break; + } + + nibbles++; + + if (nibbles > 4) { + return NXT_ERROR; + } + + group <<= 4; + + digit = c - '0'; + /* values below '0' become large unsigned integers */ + + if (digit < 10) { + group += digit; + continue; + } + + c |= 0x20; + digit = c - 'a'; + /* values below 'a' become large unsigned integers */ + + if (digit < 6) { + group += 10 + digit; + continue; + } + + return NXT_ERROR; + } + + if (nibbles == 0 && zero_start == NULL) { + return NXT_ERROR; + } + + *addr++ = (u_char) (group >> 8); + *addr++ = (u_char) (group & 0xff); + groups_left--; + + if (groups_left != 0) { + + if (zero_start != NULL) { + + /* moving part before consecutive zero groups to the end */ + + groups_left *= 2; + src = addr - 1; + dst = src + groups_left; + + while (src >= zero_start) { + *dst-- = *src--; + } + + nxt_memzero(zero_start, groups_left); + + return NXT_OK; + } + + } else { + if (zero_start == NULL) { + return NXT_OK; + } + } + + return NXT_ERROR; +} + +#endif diff --git a/src/nxt_sockaddr.h b/src/nxt_sockaddr.h new file mode 100644 index 00000000..382e8fcb --- /dev/null +++ b/src/nxt_sockaddr.h @@ -0,0 +1,167 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_SOCKADDR_H_INCLUDED_ +#define _NXT_SOCKADDR_H_INCLUDED_ + + +/* + * The nxt_sockaddr_t should be allocated using nxt_sockaddr_alloc() + * with actual "struct sockaddr_..." size: + * nxt_sockaddr_alloc(pool, sizeof(struct sockaddr_in)) + */ + +struct nxt_sockaddr_s { + /* + * A sockaddr textual representation is optional and may be in two forms: + * with port or without port. If a nxt_sockaddr_t is intended to listen(), + * bind() or connect() then the textual representation must be present and + * must include the port. nxt_event_conn_accept() creates a textual + * representation without the port. + */ + u_char *text; + + /* + * text_len, socket type and socklen are stored + * together on 64-bit platforms without sockaddr.sa_len. + */ + uint16_t text_len; + uint16_t type; +#if !(NXT_SOCKADDR_SA_LEN) + socklen_t _socklen; +#endif + + union { + struct sockaddr sockaddr; + struct sockaddr_in sockaddr_in; +#if (NXT_INET6) + struct sockaddr_in6 sockaddr_in6; +#endif +#if (NXT_HAVE_UNIX_DOMAIN) + struct sockaddr_un sockaddr_un; +#endif + } u; +}; + + +typedef struct { + nxt_job_resolve_t resolve; + nxt_str_t addr; + + uint8_t wildcard; /* 1 bit */ + uint8_t no_port; /* 1 bit */ +} nxt_job_sockaddr_parse_t; + + +NXT_EXPORT nxt_sockaddr_t *nxt_sockaddr_alloc(nxt_mem_pool_t *mp, socklen_t len) + NXT_MALLOC_LIKE; +NXT_EXPORT nxt_sockaddr_t *nxt_sockaddr_create(nxt_mem_pool_t *mp, + struct sockaddr *sockaddr, socklen_t len) + NXT_MALLOC_LIKE; +NXT_EXPORT nxt_sockaddr_t *nxt_sockaddr_copy(nxt_mem_pool_t *mp, + nxt_sockaddr_t *src) + NXT_MALLOC_LIKE; +NXT_EXPORT nxt_sockaddr_t *nxt_getsockname(nxt_mem_pool_t *mp, nxt_socket_t s) + NXT_MALLOC_LIKE; +NXT_EXPORT nxt_int_t nxt_sockaddr_text(nxt_mem_pool_t *mp, nxt_sockaddr_t *sa, + nxt_bool_t port); + + +#if (NXT_SOCKADDR_SA_LEN) + +#define \ +nxt_socklen_set(sa, len) \ + (sa)->u.sockaddr.sa_len = (socklen_t) (len) + + +#define \ +nxt_socklen(sa) \ + ((sa)->u.sockaddr.sa_len) + +#else + +#define \ +nxt_socklen_set(sa, len) \ + (sa)->_socklen = (socklen_t) (len) + + +#define \ +nxt_socklen(sa) \ + ((sa)->_socklen) + +#endif + + +NXT_EXPORT uint32_t nxt_sockaddr_port(nxt_sockaddr_t *sa); +NXT_EXPORT nxt_bool_t nxt_sockaddr_cmp(nxt_sockaddr_t *sa1, + nxt_sockaddr_t *sa2); +NXT_EXPORT size_t nxt_sockaddr_ntop(nxt_sockaddr_t *sa, u_char *buf, + u_char *end, + nxt_bool_t port); +NXT_EXPORT void nxt_job_sockaddr_parse(nxt_job_sockaddr_parse_t *jbs); +NXT_EXPORT in_addr_t nxt_inet_addr(u_char *buf, size_t len); +#if (NXT_INET6) +NXT_EXPORT nxt_int_t nxt_inet6_addr(struct in6_addr *in6_addr, u_char *buf, + size_t len); +#endif + +#if (NXT_HAVE_UNIX_DOMAIN) +#define nxt_unix_addr_path_len(sa) \ + (nxt_socklen(sa) - offsetof(struct sockaddr_un, sun_path)) +#endif + + +#define NXT_INET_ADDR_STR_LEN (sizeof("255.255.255.255") - 1) + +#define NXT_INET6_ADDR_STR_LEN \ + (sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255") - 1) + +#define NXT_UNIX_ADDR_STR_LEN \ + ((sizeof("unix:") - 1) \ + + (sizeof(struct sockaddr_un) - offsetof(struct sockaddr_un, sun_path))) + + +#if (NXT_HAVE_UNIX_DOMAIN) +#define NXT_SOCKADDR_STR_LEN NXT_UNIX_ADDR_STR_LEN + +#elif (NXT_INET6) +#define NXT_SOCKADDR_STR_LEN NXT_INET6_ADDR_STR_LEN + +#else +#define NXT_SOCKADDR_STR_LEN NXT_INET_ADDR_STR_LEN +#endif + + +#if (NXT_INET6) +#define NXT_SOCKPORT_STR_LEN (sizeof("[]:65535") - 1) + +#else +#define NXT_SOCKPORT_STR_LEN (sizeof(":65535") - 1) +#endif + + +nxt_inline size_t +nxt_sockaddr_text_len(nxt_sockaddr_t *sa) +{ + switch (sa->u.sockaddr.sa_family) { + +#if (NXT_INET6) + case AF_INET6: + return NXT_INET6_ADDR_STR_LEN; +#endif + +#if (NXT_HAVE_UNIX_DOMAIN) + case AF_UNIX: + return NXT_UNIX_ADDR_STR_LEN; +#endif + + default: + return NXT_INET_ADDR_STR_LEN; + } +} + + +#endif /* _NXT_SOCKADDR_H_INCLUDED_ */ diff --git a/src/nxt_socket.c b/src/nxt_socket.c new file mode 100644 index 00000000..19ba21c0 --- /dev/null +++ b/src/nxt_socket.c @@ -0,0 +1,317 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static const char *nxt_socket_sockopt_name(nxt_uint_t level, + nxt_uint_t sockopt); + + +nxt_socket_t +nxt_socket_create(nxt_uint_t domain, nxt_uint_t type, nxt_uint_t protocol, + nxt_uint_t flags) +{ + nxt_socket_t s; + +#if (NXT_HAVE_SOCK_NONBLOCK) + + if (flags & NXT_NONBLOCK) { + type |= SOCK_NONBLOCK; + } + +#endif + + s = socket(domain, type, protocol); + + if (nxt_slow_path(s == -1)) { + nxt_thread_log_alert("socket(%ui, 0x%uXi, %ui) failed %E", + domain, type, protocol, nxt_socket_errno); + return s; + } + + nxt_thread_log_debug("socket(): %d", s); + +#if !(NXT_HAVE_SOCK_NONBLOCK) + + if (flags & NXT_NONBLOCK) { + if (nxt_slow_path(nxt_socket_nonblocking(s) != NXT_OK)) { + nxt_socket_close(s); + return -1; + } + } + +#endif + + return s; +} + + +void +nxt_socket_close(nxt_socket_t s) +{ + if (nxt_fast_path(close(s) == 0)) { + nxt_thread_log_debug("socket close(%d)", s); + + } else { + nxt_thread_log_alert("socket close(%d) failed %E", s, nxt_socket_errno); + } +} + + +nxt_int_t +nxt_socket_getsockopt(nxt_socket_t s, nxt_uint_t level, nxt_uint_t sockopt) +{ + int val; + socklen_t len; + + len = sizeof(val); + + if (nxt_fast_path(getsockopt(s, level, sockopt, &val, &len) == 0)) { + nxt_thread_log_debug("getsockopt(%d, %ui, %s): %d", + s, level, + nxt_socket_sockopt_name(level, sockopt), val); + return val; + } + + nxt_thread_log_error(NXT_LOG_CRIT, "getsockopt(%d, %ui, %s) failed %E", + s, level, nxt_socket_sockopt_name(level, sockopt), + val, nxt_socket_errno); + + return -1; +} + + +nxt_int_t +nxt_socket_setsockopt(nxt_socket_t s, nxt_uint_t level, nxt_uint_t sockopt, + int val) +{ + socklen_t len; + + len = sizeof(val); + + if (nxt_fast_path(setsockopt(s, level, sockopt, &val, len) == 0)) { + nxt_thread_log_debug("setsockopt(%d, %ui, %s): %d", + s, level, + nxt_socket_sockopt_name(level, sockopt), val); + return NXT_OK; + } + + nxt_thread_log_error(NXT_LOG_CRIT, "setsockopt(%d, %ui, %s, %d) failed %E", + s, level, nxt_socket_sockopt_name(level, sockopt), + val, nxt_socket_errno); + + return NXT_ERROR; +} + + +static const char * +nxt_socket_sockopt_name(nxt_uint_t level, nxt_uint_t sockopt) +{ + switch (level) { + + case SOL_SOCKET: + switch (sockopt) { + + case SO_SNDBUF: + return "SO_SNDBUF"; + + case SO_RCVBUF: + return "SO_RCVBUF"; + + case SO_REUSEADDR: + return "SO_TYPE"; + + case SO_TYPE: + return "SO_TYPE"; + } + + break; + + case IPPROTO_TCP: + switch (sockopt) { + + case TCP_NODELAY: + return "TCP_NODELAY"; + +#ifdef TCP_DEFER_ACCEPT + case TCP_DEFER_ACCEPT: + return "TCP_DEFER_ACCEPT"; +#endif + } + + break; + +#if (NXT_INET6 && defined IPV6_V6ONLY) + case IPPROTO_IPV6: + + switch (sockopt) { + + case IPV6_V6ONLY: + return "IPV6_V6ONLY"; + } + + break; +#endif + + } + + return ""; +} + + +nxt_int_t +nxt_socket_bind(nxt_socket_t s, nxt_sockaddr_t *sa, nxt_bool_t test) +{ + nxt_err_t err; + + nxt_thread_log_debug("bind(%d, %*s)", s, sa->text_len, sa->text); + + if (nxt_fast_path(bind(s, &sa->u.sockaddr, nxt_socklen(sa)) == 0)) { + return NXT_OK; + } + + err = nxt_socket_errno; + + if (err == NXT_EADDRINUSE && test) { + return NXT_DECLINED; + } + + nxt_thread_log_error(NXT_LOG_CRIT, "bind(%d, %*s) failed %E", + s, sa->text_len, sa->text, err); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_socket_connect(nxt_socket_t s, nxt_sockaddr_t *sa) +{ + nxt_err_t err; + nxt_int_t ret; + nxt_uint_t level; + + nxt_thread_log_debug("connect(%d, %*s)", s, sa->text_len, sa->text); + + if (connect(s, &sa->u.sockaddr, nxt_socklen(sa)) == 0) { + return NXT_OK; + } + + err = nxt_socket_errno; + + switch (err) { + + case NXT_EINPROGRESS: + nxt_thread_log_debug("connect(%d) in progress", s); + return NXT_AGAIN; + + case NXT_ECONNREFUSED: +#if (NXT_LINUX) + case NXT_EAGAIN: + /* + * Linux returns EAGAIN instead of ECONNREFUSED + * for UNIX sockets if a listen queue is full. + */ +#endif + level = NXT_LOG_ERR; + ret = NXT_DECLINED; + break; + + case NXT_ECONNRESET: + case NXT_ENETDOWN: + case NXT_ENETUNREACH: + case NXT_EHOSTDOWN: + case NXT_EHOSTUNREACH: + level = NXT_LOG_ERR; + ret = NXT_ERROR; + break; + + default: + level = NXT_LOG_CRIT; + ret = NXT_ERROR; + } + + nxt_thread_log_error(level, "connect(%d, %*s) failed %E", + s, sa->text_len, sa->text, err); + + return ret; +} + + +void +nxt_socket_shutdown(nxt_socket_t s, nxt_uint_t how) +{ + nxt_err_t err; + nxt_uint_t level; + + if (nxt_fast_path(shutdown(s, how) == 0)) { + nxt_thread_log_debug("shutdown(%d, %ui)", s, how); + return; + } + + err = nxt_socket_errno; + + switch (err) { + + case NXT_ENOTCONN: + level = NXT_LOG_INFO; + break; + + case NXT_ECONNRESET: + case NXT_ENETDOWN: + case NXT_ENETUNREACH: + case NXT_EHOSTDOWN: + case NXT_EHOSTUNREACH: + level = NXT_LOG_ERR; + break; + + default: + level = NXT_LOG_CRIT; + } + + nxt_thread_log_error(level, "shutdown(%d, %ui) failed %E", s, how, err); +} + + +nxt_uint_t +nxt_socket_error_level(nxt_err_t err, nxt_socket_error_level_t level) +{ + switch (err) { + + case NXT_ECONNRESET: + + if ((level & NXT_SOCKET_ECONNRESET_IGNORE) != 0) { + return NXT_LOG_DEBUG; + } + + return NXT_LOG_ERR; + + case NXT_EINVAL: + + if ((level & NXT_SOCKET_EINVAL_IGNORE) != 0) { + return NXT_LOG_DEBUG; + } + + return NXT_LOG_ALERT; + + case NXT_EPIPE: + case NXT_ENOTCONN: + case NXT_ETIMEDOUT: + case NXT_ENETDOWN: + case NXT_ENETUNREACH: + case NXT_EHOSTDOWN: + case NXT_EHOSTUNREACH: + + if ((level & NXT_SOCKET_ERROR_IGNORE) != 0) { + return NXT_LOG_INFO; + } + + return NXT_LOG_ERR; + + default: + return NXT_LOG_ALERT; + } +} diff --git a/src/nxt_socket.h b/src/nxt_socket.h new file mode 100644 index 00000000..8912fb23 --- /dev/null +++ b/src/nxt_socket.h @@ -0,0 +1,132 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_SOCKET_H_INCLUDED_ +#define _NXT_UNIX_SOCKET_H_INCLUDED_ + + +typedef int nxt_socket_t; + +#define NXT_NONBLOCK 1 + + +/* + * struct sockaddr and struct sockaddr_in are 16 bytes. + * + * struct sockaddr_in6 is: + * 28 bytes on Linux, FreeBSD, MacOSX, NetBSD, OpenBSD, AIX, HP-UX; + * 32 bytes on Solaris. + * + * + * struct sockaddr_un is: + * 94 bytes on HP-UX; + * 106 bytes on FreeBSD, MacOSX, NetBSD, OpenBSD; + * 110 bytes on Linux, Solaris; + * 1025 bytes on AIX. + * + * The real maximum sockaddr_un length however different from defined length: + * OpenBSD can accept and return 105 bytes if address is not + * zero-terminated; + * Linux can accept 110 bytes and return 111 bytes; + * MacOSX and NetBSD can accept and return 255 bytes; + * Solaris can accept 257 bytes and return 258 bytes; + * FreeBSD maximum sockaddr_un length is equal to defined length. + * + * POSIX.1g renamed AF_UNIX to AF_LOCAL, however, Solaris up to 10 + * version lacks AF_LOCAL. AF_UNIX is defined even on Windows although + * struct sockaddr_un is not. + * + * Unix domain socket address without a trailing zero is accepted at least by: + * Linux, FreeBSD, Solaris, MacOSX, NetBSD, and OpenBSD. + * Linux and Solaris add the trailing zero and return sockaddr_un length + * increased by one. Others return sockaddr_un without the trailing zero. + * + * For unspecified Unix domain socket address + * NetBSD returns sockaddr_un length equal to 106 and fills sun_path[] + * with zeros; + * FreeBSD, Solaris, MacOSX, and OpenBSD return sockaddr_un length + * equal to 16 and fill sun_path[] with zeros; + * Linux returns sockaddr_un length equal to 2 without sun_path[]; + * + * 4.4BSD getsockname() and getpeername() returned zero length. + * This behaviour has been inherited by BSD flavours and has been + * eventually changed in NetBSD 1.2, FreeBSD 3.0, and OpenBSD 5.3. + * + * + * struct sockaddr_storage is: + * 128 bytes on Linux, FreeBSD, MacOSX, NetBSD; + * 256 bytes on Solaris, OpenBSD, and HP-UX; + * 1288 bytes on AIX. + * + * struct sockaddr_storage is too large on some platforms + * or less than real maximum struct sockaddr_un length. + */ + +#if (NXT_HAVE_UNIX_DOMAIN) +#define NXT_SOCKADDR_LEN sizeof(struct sockaddr_un) + +#elif (NXT_HAVE_SOCKADDR_IN6) +#define NXT_SOCKADDR_LEN sizeof(struct sockaddr_in6) + +#else +#define NXT_SOCKADDR_LEN sizeof(struct sockaddr_in) +#endif + + +typedef union { + struct sockaddr buf; + uint64_t alignment; + char space[NXT_SOCKADDR_LEN]; +} nxt_sockaddr_buf_t; + + +/* + * MAXHOSTNAMELEN is: + * 64 on Linux; + * 256 on FreeBSD, Solaris, MacOSX, NetBSD, OpenBSD. + */ +#define NXT_MAXHOSTNAMELEN MAXHOSTNAMELEN + + +typedef enum { + NXT_SOCKET_ERROR_IGNORE = 0x1, + NXT_SOCKET_ECONNRESET_IGNORE = 0x2, + NXT_SOCKET_EINVAL_IGNORE = 0x4, +} nxt_socket_error_level_t; + + +NXT_EXPORT nxt_socket_t nxt_socket_create(nxt_uint_t family, nxt_uint_t type, + nxt_uint_t protocol, nxt_uint_t flags); +NXT_EXPORT void nxt_socket_close(nxt_socket_t s); +NXT_EXPORT nxt_int_t nxt_socket_getsockopt(nxt_socket_t s, nxt_uint_t level, + nxt_uint_t sockopt); +NXT_EXPORT nxt_int_t nxt_socket_setsockopt(nxt_socket_t s, nxt_uint_t level, + nxt_uint_t sockopt, int val); +NXT_EXPORT nxt_int_t nxt_socket_bind(nxt_socket_t s, nxt_sockaddr_t *sa, + nxt_bool_t test); +NXT_EXPORT nxt_int_t nxt_socket_connect(nxt_socket_t s, nxt_sockaddr_t *sa); +NXT_EXPORT void nxt_socket_shutdown(nxt_socket_t s, nxt_uint_t how); +nxt_uint_t nxt_socket_error_level(nxt_err_t err, + nxt_socket_error_level_t level); + +NXT_EXPORT nxt_int_t nxt_socketpair_create(nxt_socket_t *pair); +NXT_EXPORT void nxt_socketpair_close(nxt_socket_t *pair); +NXT_EXPORT ssize_t nxt_socketpair_send(nxt_event_fd_t *ev, nxt_fd_t fd, + nxt_iobuf_t *iob, nxt_uint_t niob); +NXT_EXPORT ssize_t nxt_socketpair_recv(nxt_event_fd_t *ev, nxt_fd_t *fd, + nxt_iobuf_t *iob, nxt_uint_t niob); + + +#define \ +nxt_socket_nonblocking(fd) \ + nxt_fd_nonblocking(fd) + +#define \ +nxt_socket_blocking(fd) \ + nxt_fd_blocking(fd) + + +#endif /* _NXT_UNIX_SOCKET_H_INCLUDED_ */ diff --git a/src/nxt_socketpair.c b/src/nxt_socketpair.c new file mode 100644 index 00000000..20336d38 --- /dev/null +++ b/src/nxt_socketpair.c @@ -0,0 +1,291 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * SOCK_SEQPACKET protocol is supported for AF_UNIX in Solaris 8 X/Open + * sockets, Linux 2.6.4, FreeBSD 9.0, NetBSD 6.0, and OpenBSD 5.0. + */ + +/* SOCK_SEQPACKET is disabled to test SOCK_DGRAM on all platforms. */ +#if (0 || NXT_HAVE_AF_UNIX_SOCK_SEQPACKET) +#define NXT_UNIX_SOCKET SOCK_SEQPACKET +#else +#define NXT_UNIX_SOCKET SOCK_DGRAM +#endif + + +static ssize_t nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, + nxt_uint_t niob); +static ssize_t nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, + nxt_uint_t niob); + + +nxt_int_t +nxt_socketpair_create(nxt_socket_t *pair) +{ + if (nxt_slow_path(socketpair(AF_UNIX, NXT_UNIX_SOCKET, 0, pair) != 0)) { + nxt_thread_log_alert("socketpair() failed %E", nxt_errno); + return NXT_ERROR; + } + + nxt_thread_log_debug("socketpair(): %d:%d", pair[0], pair[1]); + + if (nxt_slow_path(nxt_socket_nonblocking(pair[0]) != NXT_OK)) { + goto fail; + } + + if (nxt_slow_path(fcntl(pair[0], F_SETFD, FD_CLOEXEC) == -1)) { + goto fail; + } + + if (nxt_slow_path(nxt_socket_nonblocking(pair[1]) != NXT_OK)) { + goto fail; + } + + if (nxt_slow_path(fcntl(pair[1], F_SETFD, FD_CLOEXEC) == -1)) { + goto fail; + } + + return NXT_OK; + +fail: + + nxt_socketpair_close(pair); + + return NXT_ERROR; +} + + +void +nxt_socketpair_close(nxt_socket_t *pair) +{ + nxt_socket_close(pair[0]); + nxt_socket_close(pair[1]); +} + + +ssize_t +nxt_socketpair_send(nxt_event_fd_t *ev, nxt_fd_t fd, nxt_iobuf_t *iob, + nxt_uint_t niob) +{ + ssize_t n; + nxt_err_t err; + + for ( ;; ) { + n = nxt_sendmsg(ev->fd, fd, iob, niob); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(ev->log, "sendmsg(%d, %FD, %ui): %z", + ev->fd, fd, niob, n); + + if (n > 0) { + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(ev->log, "sendmsg(%d) not ready", ev->fd); + ev->write_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(ev->log, "sendmsg(%d) interrupted", ev->fd); + continue; + + default: + nxt_log_error(NXT_LOG_CRIT, ev->log, + "sendmsg(%d, %FD, %ui) failed %E", + ev->fd, fd, niob, err); + return NXT_ERROR; + } + } +} + + +ssize_t +nxt_socketpair_recv(nxt_event_fd_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, + nxt_uint_t niob) +{ + ssize_t n; + nxt_err_t err; + + for ( ;; ) { + n = nxt_recvmsg(ev->fd, fd, iob, niob); + + err = (n == -1) ? nxt_socket_errno : 0; + + nxt_log_debug(ev->log, "recvmsg(%d, %FD, %ui): %z", + ev->fd, *fd, niob, n); + + if (n > 0) { + return n; + } + + if (n == 0) { + ev->closed = 1; + ev->read_ready = 0; + return n; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + nxt_log_debug(ev->log, "recvmsg(%d) not ready", ev->fd); + ev->read_ready = 0; + return NXT_AGAIN; + + case NXT_EINTR: + nxt_log_debug(ev->log, "recvmsg(%d) interrupted", ev->fd); + continue; + + default: + nxt_log_error(NXT_LOG_CRIT, ev->log, + "recvmsg(%d, %p, %ui) failed %E", + ev->fd, fd, niob, err); + return NXT_ERROR; + } + } +} + + +#if (NXT_HAVE_MSGHDR_MSG_CONTROL) + +/* + * Linux, FreeBSD, Solaris X/Open sockets, + * MacOSX, NetBSD, AIX, HP-UX X/Open sockets. + */ + +static ssize_t +nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, nxt_uint_t niob) +{ + struct msghdr msg; + union { + struct cmsghdr cm; + char space[CMSG_SPACE(sizeof(int))]; + } cmsg; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = iob; + msg.msg_iovlen = niob; + /* Flags are cleared just to suppress valgrind warning. */ + msg.msg_flags = 0; + + if (fd != -1) { + msg.msg_control = (caddr_t) &cmsg; + msg.msg_controllen = sizeof(cmsg); + + cmsg.cm.cmsg_len = CMSG_LEN(sizeof(int)); + cmsg.cm.cmsg_level = SOL_SOCKET; + cmsg.cm.cmsg_type = SCM_RIGHTS; + + /* + * nxt_memcpy() is used instead of simple + * *(int *) CMSG_DATA(&cmsg.cm) = fd; + * because GCC 4.4 with -O2/3/s optimization may issue a warning: + * dereferencing type-punned pointer will break strict-aliasing rules + * + * Fortunately, GCC with -O1 compiles this nxt_memcpy() + * in the same simple assignment as in the code above. + */ + nxt_memcpy(CMSG_DATA(&cmsg.cm), &fd, sizeof(int)); + + } else { + msg.msg_control = NULL; + msg.msg_controllen = 0; + } + + return sendmsg(s, &msg, 0); +} + + +static ssize_t +nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) +{ + ssize_t n; + struct msghdr msg; + union { + struct cmsghdr cm; + char space[CMSG_SPACE(sizeof(int))]; + } cmsg; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = iob; + msg.msg_iovlen = niob; + msg.msg_control = (caddr_t) &cmsg; + msg.msg_controllen = sizeof(cmsg); + + *fd = -1; + + n = recvmsg(s, &msg, 0); + + if (n > 0 + && cmsg.cm.cmsg_len == CMSG_LEN(sizeof(int)) + && cmsg.cm.cmsg_level == SOL_SOCKET + && cmsg.cm.cmsg_type == SCM_RIGHTS) + { + /* (*fd) = *(int *) CMSG_DATA(&cmsg.cm); */ + nxt_memcpy(fd, CMSG_DATA(&cmsg.cm), sizeof(int)); + } + + return n; +} + +#else + +/* Solaris 4.3BSD sockets. */ + +static ssize_t +nxt_sendmsg(nxt_socket_t s, nxt_fd_t fd, nxt_iobuf_t *iob, nxt_uint_t niob) +{ + struct msghdr msg; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = iob; + msg.msg_iovlen = niob; + + if (fd != -1) { + msg.msg_accrights = (caddr_t) &fd; + msg.msg_accrightslen = sizeof(int); + + } else { + msg.msg_accrights = NULL; + msg.msg_accrightslen = 0; + } + + return sendmsg(s, &msg, 0); +} + + +static ssize_t +nxt_recvmsg(nxt_socket_t s, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) +{ + struct msghdr msg; + + *fd = -1; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = iob; + msg.msg_iovlen = niob; + msg.msg_accrights = (caddr_t) fd; + msg.msg_accrightslen = sizeof(int); + + return recvmsg(s, &msg, 0); +} + +#endif diff --git a/src/nxt_solaris_sendfilev.c b/src/nxt_solaris_sendfilev.c new file mode 100644 index 00000000..fdd803dc --- /dev/null +++ b/src/nxt_solaris_sendfilev.c @@ -0,0 +1,170 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * sendfilev() has been introduced in Solaris 8 (7/01). + * According to sendfilev(3EXT) it can write to: + * + * a file descriptor to a regular file or to a AF_NCA, AF_INET, or + * AF_INET6 family type SOCK_STREAM socket that is open for writing. + */ + +ssize_t nxt_solaris_event_conn_io_sendfilev(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit); +static size_t nxt_solaris_buf_coalesce(nxt_buf_t *b, sendfilevec_t *sfv, + int32_t *nsfv, nxt_bool_t *sync, size_t limit); + + +ssize_t +nxt_solaris_event_conn_io_sendfilev(nxt_event_conn_t *c, nxt_buf_t *b, + size_t limit) +{ + size_t sent; + ssize_t n; + int32_t nsfv; + nxt_err_t err; + nxt_off_t size; + nxt_bool_t sync; + sendfilevec_t sfv[NXT_IOBUF_MAX]; + + if (c->sendfile == 0) { + /* AF_UNIX does not support sendfilev(). */ + return nxt_event_conn_io_sendbuf(c, b, limit); + } + + sync = 0; + + size = nxt_solaris_buf_coalesce(b, sfv, &nsfv, &sync, limit); + + nxt_log_debug(c->socket.log, "sendfilev(%d, %D)", c->socket.fd, nsfv); + + if (nsfv == 0 && sync) { + return 0; + } + + sent = 0; + n = sendfilev(c->socket.fd, sfv, nsfv, &sent); + + err = (n == -1) ? nxt_errno : 0; + + nxt_log_debug(c->socket.log, "sendfilev(): %d sent:%uz", n, sent); + + if (n == -1) { + switch (err) { + + case NXT_EAGAIN: + c->socket.write_ready = 0; + break; + + case NXT_EINTR: + break; + + default: + c->socket.error = err; + nxt_log_error(nxt_socket_error_level(err, c->socket.log_error), + c->socket.log, "sendfilev(%d, %D) failed %E", + c->socket.fd, nsfv, err); + + return NXT_ERROR; + } + + nxt_log_debug(c->socket.log, "sendfilev() %E", err); + + return sent; + } + + if ((nxt_off_t) sent < size) { + c->socket.write_ready = 0; + } + + return sent; +} + + +static size_t +nxt_solaris_buf_coalesce(nxt_buf_t *b, sendfilevec_t *sfv, int32_t *nsfv, + nxt_bool_t *sync, size_t limit) +{ + size_t size, total; + nxt_fd_t fd, last_fd; + nxt_int_t i; + nxt_off_t pos, last_pos; + + i = -1; + last_fd = -1; + last_pos = 0; + total = 0; + + for (total = 0; b != NULL && total < limit; b = b->next) { + + if (nxt_buf_is_file(b)) { + + fd = b->file->fd; + pos = b->file_pos; + size = b->file_end - pos; + + if (size == 0) { + continue; + } + + if (total + size > limit) { + size = limit - total; + } + + } else if (nxt_buf_is_mem(b)) { + + fd = SFV_FD_SELF; + pos = (uintptr_t) b->mem.pos; + size = b->mem.free - b->mem.pos; + + if (size == 0) { + continue; + } + + if (total + size > limit) { + size = limit - total; + } + + } else { + *sync = 1; + continue; + } + + if (size == 0) { + break; + } + + if (fd != last_fd || pos != last_pos) { + + if (++i >= NXT_IOBUF_MAX) { + goto done; + } + + sfv[i].sfv_fd = fd; + sfv[i].sfv_flag = 0; + sfv[i].sfv_off = pos; + sfv[i].sfv_len = size; + + } else { + sfv[i].sfv_len += size; + } + + total += size; + last_pos = pos + size; + last_fd = fd; + } + + i++; + +done: + + *nsfv = i; + + return total; +} diff --git a/src/nxt_sort.h b/src/nxt_sort.h new file mode 100644 index 00000000..b79d0919 --- /dev/null +++ b/src/nxt_sort.h @@ -0,0 +1,14 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_SORT_H_INCLUDED_ +#define _NXT_SORT_H_INCLUDED_ + + +#define nxt_qsort qsort + + +#endif /* _NXT_SORT_H_INCLUDED_ */ diff --git a/src/nxt_source.h b/src/nxt_source.h new file mode 100644 index 00000000..23bebb3c --- /dev/null +++ b/src/nxt_source.h @@ -0,0 +1,43 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_SOURCE_H_INCLUDED_ +#define _NXT_SOURCE_H_INCLUDED_ + + +/* + * A source handler should store a pointer to a passed source hook, but not + * the hook's values themselves, because a source filter may change the values. + */ +typedef struct { + void *context; + nxt_work_handler_t filter; +} nxt_source_hook_t; + + +typedef void (*nxt_source_handler_t)(void *source_context, + nxt_source_hook_t *query); + + +#define \ +nxt_source_filter(thr, wq, next, out) \ + do { \ + if (thr->engine->batch != 0) { \ + nxt_thread_work_queue_add(thr, wq, nxt_source_filter_handler, \ + next, out, thr->log); \ + \ + } else { \ + (next)->filter(thr, (next)->context, out); \ + } \ + \ + } while (0) + + +NXT_EXPORT void nxt_source_filter_handler(nxt_thread_t *thr, void *obj, + void *data); + + +#endif /* _NXT_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_spinlock.c b/src/nxt_spinlock.c new file mode 100644 index 00000000..0348ae23 --- /dev/null +++ b/src/nxt_spinlock.c @@ -0,0 +1,152 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * Linux supports pthread spinlocks since glibc 2.3. Spinlock is an + * atomic integer with zero initial value. On i386/amd64 however the + * initial value is one. Spinlock never yields control. + * + * FreeBSD 5.2 and Solaris 10 support pthread spinlocks. Spinlock is a + * structure and uses mutex implementation so it must be initialized by + * by pthread_spin_init() and destroyed by pthread_spin_destroy(). + */ + +#if (NXT_HAVE_MACOSX_SPINLOCK) + +/* + * OSSpinLockLock() tries to acquire a lock atomically. If the lock is + * busy, on SMP system it tests the lock 1000 times in a tight loop with + * "pause" instruction. If the lock has been released, OSSpinLockLock() + * tries to acquire it again. On failure it goes again in the tight loop. + * If the lock has not been released during spinning in the loop or + * on UP system, OSSpinLockLock() calls thread_switch() to run 1ms + * with depressed (the lowest) priority. + */ + +void +nxt_thread_spin_lock(nxt_thread_spinlock_t *lock) +{ + nxt_thread_log_debug("OSSpinLockLock(%p) enter", lock); + + OSSpinLockLock(lock); +} + + +nxt_bool_t +nxt_thread_spin_trylock(nxt_thread_spinlock_t *lock) +{ + nxt_thread_log_debug("OSSpinLockTry(%p) enter", lock); + + if (OSSpinLockTry(lock)) { + return 1; + } + + nxt_thread_log_debug("OSSpinLockTry(%p) failed", lock); + + return 0; +} + + +void +nxt_thread_spin_unlock(nxt_thread_spinlock_t *lock) +{ + OSSpinLockUnlock(lock); + + nxt_thread_log_debug("OSSpinLockUnlock(%p) exit", lock); +} + + +#else + + +/* It should be adjusted with the "spinlock_count" directive. */ +static nxt_uint_t nxt_spinlock_count = 1000; + + +void +nxt_thread_spin_init(nxt_uint_t ncpu, nxt_uint_t count) +{ + switch (ncpu) { + + case 0: + /* Explicit spinlock count. */ + nxt_spinlock_count = count; + break; + + case 1: + /* Spinning is useless on UP. */ + nxt_spinlock_count = 0; + break; + + default: + /* + * SMP. + * + * TODO: The count should be 10 on a virtualized system + * since virtualized CPUs may share the same physical CPU. + */ + nxt_spinlock_count = 1000; + break; + } +} + + +void +nxt_thread_spin_lock(nxt_thread_spinlock_t *lock) +{ + nxt_uint_t n; + + nxt_thread_log_debug("spin_lock(%p) enter", lock); + + for ( ;; ) { + + again: + + if (nxt_fast_path(nxt_atomic_try_lock(lock))) { + return; + } + + for (n = nxt_spinlock_count; n != 0; n--) { + + nxt_cpu_pause(); + + if (*lock == 0) { + goto again; + } + } + + nxt_thread_yield(); + } +} + + +nxt_bool_t +nxt_thread_spin_trylock(nxt_thread_spinlock_t *lock) +{ + nxt_thread_log_debug("spin_trylock(%p) enter", lock); + + if (nxt_fast_path(nxt_atomic_try_lock(lock))) { + return 1; + } + + nxt_thread_log_debug("spin_trylock(%p) failed", lock); + + return 0; +} + + +void +nxt_thread_spin_unlock(nxt_thread_spinlock_t *lock) +{ + nxt_atomic_release(lock); + + nxt_thread_log_debug("spin_unlock(%p) exit", lock); +} + +#endif diff --git a/src/nxt_spinlock.h b/src/nxt_spinlock.h new file mode 100644 index 00000000..bdc3a5f6 --- /dev/null +++ b/src/nxt_spinlock.h @@ -0,0 +1,57 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_SPINLOCK_H_INCLUDED_ +#define _NXT_UNIX_SPINLOCK_H_INCLUDED_ + + +#if (NXT_THREADS) + +#if (NXT_HAVE_MACOSX_SPINLOCK) + +#include <libkern/OSAtomic.h> + +typedef OSSpinLock nxt_thread_spinlock_t; + +#define \ +nxt_thread_spin_init(ncpu, count) + +#else + +typedef nxt_atomic_t nxt_thread_spinlock_t; + +NXT_EXPORT void nxt_thread_spin_init(nxt_uint_t ncpu, nxt_uint_t count); + +#endif + + +NXT_EXPORT void nxt_thread_spin_lock(nxt_thread_spinlock_t *lock); +NXT_EXPORT nxt_bool_t nxt_thread_spin_trylock(nxt_thread_spinlock_t *lock); +NXT_EXPORT void nxt_thread_spin_unlock(nxt_thread_spinlock_t *lock); + + +#else /* !(NXT_THREADS) */ + + +typedef nxt_atomic_t nxt_thread_spinlock_t; + +#define \ +nxt_thread_spin_init(ncpu, count) + +#define \ +nxt_thread_spin_lock(lock) + +#define \ +nxt_thread_spin_trylock(lock) \ + 1 + +#define \ +nxt_thread_spin_unlock(lock) + +#endif + + +#endif /* _NXT_UNIX_SPINLOCK_H_INCLUDED_ */ diff --git a/src/nxt_sprintf.c b/src/nxt_sprintf.c new file mode 100644 index 00000000..837d2635 --- /dev/null +++ b/src/nxt_sprintf.c @@ -0,0 +1,716 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <math.h> +#include <float.h> + + +/* + * Supported formats: + * + * %[0][width][x][X]O nxt_off_t + * %[0][width]T nxt_time_t + * %[0][width][u][x|X]z ssize_t/size_t + * %[0][width][u][x|X]d int/u_int + * %[0][width][u][x|X]l long + * %[0][width|m][u][x|X]i nxt_int_t/nxt_uint_t + * %[0][width][u][x|X]D int32_t/uint32_t + * %[0][width][u][x|X]L int64_t/uint64_t + * %[0][width|m][u][x|X]A nxt_atomic_int_t/nxt_atomic_uint_t + * %[0][width][.width]f double, max valid number fits to %18.15f + * + * %FD nxt_fd_t, int / HANDLE + * %d nxt_socket_t, int + * + * %PI nxt_pid_t, process id + * %PT nxt_tid_t, thread id + * %PF nxt_fid_t, fiber id + * %PH pthread_t handle returned by pthread_self() + * + * %s null-terminated string + * %*s length and string + * %FN nxt_file_name_t * + * + * %M nxt_msec_t + * %N nxt_nsec_t + * %r rlim_t + * %p void * + * %b nxt_bool_t + * %E nxt_err_t + * %V nxt_str_t * + * %Z '\0' + * %n '\n' + * %c char + * %% % + * + * Reserved: + * %t ptrdiff_t + * %S null-terminated wchar string + * %C wchar + * %[0][width][u][x|X]Q int182_t/uint128_t + */ + + +u_char * nxt_cdecl +nxt_sprintf(u_char *buf, u_char *end, const char *fmt, ...) +{ + u_char *p; + va_list args; + + va_start(args, fmt); + p = nxt_vsprintf(buf, end, fmt, args); + va_end(args); + + return p; +} + + +/* + * nxt_sprintf_t is used: + * to pass several parameters of nxt_integer() via single pointer + * and to store little used variables of nxt_vsprintf(). + */ + +typedef struct { + u_char *end; + const u_char *hex; + uint32_t width; + int32_t frac_width; + uint8_t max_width; + u_char padding; +} nxt_sprintf_t; + + +static u_char *nxt_integer(nxt_sprintf_t *spf, u_char *buf, uint64_t ui64); +static u_char *nxt_number(nxt_sprintf_t *spf, u_char *buf, double n); + + +/* A right way of "f == 0.0". */ +#define \ +nxt_double_is_zero(f) \ + (fabs(f) <= FLT_EPSILON) + + +u_char * +nxt_vsprintf(u_char *buf, u_char *end, const char *fmt, va_list args) +{ + u_char *p; + int d; + double f, i; + size_t len; + int64_t i64; + uint64_t ui64, frac; + nxt_str_t *v; + nxt_err_t err; + nxt_uint_t scale, n; + nxt_msec_t ms; + nxt_nsec_t ns; + nxt_bool_t sign; + nxt_sprintf_t spf; + nxt_file_name_t *fn; + + static const u_char hexadecimal[16] = "0123456789abcdef"; + static const u_char HEXADECIMAL[16] = "0123456789ABCDEF"; + static const u_char nan[] = "[nan]"; + static const u_char infinity[] = "[infinity]"; + + spf.end = end; + + while (*fmt != '\0' && buf < end) { + + /* + * "buf < end" means that we could copy at least one character: + * a plain character, "%%", "%c", or a minus without test. + */ + + if (*fmt != '%') { + *buf++ = *fmt++; + continue; + } + + fmt++; + + /* Test some often used text formats first. */ + + switch (*fmt) { + + case 'V': + fmt++; + v = va_arg(args, nxt_str_t *); + + if (nxt_fast_path(v != NULL)) { + len = v->len; + p = v->data; + goto copy; + } + + continue; + + case 's': + p = va_arg(args, u_char *); + + if (nxt_fast_path(p != NULL)) { + while (*p != '\0' && buf < end) { + *buf++ = *p++; + } + } + + fmt++; + continue; + + case '*': + len = va_arg(args, u_int); + + fmt++; + + if (*fmt == 's') { + fmt++; + p = va_arg(args, u_char *); + + if (nxt_fast_path(p != NULL)) { + goto copy; + } + } + + continue; + + default: + break; + } + + spf.hex = NULL; + spf.width = 0; + spf.frac_width = -1; + spf.max_width = 0; + spf.padding = (*fmt == '0') ? '0' : ' '; + + sign = 1; + + i64 = 0; + ui64 = 0; + + while (*fmt >= '0' && *fmt <= '9') { + spf.width = spf.width * 10 + (*fmt++ - '0'); + } + + + for ( ;; ) { + switch (*fmt) { + + case 'u': + sign = 0; + fmt++; + continue; + + case 'm': + spf.max_width = 1; + fmt++; + continue; + + case 'X': + spf.hex = HEXADECIMAL; + sign = 0; + fmt++; + continue; + + case 'x': + spf.hex = hexadecimal; + sign = 0; + fmt++; + continue; + + case '.': + fmt++; + spf.frac_width = 0; + + while (*fmt >= '0' && *fmt <= '9') { + spf.frac_width = spf.frac_width * 10 + *fmt++ - '0'; + } + + break; + + default: + break; + } + + break; + } + + + switch (*fmt) { + + case 'E': + err = va_arg(args, nxt_err_t); + + *buf++ = '('; + spf.hex = NULL; + spf.width = 0; + buf = nxt_integer(&spf, buf, err); + + if (buf < end - 1) { + *buf++ = ':'; + *buf++ = ' '; + } + + buf = nxt_strerror(err, buf, end - buf); + + if (buf < end) { + *buf++ = ')'; + } + + fmt++; + continue; + + case 'O': + i64 = (int64_t) va_arg(args, nxt_off_t); + sign = 1; + goto number; + + case 'T': + i64 = (int64_t) va_arg(args, nxt_time_t); + sign = 1; + goto number; + + case 'M': + ms = (nxt_msec_t) va_arg(args, nxt_msec_t); + if ((nxt_msec_int_t) ms == -1 && spf.hex == NULL) { + i64 = -1; + sign = 1; + } else { + ui64 = (uint64_t) ms; + sign = 0; + } + goto number; + + case 'N': + ns = (nxt_nsec_t) va_arg(args, nxt_nsec_t); + if ((nxt_nsec_int_t) ns == -1) { + i64 = -1; + sign = 1; + } else { + ui64 = (uint64_t) ns; + sign = 0; + } + goto number; + + case 'z': + if (sign) { + i64 = (int64_t) va_arg(args, ssize_t); + } else { + ui64 = (uint64_t) va_arg(args, size_t); + } + goto number; + + case 'i': + if (sign) { + i64 = (int64_t) va_arg(args, nxt_int_t); + } else { + ui64 = (uint64_t) va_arg(args, nxt_uint_t); + } + + if (spf.max_width != 0) { + spf.width = NXT_INT_T_LEN; + } + + goto number; + + case 'd': + if (sign) { + i64 = (int64_t) va_arg(args, int); + } else { + ui64 = (uint64_t) va_arg(args, u_int); + } + goto number; + + case 'l': + if (sign) { + i64 = (int64_t) va_arg(args, long); + } else { + ui64 = (uint64_t) va_arg(args, u_long); + } + goto number; + + case 'D': + if (sign) { + i64 = (int64_t) va_arg(args, int32_t); + } else { + ui64 = (uint64_t) va_arg(args, uint32_t); + } + goto number; + + case 'L': + if (sign) { + i64 = va_arg(args, int64_t); + } else { + ui64 = va_arg(args, uint64_t); + } + goto number; + + case 'A': + if (sign) { + i64 = (int64_t) va_arg(args, nxt_atomic_int_t); + } else { + ui64 = (uint64_t) va_arg(args, nxt_atomic_uint_t); + } + + if (spf.max_width != 0) { + spf.width = NXT_ATOMIC_T_LEN; + } + + goto number; + + case 'b': + ui64 = (uint64_t) va_arg(args, nxt_bool_t); + sign = 0; + goto number; + + case 'f': + fmt++; + + f = va_arg(args, double); + + if (f < 0) { + *buf++ = '-'; + f = -f; + } + + if (nxt_slow_path(isnan(f))) { + p = (u_char *) nan; + len = sizeof(nan) - 1; + + goto copy; + + } else if (nxt_slow_path(isinf(f))) { + p = (u_char *) infinity; + len = sizeof(infinity) - 1; + + goto copy; + } + + (void) modf(f, &i); + frac = 0; + + if (spf.frac_width > 0) { + + scale = 1; + for (n = spf.frac_width; n != 0; n--) { + scale *= 10; + } + + frac = (uint64_t) ((f - i) * scale + 0.5); + + if (frac == scale) { + i += 1; + frac = 0; + } + } + + buf = nxt_number(&spf, buf, i); + + if (spf.frac_width > 0) { + + if (buf < end) { + *buf++ = '.'; + + spf.hex = NULL; + spf.padding = '0'; + spf.width = spf.frac_width; + buf = nxt_integer(&spf, buf, frac); + } + + } else if (spf.frac_width < 0) { + f = modf(f, &i); + + if (!nxt_double_is_zero(f) && buf < end) { + *buf++ = '.'; + + while (!nxt_double_is_zero(f) && buf < end) { + f *= 10; + f = modf(f, &i); + *buf++ = (u_char) i + '0'; + } + } + } + + continue; + + case 'r': + i64 = (int64_t) va_arg(args, rlim_t); + sign = 1; + break; + + case 'p': + ui64 = (uintptr_t) va_arg(args, void *); + sign = 0; + spf.hex = HEXADECIMAL; + /* + * spf.width = NXT_PTR_SIZE * 2; + * spf.padding = '0'; + */ + goto number; + + case 'c': + d = va_arg(args, int); + *buf++ = (u_char) (d & 0xff); + fmt++; + + continue; + + case 'F': + fmt++; + + switch (*fmt) { + + case 'D': + i64 = (int64_t) va_arg(args, nxt_fd_t); + sign = 1; + + goto number; + + case 'N': + fn = va_arg(args, nxt_file_name_t *); + p = fn; + + while (*p != '\0' && buf < end) { + *buf++ = *p++; + } + + fmt++; + continue; + + default: + continue; + } + + case 'P': + fmt++; + + switch (*fmt) { + + case 'I': + i64 = (int64_t) va_arg(args, nxt_pid_t); + sign = 1; + goto number; + + case 'T': + ui64 = (uint64_t) va_arg(args, nxt_tid_t); + sign = 0; + goto number; + + case 'F': + ui64 = (uint64_t) va_arg(args, nxt_fid_t); + sign = 0; + goto number; + + case 'H': + ui64 = (uint64_t) (uintptr_t) va_arg(args, pthread_t); + spf.hex = HEXADECIMAL; + sign = 0; + goto number; + + default: + continue; + } + + case 'Z': + *buf++ = '\0'; + fmt++; + continue; + + case 'n': + *buf++ = NXT_LF; + fmt++; + continue; + + case '%': + *buf++ = '%'; + fmt++; + continue; + + default: + *buf++ = *fmt++; + continue; + } + + number: + + if (sign) { + if (i64 < 0) { + *buf++ = '-'; + ui64 = (uint64_t) -i64; + + } else { + ui64 = (uint64_t) i64; + } + } + + buf = nxt_integer(&spf, buf, ui64); + + fmt++; + continue; + + copy: + + buf = nxt_cpymem(buf, p, nxt_min((size_t) (end - buf), len)); + continue; + } + + return buf; +} + + +static u_char * +nxt_integer(nxt_sprintf_t *spf, u_char *buf, uint64_t ui64) +{ + u_char *p, *end; + size_t len; + u_char temp[NXT_INT64_T_LEN]; + + p = temp + NXT_INT64_T_LEN; + + if (spf->hex == NULL) { + +#if (NXT_32BIT) + + for ( ;; ) { + u_char *start; + uint32_t ui32; + + /* + * 32-bit platforms usually lack hardware support of 64-bit + * division and remainder operations. For this reason C compiler + * adds calls to the runtime library functions which provides + * these operations. These functions usually have about hundred + * lines of code. + * + * For 32-bit numbers and some constant divisors GCC, Clang and + * other compilers can use inlined multiplications and shifts + * which are faster than division or remainder operations. + * For example, unsigned "ui32 / 10" is compiled to + * + * ((uint64_t) ui32 * 0xCCCCCCCD) >> 35 + * + * So a 64-bit number is split to parts by 10^9. The parts fit + * to 32 bits and are processed separately as 32-bit numbers. A + * number of 64-bit division/remainder operations is significantly + * decreased depending on the 64-bit number's value, it is + * 0 if the 64-bit value is less than 4294967296, + * 1 if the 64-bit value is greater than 4294967295 + * and less than 4294967296000000000, + * 2 otherwise. + */ + + if (ui64 <= 0xffffffff) { + ui32 = (uint32_t) ui64; + start = NULL; + + } else { + ui32 = (uint32_t) (ui64 % 1000000000); + start = p - 9; + } + + do { + *(--p) = (u_char) (ui32 % 10 + '0'); + ui32 /= 10; + } while (ui32 != 0); + + if (start == NULL) { + break; + } + + /* Add leading zeros of part. */ + + while (p > start) { + *(--p) = '0'; + } + + ui64 /= 1000000000; + } + +#else /* NXT_64BIT */ + + do { + *(--p) = (u_char) (ui64 % 10 + '0'); + ui64 /= 10; + } while (ui64 != 0); + +#endif + + } else { + + do { + *(--p) = spf->hex[ui64 & 0xf]; + ui64 >>= 4; + } while (ui64 != 0); + } + + /* Zero or space padding. */ + + if (spf->width != 0) { + + len = (temp + NXT_INT64_T_LEN) - p; + end = buf + (spf->width - len); + end = nxt_min(end, spf->end); + + while (buf < end) { + *buf++ = spf->padding; + } + } + + /* Number copying. */ + + len = (temp + NXT_INT64_T_LEN) - p; + end = buf + len; + end = nxt_min(end, spf->end); + + while (buf < end) { + *buf++ = *p++; + } + + return buf; +} + + +static u_char * +nxt_number(nxt_sprintf_t *spf, u_char *buf, double n) +{ + u_char *p, *end; + size_t len; + u_char temp[NXT_DOUBLE_LEN]; + + p = temp + NXT_DOUBLE_LEN; + + do { + *(--p) = (u_char) (fmod(n, 10) + '0'); + n = trunc(n / 10); + } while (!nxt_double_is_zero(n)); + + /* Zero or space padding. */ + + if (spf->width != 0) { + len = (temp + NXT_DOUBLE_LEN) - p; + end = buf + (spf->width - len); + end = nxt_min(end, spf->end); + + while (buf < end) { + *buf++ = spf->padding; + } + } + + /* Number copying. */ + + len = (temp + NXT_DOUBLE_LEN) - p; + + end = buf + len; + end = nxt_min(end, spf->end); + + while (buf < end) { + *buf++ = *p++; + } + + return buf; +} diff --git a/src/nxt_sprintf.h b/src/nxt_sprintf.h new file mode 100644 index 00000000..0a927f71 --- /dev/null +++ b/src/nxt_sprintf.h @@ -0,0 +1,21 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_SPRINTF_H_INCLUDED_ +#define _NXT_SPRINTF_H_INCLUDED_ + + +/* STUB */ +#define NXT_DOUBLE_LEN (1 + DBL_MAX_10_EXP) + + +NXT_EXPORT u_char *nxt_cdecl nxt_sprintf(u_char *buf, u_char *end, + const char *fmt, ...); +NXT_EXPORT u_char *nxt_vsprintf(u_char *buf, u_char *end, + const char *fmt, va_list args); + + +#endif /* _NXT_SPRINTF_H_INCLUDED_ */ diff --git a/src/nxt_ssltls.c b/src/nxt_ssltls.c new file mode 100644 index 00000000..3899a3ce --- /dev/null +++ b/src/nxt_ssltls.c @@ -0,0 +1,7 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> diff --git a/src/nxt_ssltls.h b/src/nxt_ssltls.h new file mode 100644 index 00000000..7bdc1946 --- /dev/null +++ b/src/nxt_ssltls.h @@ -0,0 +1,70 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_SSLTLS_H_INCLUDED_ +#define _NXT_SSLTLS_H_INCLUDED_ + + +/* + * The SSL/TLS libraries lack vector I/O interface yet add noticeable + * overhead to each SSL/TLS record so buffering allows to decrease the + * overhead. The typical overhead size is about 30 bytes, however, TLS + * supports also random padding up to 255 bytes. The maximum SSLv3/TLS + * record size is 16K. However, large records increase decryption latency. + * 4K is good compromise between 1-6% of SSL/TLS overhead and the latency. + * 4K buffer allows to send one SSL/TLS record (4096-bytes data and up to + * 224-bytes overhead) in three 1440-bytes TCP/IPv4 packets with timestamps + * and compatible with tunnels. + */ + +#define NXT_SSLTLS_BUFFER_SIZE 4096 + + +typedef struct nxt_ssltls_conf_s nxt_ssltls_conf_t; + + +typedef struct { + nxt_int_t (*server_init)(nxt_ssltls_conf_t *conf); + nxt_int_t (*set_versions)(nxt_ssltls_conf_t *conf); +} nxt_ssltls_lib_t; + + +struct nxt_ssltls_conf_s { + void *ctx; + void (*conn_init)(nxt_thread_t *thr, + nxt_ssltls_conf_t *conf, + nxt_event_conn_t *c); + + const nxt_ssltls_lib_t *lib; + + char *certificate; + char *certificate_key; + char *ciphers; + + char *ca_certificate; + + size_t buffer_size; +}; + + +#if (NXT_HAVE_OPENSSL) +extern const nxt_ssltls_lib_t nxt_openssl_lib; +#endif + +#if (NXT_HAVE_GNUTLS) +extern const nxt_ssltls_lib_t nxt_gnutls_lib; +#endif + +#if (NXT_HAVE_CYASSL) +extern const nxt_ssltls_lib_t nxt_cyassl_lib; +#endif + +#if (NXT_HAVE_POLARSSL) +extern const nxt_ssltls_lib_t nxt_polar_lib; +#endif + + +#endif /* _NXT_SSLTLS_H_INCLUDED_ */ diff --git a/src/nxt_stream_source.c b/src/nxt_stream_source.c new file mode 100644 index 00000000..33159ee9 --- /dev/null +++ b/src/nxt_stream_source.c @@ -0,0 +1,476 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void nxt_stream_source_connected(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_stream_source_write_ready(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_stream_source_read_ready(nxt_thread_t *thr, void *obj, + void *data); +static nxt_buf_t *nxt_stream_source_process_buffers(nxt_stream_source_t *stream, + nxt_event_conn_t *c); +static void nxt_stream_source_buf_completion(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_stream_source_read_done(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_stream_source_refused(nxt_thread_t *thr, void *obj, void *data); +static void nxt_stream_source_closed(nxt_thread_t *thr, void *obj, void *data); +static void nxt_stream_source_error(nxt_thread_t *thr, void *obj, void *data); +static void nxt_stream_source_close(nxt_thread_t *thr, + nxt_stream_source_t *stream); + + +static const nxt_event_conn_state_t nxt_stream_source_connect_state; +static const nxt_event_conn_state_t nxt_stream_source_request_write_state; +static const nxt_event_conn_state_t nxt_stream_source_response_ready_state; +static const nxt_event_conn_state_t nxt_stream_source_response_read_state; + + +void +nxt_stream_source_connect(nxt_stream_source_t *stream) +{ + nxt_thread_t *thr; + nxt_event_conn_t *c; + nxt_upstream_source_t *us; + + thr = nxt_thread(); + + us = stream->upstream; + + if (nxt_slow_path(!nxt_buf_pool_obtainable(&us->buffers))) { + nxt_thread_log_error(NXT_LOG_ERR, "%d buffers %uDK each " + "are not enough to read upstream response", + us->buffers.max, us->buffers.size / 1024); + goto fail; + } + + c = nxt_event_conn_create(us->buffers.mem_pool, thr->log); + if (nxt_slow_path(c == NULL)) { + goto fail; + } + + stream->conn = c; + c->socket.data = stream; + + nxt_event_conn_work_queue_set(c, us->work_queue); + + c->remote = us->peer->sockaddr; + c->write_state = &nxt_stream_source_connect_state; + + nxt_event_conn_connect(thr, c); + return; + +fail: + + stream->error_handler(stream); +} + + +static const nxt_event_conn_state_t nxt_stream_source_connect_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_stream_source_connected, + nxt_stream_source_refused, + nxt_stream_source_error, + + NULL, /* timeout */ + NULL, /* timeout value */ + 0, /* connect_timeout */ +}; + + +static void +nxt_stream_source_connected(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + nxt_stream_source_t *stream; + + c = obj; + stream = data; + + nxt_log_debug(thr->log, "stream source connected fd:%d", c->socket.fd); + + c->read_state = &nxt_stream_source_response_ready_state; + c->write = stream->out; + c->write_state = &nxt_stream_source_request_write_state; + + if (thr->engine->batch != 0) { + nxt_event_conn_write(thr, c); + + } else { + stream->read_queued = 1; + nxt_thread_work_queue_add(thr, &thr->engine->read_work_queue, + c->io->read, c, stream, thr->log); + + c->io->write(thr, c, stream); + } +} + + +static const nxt_event_conn_state_t nxt_stream_source_request_write_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_stream_source_write_ready, + NULL, + nxt_stream_source_error, + + NULL, /* timeout */ + NULL, /* timeout value */ + 0, /* connect_timeout */ +}; + + +static const nxt_event_conn_state_t nxt_stream_source_response_ready_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_stream_source_read_ready, + nxt_stream_source_closed, + nxt_stream_source_error, + + NULL, /* timeout */ + NULL, /* timeout value */ + 0, /* connect_timeout */ +}; + + +static void +nxt_stream_source_write_ready(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "stream source write ready fd:%d", c->socket.fd); + + nxt_event_conn_read(thr, c); +} + + +static void +nxt_stream_source_read_ready(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_int_t ret; + nxt_buf_t *b; + nxt_buf_pool_t *buffers; + nxt_event_conn_t *c; + nxt_stream_source_t *stream; + + c = obj; + stream = data; + stream->read_queued = 0; + + nxt_log_debug(thr->log, "stream source read ready fd:%d", c->socket.fd); + + if (c->read == NULL) { + + buffers = &stream->upstream->buffers; + + ret = nxt_buf_pool_mem_alloc(buffers, 0); + + if (nxt_slow_path(ret != NXT_OK)) { + + if (nxt_slow_path(ret == NXT_ERROR)) { + goto fail; + } + + /* ret == NXT_AGAIN */ + + nxt_log_debug(thr->log, "stream source flush"); + + b = nxt_buf_sync_alloc(buffers->mem_pool, NXT_BUF_SYNC_NOBUF); + + if (nxt_slow_path(b == NULL)) { + goto fail; + } + + nxt_event_fd_block_read(thr->engine, &c->socket); + + nxt_source_filter(thr, c->write_work_queue, stream->next, b); + return; + } + + c->read = buffers->current; + buffers->current = NULL; + } + + c->read_state = &nxt_stream_source_response_read_state; + + nxt_event_conn_read(thr, c); + return; + +fail: + + nxt_stream_source_close(thr, stream); +} + + +static const nxt_event_conn_state_t nxt_stream_source_response_read_state + nxt_aligned(64) = +{ + NXT_EVENT_NO_BUF_PROCESS, + NXT_EVENT_TIMER_AUTORESET, + + nxt_stream_source_read_done, + nxt_stream_source_closed, + nxt_stream_source_error, + + NULL, /* timeout */ + NULL, /* timeout value */ + 0, /* connect_timeout */ +}; + + +static void +nxt_stream_source_read_done(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_bool_t batch; + nxt_event_conn_t *c; + nxt_stream_source_t *stream; + + c = obj; + stream = data; + + nxt_log_debug(thr->log, "stream source read done fd:%d", c->socket.fd); + + if (c->read != NULL) { + b = nxt_stream_source_process_buffers(stream, c); + + if (nxt_slow_path(b == NULL)) { + nxt_stream_source_close(thr, stream); + return; + } + + batch = (thr->engine->batch != 0); + + if (batch) { + nxt_thread_work_queue_add(thr, stream->upstream->work_queue, + nxt_source_filter_handler, + stream->next, b, thr->log); + } + + if (!stream->read_queued) { + stream->read_queued = 1; + nxt_thread_work_queue_add(thr, stream->upstream->work_queue, + nxt_stream_source_read_ready, + c, stream, thr->log); + } + + if (!batch) { + stream->next->filter(thr, stream->next->context, b); + } + } +} + + +static nxt_buf_t * +nxt_stream_source_process_buffers(nxt_stream_source_t *stream, + nxt_event_conn_t *c) +{ + size_t size, nbytes; + nxt_buf_t *b, *in, *head, **prev; + + nbytes = c->nbytes; + prev = &head; + + do { + b = nxt_buf_mem_alloc(stream->upstream->buffers.mem_pool, 0, 0); + + if (nxt_slow_path(b == NULL)) { + return NULL; + } + + *prev = b; + + b->data = stream; + b->completion_handler = nxt_stream_source_buf_completion; + + in = c->read; + in->retain++; + b->parent = in; + + b->mem.pos = in->mem.free; + b->mem.start = in->mem.free; + + size = nxt_buf_mem_free_size(&in->mem); + + if (nbytes < size) { + in->mem.free += nbytes; + + b->mem.free = in->mem.free; + b->mem.end = in->mem.free; + + break; + } + + in->mem.free = in->mem.end; + + b->mem.free = in->mem.free; + b->mem.end = in->mem.free; + nbytes -= size; + + prev = &b->next; + c->read = in->next; + in->next = NULL; + + } while (c->read != NULL); + + return head; +} + + +static void +nxt_stream_source_buf_completion(nxt_thread_t *thr, void *obj, void *data) +{ + size_t size; + nxt_buf_t *b, *parent; + nxt_stream_source_t *stream; + + b = obj; + parent = data; + +#if 0 + nxt_log_debug(thr->log, + "stream source buf completion: %p parent:%p retain:%uD", + b, parent, parent->retain); +#endif + + stream = b->data; + + /* A parent is a buffer where stream reads data. */ + + parent->mem.pos = b->mem.pos; + parent->retain--; + + if (parent->retain == 0 && !stream->conn->socket.closed) { + size = nxt_buf_mem_size(&parent->mem); + + parent->mem.pos = parent->mem.start; + parent->mem.free = parent->mem.start; + + /* + * A buffer's original size can be changed by filters + * so reuse the buffer only if it is still large enough. + */ + if (size >= 256 || size >= stream->upstream->buffers.size) { + + if (stream->conn->read != parent) { + nxt_buf_chain_add(&stream->conn->read, parent); + } + + if (!stream->read_queued) { + stream->read_queued = 1; + nxt_thread_work_queue_add(thr, stream->upstream->work_queue, + nxt_stream_source_read_ready, + stream->conn, + stream->conn->socket.data, + stream->conn->socket.log); + } + } + } + + nxt_buf_free(stream->upstream->buffers.mem_pool, b); +} + + +static void +nxt_stream_source_refused(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_stream_source_t *stream; + + stream = data; + +#if (NXT_DEBUG) + { + nxt_event_conn_t *c; + + c = obj; + + nxt_log_debug(thr->log, "stream source refused fd:%d", c->socket.fd); + } +#endif + + nxt_stream_source_close(thr, stream); +} + + +static void +nxt_stream_source_closed(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_event_conn_t *c; + nxt_stream_source_t *stream; + + c = obj; + stream = data; + + nxt_log_debug(thr->log, "stream source closed fd:%d", c->socket.fd); + + nxt_event_conn_close(thr, c); + + b = nxt_buf_sync_alloc(stream->upstream->buffers.mem_pool, + NXT_BUF_SYNC_LAST); + + if (nxt_slow_path(b == NULL)) { + stream->error_handler(stream); + return; + } + + nxt_source_filter(thr, c->write_work_queue, stream->next, b); +} + + +static void +nxt_stream_source_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_stream_source_t *stream; + + stream = data; + +#if (NXT_DEBUG) + { + nxt_event_fd_t *ev; + + ev = obj; + + nxt_log_debug(thr->log, "stream source error fd:%d", ev->fd); + } +#endif + + nxt_stream_source_close(thr, stream); +} + + +static void +nxt_stream_source_close(nxt_thread_t *thr, nxt_stream_source_t *stream) +{ + nxt_event_conn_close(thr, stream->conn); + + stream->error_handler(stream); +} + + +void +nxt_source_filter_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_source_hook_t *next; + + next = obj; + + next->filter(thr, next->context, data); +} diff --git a/src/nxt_stream_source.h b/src/nxt_stream_source.h new file mode 100644 index 00000000..ae8d5b71 --- /dev/null +++ b/src/nxt_stream_source.h @@ -0,0 +1,31 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_STREAM_SOURCE_H_INCLUDED_ +#define _NXT_STREAM_SOURCE_H_INCLUDED_ + + +typedef struct nxt_stream_source_s nxt_stream_source_t; + +typedef void (*nxt_stream_source_handler_t)(nxt_stream_source_t *u); + +struct nxt_stream_source_s { + nxt_event_conn_t *conn; + nxt_source_hook_t *next; + nxt_upstream_source_t *upstream; + + nxt_buf_t *out; + + uint32_t read_queued; /* 1 bit */ + + nxt_stream_source_handler_t error_handler; +}; + + +void nxt_stream_source_connect(nxt_stream_source_t *stream); + + +#endif /* _NXT_STREAM_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_string.c b/src/nxt_string.c new file mode 100644 index 00000000..45301e94 --- /dev/null +++ b/src/nxt_string.c @@ -0,0 +1,317 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_str_t * +nxt_str_alloc(nxt_mem_pool_t *mp, size_t len) +{ + nxt_str_t *s; + + /* The string data is allocated aligned to be close to nxt_str_t. */ + s = nxt_mem_alloc(mp, sizeof(nxt_str_t) + len); + + if (nxt_fast_path(s != NULL)) { + s->len = len; + s->data = (u_char *) s + sizeof(nxt_str_t); + } + + return s; +} + + +/* + * nxt_str_dup() creates a new string with a copy of a source string. + * If length of the source string is zero, then the new string anyway + * gets a pointer somewhere in mem_pool. + */ + +nxt_str_t * +nxt_str_dup(nxt_mem_pool_t *mp, nxt_str_t *dst, const nxt_str_t *src) +{ + u_char *p; + + if (dst == NULL) { + /* The string data is allocated aligned to be close to nxt_str_t. */ + dst = nxt_mem_alloc(mp, sizeof(nxt_str_t) + src->len); + if (nxt_slow_path(dst == NULL)) { + return NULL; + } + + p = (u_char *) dst; + p += sizeof(nxt_str_t); + dst->data = p; + + } else { + dst->data = nxt_mem_nalloc(mp, src->len); + if (nxt_slow_path(dst->data == NULL)) { + return NULL; + } + } + + nxt_memcpy(dst->data, src->data, src->len); + dst->len = src->len; + + return dst; +} + + +/* + * nxt_str_copy() creates a C style zero-terminated copy of a source + * nxt_str_t. The function is intended to create strings suitable + * for libc and kernel interfaces so result is pointer to char instead + * of u_char to minimize casts. The copy is aligned to 2 bytes thus + * the lowest bit may be used as marker. + */ + +char * +nxt_str_copy(nxt_mem_pool_t *mp, const nxt_str_t *src) +{ + char *p, *dst; + + dst = nxt_mem_align(mp, 2, src->len + 1); + + if (nxt_fast_path(dst != NULL)) { + p = nxt_cpymem(dst, src->data, src->len); + *p = '\0'; + } + + return dst; +} + + +void +nxt_memcpy_lowcase(u_char *dst, const u_char *src, size_t len) +{ + u_char c; + + while (len != 0) { + c = *src++; + *dst++ = nxt_lowcase(c); + len--; + } +} + + +u_char * +nxt_cpystrn(u_char *dst, const u_char *src, size_t len) +{ + if (len == 0) { + return dst; + } + + while (--len != 0) { + *dst = *src; + + if (*dst == '\0') { + return dst; + } + + dst++; + src++; + } + + *dst = '\0'; + + return dst; +} + + +nxt_int_t +nxt_strcasecmp(const u_char *s1, const u_char *s2) +{ + u_char c1, c2; + nxt_int_t n; + + for ( ;; ) { + c1 = *s1++; + c2 = *s2++; + + c1 = nxt_lowcase(c1); + c2 = nxt_lowcase(c2); + + n = c1 - c2; + + if (n != 0) { + return n; + } + + if (c1 == 0) { + return 0; + } + } +} + + +nxt_int_t +nxt_strncasecmp(const u_char *s1, const u_char *s2, size_t len) +{ + u_char c1, c2; + nxt_int_t n; + + while (len-- != 0) { + c1 = *s1++; + c2 = *s2++; + + c1 = nxt_lowcase(c1); + c2 = nxt_lowcase(c2); + + n = c1 - c2; + + if (n != 0) { + return n; + } + + if (c1 == 0) { + return 0; + } + } + + return 0; +} + + +nxt_int_t +nxt_memcasecmp(const u_char *s1, const u_char *s2, size_t len) +{ + u_char c1, c2; + nxt_int_t n; + + while (len-- != 0) { + c1 = *s1++; + c2 = *s2++; + + c1 = nxt_lowcase(c1); + c2 = nxt_lowcase(c2); + + n = c1 - c2; + + if (n != 0) { + return n; + } + } + + return 0; +} + + +/* + * nxt_memstrn() is intended for search of static substring "ss" + * with known length "len" in string "s" limited by parameter "end". + * Zeros are ignored in both strings. + */ + +u_char * +nxt_memstrn(const u_char *s, const u_char *end, const char *ss, size_t len) +{ + u_char c1, c2, *s2; + + s2 = (u_char *) ss; + c2 = *s2++; + len--; + + while (s < end) { + c1 = *s++; + + if (c1 == c2) { + + if (s + len > end) { + return NULL; + } + + if (nxt_memcmp(s, s2, len) == 0) { + return (u_char *) s - 1; + } + } + } + + return NULL; +} + + +/* + * nxt_strcasestrn() is intended for caseless search of static substring + * "ss" with known length "len" in string "s" limited by parameter "end". + * Zeros are ignored in both strings. + */ + +u_char * +nxt_memcasestrn(const u_char *s, const u_char *end, const char *ss, size_t len) +{ + u_char c1, c2, *s2; + + s2 = (u_char *) ss; + c2 = *s2++; + c2 = nxt_lowcase(c2); + len--; + + while (s < end) { + c1 = *s++; + c1 = nxt_lowcase(c1); + + if (c1 == c2) { + + if (s + len > end) { + return NULL; + } + + if (nxt_memcasecmp(s, s2, len) == 0) { + return (u_char *) s - 1; + } + } + } + + return NULL; +} + + +/* + * nxt_rstrstrn() is intended to search for static substring "ss" + * with known length "len" in string "s" limited by parameter "end" + * in reverse order. Zeros are ignored in both strings. + */ + +u_char * +nxt_rmemstrn(const u_char *s, const u_char *end, const char *ss, size_t len) +{ + u_char c1, c2; + const u_char *s1, *s2; + + s1 = end - len; + s2 = (u_char *) ss; + c2 = *s2++; + len--; + + while (s < s1) { + c1 = *s1; + + if (c1 == c2) { + if (nxt_memcmp(s1 + 1, s2, len) == 0) { + return (u_char *) s1; + } + } + + s1--; + } + + return NULL; +} + + +size_t +nxt_str_strip(u_char *start, u_char *end) +{ + u_char *p; + + for (p = end - 1; p >= start; p--) { + if (*p != NXT_CR && *p != NXT_LF) { + break; + } + } + + return (p + 1) - start; +} diff --git a/src/nxt_string.h b/src/nxt_string.h new file mode 100644 index 00000000..1938bf4b --- /dev/null +++ b/src/nxt_string.h @@ -0,0 +1,173 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_STRING_H_INCLUDED_ +#define _NXT_STRING_H_INCLUDED_ + + +#define \ +nxt_lowcase(c) \ + (u_char) ((c >= 'A' && c <= 'Z') ? c | 0x20 : c) + +#define \ +nxt_upcase(c) \ + (u_char) ((c >= 'a' && c <= 'z') ? c & ~0x20 : c) + + +#define NXT_CR (u_char) 13 +#define NXT_LF (u_char) 10 +#define NXT_CRLF "\x0d\x0a" +#define NXT_CRLF_SIZE (sizeof(NXT_CRLF) - 1) + + +#define NXT_LINEFEED_SIZE 1 + +#define \ +nxt_linefeed(p) \ + *p++ = NXT_LF + + +#define \ +nxt_strlen(s) \ + strlen((char *) s) + + +#define \ +nxt_memzero(buf, len) \ + (void) memset(buf, 0, len) + + +#define \ +nxt_memset(buf, c, len) \ + (void) memset(buf, c, len) + + +#define \ +nxt_memcpy(dst, src, len) \ + (void) memcpy(dst, src, len) + + +NXT_EXPORT void nxt_memcpy_lowcase(u_char *dst, const u_char *src, size_t len); + + +/* + * nxt_cpymem() is an inline function but not macro to + * eliminate possible double evaluation of length "len". + */ +nxt_inline void * +nxt_cpymem(void *dst, const void *src, size_t len) +{ + return ((u_char *) memcpy(dst, src, len)) + len; +} + + +#define \ +nxt_memmove(dst, src, len) \ + (void) memmove(dst, src, len) + + +#define \ +nxt_memcmp(s1, s2, len) \ + memcmp((char *) s1, (char *) s2, len) + + +#define \ +nxt_memchr(s, c, len) \ + memchr((char *) s, c, len) + + +#define \ +nxt_strcmp(s1, s2) \ + strcmp((char *) s1, (char *) s2) + + +#define \ +nxt_strncmp(s1, s2, len) \ + strncmp((char *) s1, (char *) s2, len) + + +NXT_EXPORT u_char *nxt_cpystrn(u_char *dst, const u_char *src, size_t len); +NXT_EXPORT nxt_int_t nxt_strcasecmp(const u_char *s1, const u_char *s2); +NXT_EXPORT nxt_int_t nxt_strncasecmp(const u_char *s1, const u_char *s2, + size_t len); +NXT_EXPORT nxt_int_t nxt_memcasecmp(const u_char *s1, const u_char *s2, + size_t len); + +NXT_EXPORT u_char *nxt_memstrn(const u_char *s, const u_char *end, + const char *ss, size_t len); +NXT_EXPORT u_char *nxt_memcasestrn(const u_char *s, const u_char *end, + const char *ss, size_t len); +NXT_EXPORT u_char *nxt_rmemstrn(const u_char *s, const u_char *end, + const char *ss, size_t len); +NXT_EXPORT size_t nxt_str_strip(u_char *start, u_char *end); + + +typedef struct { + size_t len; + u_char *data; +} nxt_str_t; + + +#define nxt_string(str) { sizeof(str) - 1, (u_char *) str } +#define nxt_string_zero(str) { sizeof(str), (u_char *) str } +#define nxt_null_string { 0, NULL } + + +#define \ +nxt_str_set(str, text) \ + do { \ + (str)->len = sizeof(text) - 1; \ + (str)->data = (u_char *) text; \ + } while (0) + + +#define \ +nxt_str_null(str) \ + do { \ + (str)->len = 0; \ + (str)->data = NULL; \ + } while (0) + + +NXT_EXPORT nxt_str_t *nxt_str_alloc(nxt_mem_pool_t *mp, size_t len); +NXT_EXPORT nxt_str_t *nxt_str_dup(nxt_mem_pool_t *mp, nxt_str_t *dst, + const nxt_str_t *src); +NXT_EXPORT char *nxt_str_copy(nxt_mem_pool_t *mp, const nxt_str_t *src); + + +#define \ +nxt_strstr_eq(s1, s2) \ + (((s1)->len == (s2)->len) \ + && (nxt_memcmp((s1)->data, (s2)->data, (s1)->len) == 0)) + + +#define \ +nxt_strcasestr_eq(s1, s2) \ + (((s1)->len == (s2)->len) \ + && (nxt_memcasecmp((s1)->data, (s2)->data, (s1)->len) == 0)) + + +#define \ +nxt_str_eq(s, p, _len) \ + (((s)->len == _len) && (nxt_memcmp((s)->data, p, _len) == 0)) + + +#define \ +nxt_str_start(s, p, _len) \ + (((s)->len > _len) && (nxt_memcmp((s)->data, p, _len) == 0)) + + +#define \ +nxt_strchr_eq(s, c) \ + (((s)->len == 1) && ((s)->data[0] == c)) + + +#define \ +nxt_strchr_start(s, c) \ + (((s)->len != 0) && ((s)->data[0] == c)) + + +#endif /* _NXT_STRING_H_INCLUDED_ */ diff --git a/src/nxt_test_build.c b/src/nxt_test_build.c new file mode 100644 index 00000000..aefd8c13 --- /dev/null +++ b/src/nxt_test_build.c @@ -0,0 +1,153 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#include <nxt_main.h> + + +#if (NXT_TEST_BUILD_EPOLL) + +int +epoll_create(int size) +{ + return -1; +} + + +int +epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) +{ + return -1; +} + + +int +epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout) +{ + return -1; +} + +int +eventfd(u_int initval, int flags) +{ + return -1; +} + + +int +signalfd(int fd, const sigset_t *mask, int flags) +{ + return -1; +} + + +int +accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) +{ + return -1; +} + +#endif + + +#if (NXT_TEST_BUILD_EVENTPORT) + +int +port_create(void) +{ + return -1; +} + + +int +port_associate(int port, int source, uintptr_t object, int events, void *user) +{ + return -1; +} + + +int +port_dissociate(int port, int source, uintptr_t object) +{ + return -1; +} + + +int +port_send(int port, int events, void *user) +{ + return -1; +} + + +int port_getn(int port, port_event_t list[], uint_t max, uint_t *nget, + const timespec_t *timeout) +{ + return -1; +} + +#endif + + +#if (NXT_TEST_BUILD_POLLSET) + +pollset_t +pollset_create(int maxfd) +{ + return -1; +} + + +int +pollset_destroy(pollset_t ps) +{ + return -1; +} + + +int +pollset_query(pollset_t ps, struct pollfd *pollfd_query) +{ + return -1; +} + + +int +pollset_ctl(pollset_t ps, struct poll_ctl *pollctl_array, int array_length) +{ + return -1; +} + + +int +pollset_poll(pollset_t ps, struct pollfd *polldata_array, int array_length, + int timeout) +{ + return -1; +} + +#endif + + +#if (NXT_TEST_BUILD_SOLARIS_SENDFILEV) + +ssize_t sendfilev(int fd, const struct sendfilevec *vec, + int sfvcnt, size_t *xferred) +{ + return -1; +} + +#endif + + +#if (NXT_TEST_BUILD_AIX_SEND_FILE) + +ssize_t send_file(int *s, struct sf_parms *sf_iobuf, uint_t flags) +{ + return -1; +} + +#endif diff --git a/src/nxt_test_build.h b/src/nxt_test_build.h new file mode 100644 index 00000000..6513e748 --- /dev/null +++ b/src/nxt_test_build.h @@ -0,0 +1,274 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#ifndef _NXT_UNIX_TEST_BUILD_H_INCLUDED_ +#define _NXT_UNIX_TEST_BUILD_H_INCLUDED_ + + +#if (NXT_TEST_BUILD_EPOLL) + +#define NXT_HAVE_EPOLL 1 +#define NXT_HAVE_EPOLL_EDGE 1 +#define NXT_HAVE_EVENTFD 1 +#define NXT_HAVE_SIGNALFD 1 +#define NXT_HAVE_ACCEPT4 1 + +/* Linux epoll declarations */ + +#define EPOLLIN 0x00000001 +#define EPOLLPRI 0x00000002 +#define EPOLLOUT 0x00000004 +#define EPOLLERR 0x00000008 +#define EPOLLHUP 0x00000010 +#define EPOLLRDNORM 0x00000040 +#define EPOLLRDBAND 0x00000080 +#define EPOLLWRNORM 00000x0100 +#define EPOLLWRBAND 0x00000200 +#define EPOLLMSG 0x00000400 +#define EPOLLRDHUP 0x00002000 + +#define EPOLLET 0x80000000 +#define EPOLLONESHOT 0x40000000 + +#define EPOLL_CTL_ADD 1 +#define EPOLL_CTL_DEL 2 +#define EPOLL_CTL_MOD 3 + +#define EFD_SEMAPHORE 1 +#define EFD_NONBLOCK 04000 + + +typedef union epoll_data { + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; +} epoll_data_t; + + +struct epoll_event { + uint32_t events; + epoll_data_t data; +}; + + +struct signalfd_siginfo { + uint32_t ssi_signo; /* Signal number */ + int32_t ssi_errno; /* Error number (unused) */ + int32_t ssi_code; /* Signal code */ + uint32_t ssi_pid; /* PID of sender */ + uint32_t ssi_uid; /* Real UID of sender */ + int32_t ssi_fd; /* File descriptor (SIGIO) */ + uint32_t ssi_tid; /* Kernel timer ID (POSIX timers) */ + uint32_t ssi_band; /* Band event (SIGIO) */ + uint32_t ssi_overrun; /* POSIX timer overrun count */ + uint32_t ssi_trapno; /* Trap number that caused signal */ + int32_t ssi_status; /* Exit status or signal (SIGCHLD) */ + int32_t ssi_int; /* Integer sent by sigqueue(2) */ + uint64_t ssi_ptr; /* Pointer sent by sigqueue(2) */ + uint64_t ssi_utime; /* User CPU time consumed (SIGCHLD) */ + uint64_t ssi_stime; /* System CPU time consumed (SIGCHLD) */ + uint64_t ssi_addr; /* Address that generated signal + (for hardware-generated signals) */ + uint8_t pad[8]; /* Pad size to 128 bytes (allow for + additional fields in the future) */ +}; + + +int epoll_create(int size); +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); +int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout); + +int eventfd(u_int initval, int flags); +int signalfd(int fd, const sigset_t *mask, int flags); + +#define SOCK_NONBLOCK 04000 + +int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags); + +#endif + + +#if (NXT_TEST_BUILD_EVENTPORT) + +#include <poll.h> + +#define NXT_HAVE_EVENTPORT 1 + +#define ushort_t u_short +#define uint_t u_int + +/* Solaris eventport declarations */ + +#define PORT_SOURCE_AIO 1 +#define PORT_SOURCE_TIMER 2 +#define PORT_SOURCE_USER 3 +#define PORT_SOURCE_FD 4 +#define PORT_SOURCE_ALERT 5 +#define PORT_SOURCE_MQ 6 +#define PORT_SOURCE_FILE 7 + +#ifndef ETIME +#define ETIME 62 +#endif + + +typedef struct { + int portev_events; /* event data is source specific */ + ushort_t portev_source; /* event source */ + ushort_t portev_pad; /* port internal use */ + uintptr_t portev_object; /* source specific object */ + void *portev_user; /* user cookie */ +} port_event_t; + + +typedef struct timespec timespec_t; +typedef struct timespec timestruc_t; + + +typedef struct file_obj { + timestruc_t fo_atime; /* Access time from stat(2) */ + timestruc_t fo_mtime; /* Modification time from stat(2) */ + timestruc_t fo_ctime; /* Change time from stat(2) */ + uintptr_t fo_pad[3]; /* For future expansion */ + char *fo_name; /* Null terminated file name */ +} file_obj_t; + + +int port_create(void); +int port_associate(int port, int source, uintptr_t object, int events, + void *user); +int port_dissociate(int port, int source, uintptr_t object); +int port_send(int port, int events, void *user); +int port_getn(int port, port_event_t list[], uint_t max, uint_t *nget, + const timespec_t *timeout); + +#endif + + +#if (NXT_TEST_BUILD_DEVPOLL) + +#define NXT_HAVE_DEVPOLL 1 + +#include <poll.h> +#include <sys/ioctl.h> + +/* Solaris /dev/poll declarations */ + +#define POLLREMOVE 0x0800 +#define DP_POLL 0xD001 +#define DP_ISPOLLED 0xD002 + + +struct dvpoll { + struct pollfd *dp_fds; + int dp_nfds; + int dp_timeout; +}; + +#endif + + +#if (NXT_TEST_BUILD_POLLSET) + +#define NXT_HAVE_POLLSET 1 + +#include <poll.h> + +/* AIX pollset declarations */ + +#define PS_ADD 0x0 +#define PS_MOD 0x1 +#define PS_DELETE 0x2 + + +typedef int pollset_t; + +struct poll_ctl { + short cmd; + short events; + int fd; +}; + + +pollset_t pollset_create(int maxfd); +int pollset_destroy(pollset_t ps); +int pollset_query(pollset_t ps, struct pollfd *pollfd_query); +int pollset_ctl(pollset_t ps, struct poll_ctl *pollctl_array, int array_length); +int pollset_poll(pollset_t ps, struct pollfd *polldata_array, int array_length, + int timeout); + +#endif + + +#if (NXT_TEST_BUILD_FREEBSD_SENDFILE || NXT_TEST_BUILD_MACOSX_SENDFILE) + +#if !(NXT_FREEBSD) && !(NXT_MACOSX) + +struct sf_hdtr { + struct iovec *headers; + int hdr_cnt; + struct iovec *trailers; + int trl_cnt; +}; + +#endif + +#endif + + +#if (NXT_TEST_BUILD_SOLARIS_SENDFILEV) + +/* Solaris declarations */ + +typedef struct sendfilevec { + int sfv_fd; + u_int sfv_flag; + off_t sfv_off; + size_t sfv_len; +} sendfilevec_t; + +#define SFV_FD_SELF -2 + +ssize_t sendfilev(int fd, const struct sendfilevec *vec, int sfvcnt, + size_t *xferred); + +#endif + + +#if (NXT_TEST_BUILD_AIX_SEND_FILE) + +#ifndef uint_t +#define uint_t u_int +#endif + +struct sf_parms { + void *header_data; + uint_t header_length; + + int file_descriptor; + uint64_t file_size; + uint64_t file_offset; + int64_t file_bytes; + + void *trailer_data; + uint_t trailer_length; + + uint64_t bytes_sent; +}; + +#define SF_CLOSE 0x00000001 /* close the socket after completion */ +#define SF_REUSE 0x00000002 /* reuse socket. not supported */ +#define SF_DONT_CACHE 0x00000004 /* don't apply network buffer cache */ +#define SF_SYNC_CACHE 0x00000008 /* sync/update network buffer cache */ + +ssize_t send_file(int *s, struct sf_parms *sf_iobuf, uint_t flags); + +#endif + + +#endif /* _NXT_UNIX_TEST_BUILD_H_INCLUDED_ */ diff --git a/src/nxt_thread.c b/src/nxt_thread.c new file mode 100644 index 00000000..415253c0 --- /dev/null +++ b/src/nxt_thread.c @@ -0,0 +1,228 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static void *nxt_thread_trampoline(void *data); +static void nxt_thread_time_cleanup(void *data); + + +#if (NXT_HAVE_PTHREAD_SPECIFIC_DATA) + +static void nxt_thread_key_dtor(void *data); + + +void +nxt_thread_init_data(nxt_thread_specific_data_t tsd) +{ + void *p; + nxt_err_t err; + pthread_key_t key; + + while ((nxt_atomic_int_t) tsd->key < 0) { + /* + * Atomic allocation of a key number. + * -1 means an uninitialized key, + * -2 is the initializing lock to assure the single value for the key. + */ + if (nxt_atomic_cmp_set(&tsd->key, -1, -2)) { + + err = pthread_key_create(&key, nxt_thread_key_dtor); + if (err != 0) { + nxt_main_log_emerg("pthread_key_create() failed %E", err); + goto fail; + } + + tsd->key = (nxt_atomic_t) key; + + nxt_main_log_debug("pthread_key_create(): %A", tsd->key); + } + } + + if (pthread_getspecific((pthread_key_t) tsd->key) != NULL) { + return; + } + + p = nxt_zalloc(tsd->size); + if (p == NULL) { + goto fail; + } + + err = pthread_setspecific((pthread_key_t) tsd->key, p); + if (err == 0) { + return; + } + + nxt_main_log_alert("pthread_setspecific(%A) failed %E", tsd->key, err); + +fail: + + pthread_exit(NULL); + nxt_unreachable(); +} + + +static void +nxt_thread_key_dtor(void *data) +{ + nxt_main_log_debug("pthread key dtor: %p", data); + + nxt_free(data); +} + +#endif + + +nxt_int_t +nxt_thread_create(nxt_thread_handle_t *handle, nxt_thread_link_t *link) +{ + nxt_err_t err; + + err = pthread_create(handle, NULL, nxt_thread_trampoline, link); + + if (nxt_fast_path(err == 0)) { + nxt_thread_log_debug("pthread_create(): %PH", *handle); + + return NXT_OK; + } + + nxt_thread_log_alert("pthread_create() failed %E", err); + + nxt_free(link); + + return NXT_ERROR; +} + + +static void * +nxt_thread_trampoline(void *data) +{ + nxt_thread_t *thr; + nxt_thread_link_t *link; + nxt_thread_start_t start; + + link = data; + + thr = nxt_thread_init(); + + nxt_log_debug(thr->log, "thread trampoline: %PH", thr->handle); + + pthread_cleanup_push(nxt_thread_time_cleanup, thr); + + start = link->start; + data = link->data; + + if (link->engine != NULL) { + thr->link = link; + + } else { + nxt_free(link); + } + + start(data); + + /* + * nxt_thread_time_cleanup() should be called only if a thread + * would be canceled, so ignore it here because nxt_thread_exit() + * calls nxt_thread_time_free() as well. + */ + pthread_cleanup_pop(0); + + nxt_thread_exit(thr); + nxt_unreachable(); + return NULL; +} + + +nxt_thread_t * +nxt_thread_init(void) +{ + nxt_thread_t *thr; + + nxt_thread_init_data(nxt_thread_context); + + thr = nxt_thread(); + + if (thr->log == NULL) { + thr->log = &nxt_main_log; + thr->handle = nxt_thread_handle(); + + /* + * Threads are never preempted by asynchronous signals, since + * the signals are processed synchronously by dedicated thread. + */ + thr->time.signal = -1; + + nxt_thread_time_update(thr); + } + + return thr; +} + + +static void +nxt_thread_time_cleanup(void *data) +{ + nxt_thread_t *thr; + + thr = data; + + nxt_log_debug(thr->log, "thread time cleanup"); + + nxt_thread_time_free(thr); +} + + +void +nxt_thread_exit(nxt_thread_t *thr) +{ + nxt_log_debug(thr->log, "thread exit"); + + if (thr->link != NULL) { + nxt_event_engine_post(thr->link->engine, thr->link->exit, + (void *) (uintptr_t) thr->handle, NULL, + &nxt_main_log); + + nxt_free(thr->link); + thr->link = NULL; + } + + nxt_thread_time_free(thr); + + pthread_exit(NULL); + nxt_unreachable(); +} + + +void +nxt_thread_cancel(nxt_thread_handle_t handle) +{ + nxt_err_t err; + + nxt_thread_log_debug("thread cancel: %PH", handle); + + err = pthread_cancel(handle); + + if (err != 0) { + nxt_main_log_alert("pthread_cancel(%PH) failed %E", handle, err); + } +} + + +void +nxt_thread_wait(nxt_thread_handle_t handle) +{ + nxt_err_t err; + + nxt_thread_log_debug("thread wait: %PH", handle); + + err = pthread_join(handle, NULL); + + if (err != 0) { + nxt_main_log_alert("pthread_join(%PH) failed %E", handle, err); + } +} diff --git a/src/nxt_thread.h b/src/nxt_thread.h new file mode 100644 index 00000000..ebff808f --- /dev/null +++ b/src/nxt_thread.h @@ -0,0 +1,192 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_THREAD_H_INCLUDED_ +#define _NXT_UNIX_THREAD_H_INCLUDED_ + + +#if (NXT_THREADS) + +/* + * Thread Specific Data + * + * The interface unifies two TSD implementations: the __thread storage + * class and pthread specific data. It works also in non-threaded mode. + * The interface is optimized for the __thread storage class and non-threaded + * mode, since the __thread storage is faster and is supported in modern + * versions of Linux, FreeBSD, Solaris, and MacOSX. Pthread specific data + * is considered as a fallback option. + * + * The underlining interfaces are different: pthread data must be allocated + * by hand and may be accessed only by using pointers whereas __thread data + * allocation is transparent and it is accessed directly. + * + * pthread_getspecific() is usually faster than pthread_setspecific() + * (much faster on MacOSX), so there is no nxt_thread_set_data() interface + * for this reason. It is better to store frequently alterable thread + * log pointer in nxt_thread_t, but not in a dedicated key. + */ + +#if (NXT_HAVE_THREAD_STORAGE_CLASS) + +#define \ +nxt_thread_extern_data(type, tsd) \ + NXT_EXPORT extern __thread type tsd + +#define \ +nxt_thread_declare_data(type, tsd) \ + __thread type tsd + +#define \ +nxt_thread_init_data(tsd) + +#define \ +nxt_thread_get_data(tsd) \ + &tsd + + +#else /* NXT_HAVE_PTHREAD_SPECIFIC_DATA */ + +/* + * nxt_thread_get_data() is used as + * p = nxt_thread_get_data(tsd), + * but the tsd address is actually required. This could be resolved by macro + * #define nxt_thread_get_data(tsd) nxt_thread_get_data_addr(&tsd) + * or by definition nxt_thread_specific_data_t as an array. + * + * On Linux and Solaris pthread_key_t is unsigned integer. + * On FreeBSD, NetBSD, OpenBSD, and HP-UX pthread_key_t is integer. + * On MacOSX and AIX pthread_key_t is unsigned long integer. + * On Cygwin pthread_key_t is pointer to void. + */ + +typedef struct { + nxt_atomic_t key; + size_t size; +} nxt_thread_specific_data_t[1]; + + +#define \ +nxt_thread_extern_data(type, tsd) \ + NXT_EXPORT extern nxt_thread_specific_data_t tsd + +#define \ +nxt_thread_declare_data(type, tsd) \ + nxt_thread_specific_data_t tsd = { { (nxt_atomic_int_t) -1, sizeof(type) } } + +NXT_EXPORT void nxt_thread_init_data(nxt_thread_specific_data_t tsd); + +#define \ +nxt_thread_get_data(tsd) \ + pthread_getspecific((pthread_key_t) tsd->key) + +#endif + + +typedef void (*nxt_thread_start_t)(void *data); + +typedef struct { + nxt_thread_start_t start; + void *data; + nxt_event_engine_t *engine; + nxt_work_handler_t exit; +} nxt_thread_link_t; + + +NXT_EXPORT nxt_int_t nxt_thread_create(nxt_thread_handle_t *handle, + nxt_thread_link_t *link); +NXT_EXPORT nxt_thread_t *nxt_thread_init(void); +NXT_EXPORT void nxt_thread_exit(nxt_thread_t *thr); +NXT_EXPORT void nxt_thread_cancel(nxt_thread_handle_t handle); +NXT_EXPORT void nxt_thread_wait(nxt_thread_handle_t handle); + + +#define \ +nxt_thread_handle() \ + pthread_self() + + +typedef pthread_mutex_t nxt_thread_mutex_t; + +NXT_EXPORT nxt_int_t nxt_thread_mutex_create(nxt_thread_mutex_t *mtx); +NXT_EXPORT void nxt_thread_mutex_destroy(nxt_thread_mutex_t *mtx); +NXT_EXPORT nxt_int_t nxt_thread_mutex_lock(nxt_thread_mutex_t *mtx); +NXT_EXPORT nxt_bool_t nxt_thread_mutex_trylock(nxt_thread_mutex_t *mtx); +NXT_EXPORT nxt_int_t nxt_thread_mutex_unlock(nxt_thread_mutex_t *mtx); + + +typedef pthread_cond_t nxt_thread_cond_t; + +NXT_EXPORT nxt_int_t nxt_thread_cond_create(nxt_thread_cond_t *cond); +NXT_EXPORT void nxt_thread_cond_destroy(nxt_thread_cond_t *cond); +NXT_EXPORT nxt_int_t nxt_thread_cond_signal(nxt_thread_cond_t *cond); +NXT_EXPORT nxt_err_t nxt_thread_cond_wait(nxt_thread_cond_t *cond, + nxt_thread_mutex_t *mtx, nxt_nsec_t timeout); + + +#else /* !(NXT_THREADS) */ + +#define \ +nxt_thread_extern_data(type, tsd) \ + NXT_EXPORT extern type tsd + +#define \ +nxt_thread_declare_data(type, tsd) \ + type tsd + +#define \ +nxt_thread_init_data(tsd) + +#define \ +nxt_thread_get_data(tsd) \ + &tsd + +#endif /* NXT_THREADS */ + + +#if (NXT_HAVE_PTHREAD_YIELD) +#define \ +nxt_thread_yield() \ + pthread_yield() + +#elif (NXT_HAVE_PTHREAD_YIELD_NP) +#define \ +nxt_thread_yield() \ + pthread_yield_np() + +#else +#define \ +nxt_thread_yield() \ + nxt_sched_yield() + +#endif + + +struct nxt_thread_s { + nxt_log_t *log; + nxt_log_t main_log; + + nxt_tid_t tid; + nxt_thread_handle_t handle; +#if (NXT_THREADS) + nxt_thread_link_t *link; + nxt_thread_pool_t *thread_pool; +#endif + + nxt_thread_time_t time; + + nxt_event_engine_t *engine; + nxt_thread_work_queue_t work_queue; + + /* + * Although pointer to a current fiber should be a property of + * engine->fibers, its placement here eliminates 2 memory accesses. + */ + nxt_fiber_t *fiber; +}; + + +#endif /* _NXT_UNIX_THREAD_H_INCLUDED_ */ diff --git a/src/nxt_thread_cond.c b/src/nxt_thread_cond.c new file mode 100644 index 00000000..8db5a337 --- /dev/null +++ b/src/nxt_thread_cond.c @@ -0,0 +1,107 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_int_t +nxt_thread_cond_create(nxt_thread_cond_t *cond) +{ + nxt_err_t err; + + err = pthread_cond_init(cond, NULL); + if (err == 0) { + nxt_thread_log_debug("pthread_cond_init(%p)", cond); + return NXT_OK; + } + + nxt_thread_log_emerg("pthread_cond_init() failed %E", err); + return NXT_ERROR; +} + + +void +nxt_thread_cond_destroy(nxt_thread_cond_t *cond) +{ + nxt_err_t err; + + err = pthread_cond_destroy(cond); + if (err != 0) { + nxt_thread_log_alert("pthread_cond_destroy() failed %E", err); + } + + nxt_thread_log_debug("pthread_cond_destroy(%p)", cond); +} + + +nxt_int_t +nxt_thread_cond_signal(nxt_thread_cond_t *cond) +{ + nxt_err_t err; + + err = pthread_cond_signal(cond); + if (nxt_fast_path(err == 0)) { + nxt_thread_log_debug("pthread_cond_signal(%p)", cond); + return NXT_OK; + } + + nxt_thread_log_alert("pthread_cond_signal() failed %E", err); + + return NXT_ERROR; +} + + +nxt_err_t +nxt_thread_cond_wait(nxt_thread_cond_t *cond, nxt_thread_mutex_t *mtx, + nxt_nsec_t timeout) +{ + nxt_err_t err; + nxt_nsec_t ns; + nxt_thread_t *thr; + nxt_realtime_t *now; + struct timespec ts; + + thr = nxt_thread(); + + if (timeout == NXT_INFINITE_NSEC) { + nxt_log_debug(thr->log, "pthread_cond_wait(%p) enter", cond); + + err = pthread_cond_wait(cond, mtx); + + nxt_thread_time_update(thr); + + if (nxt_fast_path(err == 0)) { + nxt_log_debug(thr->log, "pthread_cond_wait(%p) exit", cond); + return 0; + } + + nxt_log_alert(thr->log, "pthread_cond_wait() failed %E", err); + + } else { + nxt_log_debug(thr->log, "pthread_cond_timedwait(%p, %N) enter", + cond, timeout); + + now = nxt_thread_realtime(thr); + + ns = now->nsec + timeout; + ts.tv_sec = now->sec + ns / 1000000000; + ts.tv_nsec = ns % 1000000000; + + err = pthread_cond_timedwait(cond, mtx, &ts); + + nxt_thread_time_update(thr); + + if (nxt_fast_path(err == 0 || err == NXT_ETIMEDOUT)) { + nxt_log_debug(thr->log, "pthread_cond_timedwait(%p) exit: %d", + cond, err); + return err; + } + + nxt_log_alert(thr->log, "pthread_cond_timedwait() failed %E", err); + } + + return NXT_ERROR; +} diff --git a/src/nxt_thread_id.c b/src/nxt_thread_id.c new file mode 100644 index 00000000..44f10d25 --- /dev/null +++ b/src/nxt_thread_id.c @@ -0,0 +1,170 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +#if (NXT_LINUX) + +/* + * Linux thread id is a pid of thread created by clone(2), + * glibc does not provide a wrapper for gettid(). + */ + +nxt_inline nxt_tid_t +nxt_thread_get_tid(void) +{ + return syscall(SYS_gettid); +} + +#elif (NXT_FREEBSD) + +/* + * FreeBSD 9.0 provides pthread_getthreadid_np(), here is its + * emulation. Kernel thread id is the first field of struct pthread. + * Although kernel exports a thread id as long type, lwpid_t is 32bit. + * Thread id is a number above 100,000. + */ + +nxt_inline nxt_tid_t +nxt_thread_get_tid(void) +{ + return (uint32_t) (*(long *) pthread_self()); +} + +#elif (NXT_SOLARIS) + +/* Solaris pthread_t are numbers starting with 1 */ + +nxt_inline nxt_tid_t +nxt_thread_get_tid(void) +{ + return pthread_self(); +} + +#elif (NXT_MACOSX) + +/* + * MacOSX thread has two thread ids: + * + * 1) MacOSX 10.6 (Snow Leoprad) has pthread_threadid_np() returning + * an uint64_t value, which is obtained using the __thread_selfid() + * syscall. It is a number above 300,000. + */ + +nxt_inline nxt_tid_t +nxt_thread_get_tid(void) +{ + uint64_t tid; + + (void) pthread_threadid_np(NULL, &tid); + return tid; +} + +/* + * 2) Kernel thread mach_port_t returned by pthread_mach_thread_np(). + * It is a number in range 100-100,000. + * + * return pthread_mach_thread_np(pthread_self()); + */ + +#elif (NXT_AIX) + +/* + * pthread_self() in main thread returns 1. + * pthread_self() in other threads returns 258, 515, etc. + * + * pthread_getthrds_np(PTHRDSINFO_QUERY_TID) returns kernel tid + * shown in "ps -ef -m -o THREAD" output. + */ + +nxt_inline nxt_tid_t +nxt_thread_get_tid(void) +{ + int err, size; + pthread_t pt; + struct __pthrdsinfo ti; + + size = 0; + pt = pthread_self(); + + err = pthread_getthrds_np(&pt, PTHRDSINFO_QUERY_TID, &ti, + sizeof(struct __pthrdsinfo), NULL, size); + + if (nxt_fast_path(err == 0)) { + return ti.__pi_tid; + } + + nxt_main_log_alert("pthread_getthrds_np(PTHRDSINFO_QUERY_TID) failed %E", + err); + return 0; +} + +/* + * AIX pthread_getunique_np() returns thread unique number starting with 1. + * OS/400 and i5/OS have pthread_getthreadid_np(), but AIX lacks their + * counterpart. + * + * + * int tid; + * pthread_t pt; + * + * pt = pthread_self(); + * pthread_getunique_np(&pt, &tid); + * return tid; + */ + +#elif (NXT_HPUX) + +/* HP-UX pthread_t are numbers starting with 1 */ + +nxt_inline nxt_tid_t +nxt_thread_get_tid(void) +{ + return pthread_self(); +} + +#else + +nxt_inline nxt_tid_t +nxt_thread_get_tid(void) +{ + return pthread_self(); +} + +#endif + + +nxt_tid_t +nxt_thread_tid(nxt_thread_t *thr) +{ + if (thr == NULL) { + thr = nxt_thread(); + } + +#if (NXT_HAVE_THREAD_STORAGE_CLASS) + + if (nxt_slow_path(thr->tid == 0)) { + thr->tid = nxt_thread_get_tid(); + } + + return thr->tid; + +#else + + if (nxt_fast_path(thr != NULL)) { + + if (nxt_slow_path(thr->tid == 0)) { + thr->tid = nxt_thread_get_tid(); + } + + return thr->tid; + } + + return nxt_thread_get_tid(); + +#endif +} diff --git a/src/nxt_thread_id.h b/src/nxt_thread_id.h new file mode 100644 index 00000000..d083b8fc --- /dev/null +++ b/src/nxt_thread_id.h @@ -0,0 +1,81 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_THREAD_ID_H_INCLUDED_ +#define _NXT_UNIX_THREAD_ID_H_INCLUDED_ + + +#if (NXT_THREADS) + + +#if (NXT_LINUX) + +typedef pid_t nxt_tid_t; + +#elif (NXT_FREEBSD) + +typedef uint32_t nxt_tid_t; + +#elif (NXT_SOLARIS) + +typedef pthread_t nxt_tid_t; + +#elif (NXT_MACOSX) + +typedef uint64_t nxt_tid_t; + +#elif (NXT_AIX) + +typedef tid_t nxt_tid_t; + +#elif (NXT_HPUX) + +typedef pthread_t nxt_tid_t; + +#else + +typedef pthread_t nxt_tid_t; + +#endif + + +NXT_EXPORT nxt_tid_t nxt_thread_tid(nxt_thread_t *thr); + + +/* + * On Linux pthread_t is unsigned long integer. + * On FreeBSD, MacOSX, NetBSD, and OpenBSD pthread_t is pointer to a struct. + * On Solaris and AIX pthread_t is unsigned integer. + * On HP-UX pthread_t is int. + * On Cygwin pthread_t is pointer to void. + * On z/OS pthread_t is "struct { char __[0x08]; }". + */ +typedef pthread_t nxt_thread_handle_t; + + +#define \ +nxt_thread_handle_clear(th) \ + th = (pthread_t) 0 + +#define \ +nxt_thread_handle_equal(th0, th1) \ + pthread_equal(th0, th1) + + +#else /* !(NXT_THREADS) */ + +typedef uint32_t nxt_tid_t; +typedef uint32_t nxt_thread_handle_t; + + +#define \ +nxt_thread_tid(thr) \ + 0 + +#endif + + +#endif /* _NXT_UNIX_THREAD_ID_H_INCLUDED_ */ diff --git a/src/nxt_thread_log.h b/src/nxt_thread_log.h new file mode 100644 index 00000000..2950ae3e --- /dev/null +++ b/src/nxt_thread_log.h @@ -0,0 +1,70 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_THREAD_LOG_H_INCLUDED_ +#define _NXT_THREAD_LOG_H_INCLUDED_ + + +#define nxt_thread_log_emerg(...) \ + do { \ + nxt_thread_t *_thr = nxt_thread(); \ + \ + nxt_log_emerg(_thr->log, __VA_ARGS__); \ + \ + } while (0) + + +#define nxt_thread_log_alert(...) \ + do { \ + nxt_thread_t *_thr = nxt_thread(); \ + \ + nxt_log_alert(_thr->log, __VA_ARGS__); \ + \ + } while (0) + + +#define nxt_thread_log_error(_level, ...) \ + do { \ + nxt_thread_t *_thr = nxt_thread(); \ + \ + nxt_log_error(_level, _thr->log, __VA_ARGS__); \ + \ + } while (0) + + +#if (NXT_DEBUG) + +#define nxt_thread_log_debug(...) \ + do { \ + nxt_thread_t *_thr = nxt_thread(); \ + \ + nxt_log_debug(_thr->log, __VA_ARGS__); \ + \ + } while (0) + + +#define nxt_thread_debug(thr) \ + nxt_thread_t *thr = nxt_thread() + +#else + +#define nxt_thread_log_debug(...) +#define nxt_thread_debug(thr) + +#endif + + +nxt_inline nxt_log_t * +nxt_thread_log(void) +{ + nxt_thread_t *thr; + + thr = nxt_thread(); + return thr->log; +} + + +#endif /* _NXT_THREAD_LOG_H_INCLUDED_ */ diff --git a/src/nxt_thread_mutex.c b/src/nxt_thread_mutex.c new file mode 100644 index 00000000..10c08c2a --- /dev/null +++ b/src/nxt_thread_mutex.c @@ -0,0 +1,192 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * All modern pthread mutex implementations try to acquire a lock atomically + * in userland before going to sleep in kernel. Some spins on SMP systems + * before the sleeping. + * + * In Solaris since version 8 all mutex types spin before sleeping. + * The default spin count is 1000. It can be overridden using + * _THREAD_ADAPTIVE_SPIN=100 environment variable. + * + * In MacOSX all mutex types spin to acquire a lock protecting a mutex's + * internals. If the mutex is busy, thread calls Mach semaphore_wait(). + * + * + * PTHREAD_MUTEX_NORMAL lacks deadlock detection and is the fastest + * mutex type. + * + * Linux: No spinning. The internal name PTHREAD_MUTEX_TIMED_NP + * remains from the times when pthread_mutex_timedlock() was + * non-standard extension. Alias name: PTHREAD_MUTEX_FAST_NP. + * FreeBSD: No spinning. + * + * + * PTHREAD_MUTEX_ERRORCHECK is usually as fast as PTHREAD_MUTEX_NORMAL + * yet has lightweight deadlock detection. + * + * Linux: No spinning. The internal name: PTHREAD_MUTEX_ERRORCHECK_NP. + * FreeBSD: No spinning. + * + * + * PTHREAD_MUTEX_RECURSIVE allows recursive locking. + * + * Linux: No spinning. The internal name: PTHREAD_MUTEX_RECURSIVE_NP. + * FreeBSD: No spinning. + * + * + * PTHREAD_MUTEX_ADAPTIVE_NP spins on SMP systems before sleeping. + * + * Linux: No deadlock detection. Dynamically changes a spin count + * for each mutex from 10 to 100 based on spin count taken + * previously. + * + * FreeBSD: Deadlock detection. The default spin count is 2000. + * It can be overriden using LIBPTHREAD_SPINLOOPS environment + * variable or by pthread_mutex_setspinloops_np(). If a lock + * is still busy, sched_yield() can be called on both UP and + * SMP systems. The default yield loop count is zero, but it + * can be set by LIBPTHREAD_YIELDLOOPS environment variable or + * by pthread_mutex_setyieldloops_np(). sched_yield() moves + * a thread to the end of CPU scheduler run queue and this is + * cheaper than removing the thread from the queue and sleeping. + * + * Solaris: No PTHREAD_MUTEX_ADAPTIVE_NP . + * MacOSX: No PTHREAD_MUTEX_ADAPTIVE_NP. + * + * + * PTHREAD_MUTEX_ELISION_NP is a Linux extension to elide locks using + * Intel Restricted Transactional Memory. It is the most suitable for + * rwlock pattern access because it allows simultaneous reads without lock. + * Supported since glibc 2.18. + * + * + * PTHREAD_MUTEX_DEFAULT is default mutex type. + * + * Linux: PTHREAD_MUTEX_NORMAL. + * FreeBSD: PTHREAD_MUTEX_ERRORCHECK. + * Solaris: PTHREAD_MUTEX_NORMAL. + * MacOSX: PTHREAD_MUTEX_NORMAL. + */ + + +nxt_int_t +nxt_thread_mutex_create(nxt_thread_mutex_t *mtx) +{ + nxt_err_t err; + pthread_mutexattr_t attr; + + err = pthread_mutexattr_init(&attr); + if (err != 0) { + nxt_thread_log_emerg("pthread_mutexattr_init() failed %E", err); + return NXT_ERROR; + } + + err = pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + if (err != 0) { + nxt_thread_log_emerg("pthread_mutexattr_settype" + "(PTHREAD_MUTEX_ERRORCHECK) failed %E", err); + return NXT_ERROR; + } + + err = pthread_mutex_init(mtx, &attr); + if (err != 0) { + nxt_thread_log_emerg("pthread_mutex_init() failed %E", err); + return NXT_ERROR; + } + + err = pthread_mutexattr_destroy(&attr); + if (err != 0) { + nxt_thread_log_alert("pthread_mutexattr_destroy() failed %E", err); + } + + nxt_thread_log_debug("pthread_mutex_init(%p)", mtx); + + return NXT_OK; +} + + +void +nxt_thread_mutex_destroy(nxt_thread_mutex_t *mtx) +{ + nxt_err_t err; + + err = pthread_mutex_destroy(mtx); + if (nxt_slow_path(err != 0)) { + nxt_thread_log_alert("pthread_mutex_destroy() failed %E", err); + } + + nxt_thread_log_debug("pthread_mutex_destroy(%p)", mtx); +} + + +nxt_int_t +nxt_thread_mutex_lock(nxt_thread_mutex_t *mtx) +{ + nxt_err_t err; + + nxt_thread_log_debug("pthread_mutex_lock(%p) enter", mtx); + + err = pthread_mutex_lock(mtx); + if (nxt_fast_path(err == 0)) { + return NXT_OK; + } + + nxt_thread_log_alert("pthread_mutex_lock() failed %E", err); + + return NXT_ERROR; +} + + +nxt_bool_t +nxt_thread_mutex_trylock(nxt_thread_mutex_t *mtx) +{ + nxt_err_t err; + + nxt_thread_debug(thr); + + nxt_log_debug(thr->log, "pthread_mutex_trylock(%p) enter", mtx); + + err = pthread_mutex_trylock(mtx); + if (nxt_fast_path(err == 0)) { + return 1; + } + + if (err == NXT_EBUSY) { + nxt_log_debug(thr->log, "pthread_mutex_trylock(%p) failed", mtx); + + } else { + nxt_thread_log_alert("pthread_mutex_trylock() failed %E", err); + } + + return 0; +} + + +nxt_int_t +nxt_thread_mutex_unlock(nxt_thread_mutex_t *mtx) +{ + nxt_err_t err; + nxt_thread_t *thr; + + err = pthread_mutex_unlock(mtx); + + thr = nxt_thread(); + nxt_thread_time_update(thr); + + if (nxt_fast_path(err == 0)) { + nxt_log_debug(thr->log, "pthread_mutex_unlock(%p) exit", mtx); + return NXT_OK; + } + + nxt_log_alert(thr->log, "pthread_mutex_unlock() failed %E", err); + + return NXT_ERROR; +} diff --git a/src/nxt_thread_pool.c b/src/nxt_thread_pool.c new file mode 100644 index 00000000..a9708ed2 --- /dev/null +++ b/src/nxt_thread_pool.c @@ -0,0 +1,308 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_thread_pool_init(nxt_thread_pool_t *tp); +static void nxt_thread_pool_exit(nxt_thread_t *thr, void *obj, void *data); +static void nxt_thread_pool_start(void *ctx); +static void nxt_thread_pool_wait(nxt_thread_pool_t *tp); + + +nxt_thread_pool_t * +nxt_thread_pool_create(nxt_uint_t max_threads, nxt_nsec_t timeout, + nxt_thread_pool_init_t init, nxt_event_engine_t *engine, + nxt_work_handler_t exit) +{ + nxt_thread_pool_t *tp; + + tp = nxt_zalloc(sizeof(nxt_thread_pool_t)); + if (tp == NULL) { + return NULL; + } + + tp->max_threads = max_threads; + tp->timeout = timeout; + tp->engine = engine; + tp->init = init; + tp->exit = exit; + + return tp; +} + + +nxt_int_t +nxt_thread_pool_post(nxt_thread_pool_t *tp, nxt_work_handler_t handler, + void *obj, void *data, nxt_log_t *log) +{ + nxt_thread_log_debug("thread pool post"); + + if (nxt_slow_path(nxt_thread_pool_init(tp) != NXT_OK)) { + return NXT_ERROR; + } + + nxt_locked_work_queue_add(&tp->work_queue, handler, obj, data, log); + + (void) nxt_sem_post(&tp->sem); + + return NXT_OK; +} + + +static nxt_int_t +nxt_thread_pool_init(nxt_thread_pool_t *tp) +{ + nxt_int_t ret; + nxt_thread_link_t *link; + nxt_thread_handle_t handle; + + if (nxt_fast_path(tp->ready)) { + return NXT_OK; + } + + nxt_thread_spin_lock(&tp->work_queue.lock); + + ret = NXT_OK; + + if (!tp->ready) { + + nxt_thread_log_debug("thread pool init"); + + (void) nxt_atomic_fetch_add(&tp->threads, 1); + + if (nxt_fast_path(nxt_sem_init(&tp->sem, 0) == NXT_OK)) { + + nxt_locked_work_queue_create(&tp->work_queue, 0); + + link = nxt_malloc(sizeof(nxt_thread_link_t)); + + if (nxt_fast_path(link != NULL)) { + link->start = nxt_thread_pool_start; + link->data = tp; + link->engine = tp->engine; + /* + * link->exit is not used. link->engine is used just to + * set thr->link by nxt_thread_trampoline() and the link + * is a mark of the first thread of pool. + */ + if (nxt_thread_create(&handle, link) == NXT_OK) { + tp->ready = 1; + goto done; + } + } + + nxt_sem_destroy(&tp->sem); + } + + (void) nxt_atomic_fetch_add(&tp->threads, -1); + + nxt_locked_work_queue_destroy(&tp->work_queue); + + ret = NXT_ERROR; + } + +done: + + nxt_thread_spin_unlock(&tp->work_queue.lock); + + return ret; +} + + +static void +nxt_thread_pool_start(void *ctx) +{ + void *obj, *data; + nxt_thread_t *thr; + nxt_thread_pool_t *tp; + nxt_work_handler_t handler; + + tp = ctx; + thr = nxt_thread(); + + if (thr->link != NULL) { + /* Only the first thread has a link. */ + tp->main = thr->handle; + nxt_free(thr->link); + thr->link = NULL; + } + + thr->thread_pool = tp; + + if (tp->init != NULL) { + tp->init(); + } + + nxt_thread_work_queue_create(thr, 8); + + for ( ;; ) { + nxt_thread_pool_wait(tp); + + handler = nxt_locked_work_queue_pop(&tp->work_queue, &obj, + &data, &thr->log); + + if (nxt_fast_path(handler != NULL)) { + nxt_log_debug(thr->log, "locked work queue"); + handler(thr, obj, data); + } + + for ( ;; ) { + thr->log = &nxt_main_log; + + handler = nxt_thread_work_queue_pop(thr, &obj, &data, &thr->log); + + if (handler == NULL) { + break; + } + + handler(thr, obj, data); + } + + thr->log = &nxt_main_log; + } +} + + +static void +nxt_thread_pool_wait(nxt_thread_pool_t *tp) +{ + nxt_err_t err; + nxt_thread_t *thr; + nxt_atomic_uint_t waiting, threads; + nxt_thread_link_t *link; + nxt_thread_handle_t handle; + + thr = nxt_thread(); + + nxt_log_debug(thr->log, "thread pool wait"); + + (void) nxt_atomic_fetch_add(&tp->waiting, 1); + + for ( ;; ) { + err = nxt_sem_wait(&tp->sem, tp->timeout); + + if (err == 0) { + waiting = nxt_atomic_fetch_add(&tp->waiting, -1); + break; + } + + if (err == NXT_ETIMEDOUT) { + if (nxt_thread_handle_equal(thr->handle, tp->main)) { + continue; + } + } + + (void) nxt_atomic_fetch_add(&tp->waiting, -1); + (void) nxt_atomic_fetch_add(&tp->threads, -1); + + nxt_thread_exit(thr); + nxt_unreachable(); + } + + nxt_log_debug(thr->log, "thread pool awake, waiting: %A", waiting); + + if (waiting > 1) { + return; + } + + do { + threads = tp->threads; + + if (threads >= tp->max_threads) { + return; + } + + } while (!nxt_atomic_cmp_set(&tp->threads, threads, threads + 1)); + + link = nxt_zalloc(sizeof(nxt_thread_link_t)); + + if (nxt_fast_path(link != NULL)) { + link->start = nxt_thread_pool_start; + link->data = tp; + + if (nxt_thread_create(&handle, link) != NXT_OK) { + (void) nxt_atomic_fetch_add(&tp->threads, -1); + } + } +} + + +void +nxt_thread_pool_destroy(nxt_thread_pool_t *tp) +{ + nxt_thread_t *thr; + + if (!tp->ready) { + thr = nxt_thread(); + + nxt_thread_work_queue_add(thr, &thr->work_queue.main, tp->exit, + tp, NULL, &nxt_main_log); + return; + } + + if (tp->max_threads != 0) { + /* Disable new threads creation and mark a pool as being destroyed. */ + tp->max_threads = 0; + + nxt_thread_pool_post(tp, nxt_thread_pool_exit, tp, NULL, &nxt_main_log); + } +} + + +/* + * A thread handle (pthread_t) is either pointer or integer, so it can be + * passed as work handler pointer "data" argument. To convert void pointer + * to pthread_t and vice versa the source argument should be cast first to + * uintptr_t type and then to the destination type. + * + * If the handle would be a struct it should be stored in thread pool and + * the thread pool must be freed in the thread pool exit procedure after + * the last thread of pool will exit. + */ + +static void +nxt_thread_pool_exit(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_thread_pool_t *tp; + nxt_atomic_uint_t threads; + nxt_thread_handle_t handle; + + tp = obj; + + nxt_log_debug(thr->log, "thread pool exit"); + + if (data != NULL) { + handle = (nxt_thread_handle_t) (uintptr_t) data; + nxt_thread_wait(handle); + } + + threads = nxt_atomic_fetch_add(&tp->threads, -1); + + nxt_log_debug(thr->log, "thread pool threads: %A", threads); + + if (threads > 1) { + nxt_thread_pool_post(tp, nxt_thread_pool_exit, tp, + (void *) (uintptr_t) thr->handle, &nxt_main_log); + + } else { + nxt_main_log_debug("thread pool destroy"); + + nxt_event_engine_post(tp->engine, tp->exit, tp, + (void *) (uintptr_t) thr->handle, &nxt_main_log); + + nxt_sem_destroy(&tp->sem); + + nxt_locked_work_queue_destroy(&tp->work_queue); + + nxt_free(tp); + } + + nxt_thread_work_queue_destroy(thr); + + nxt_thread_exit(thr); + nxt_unreachable(); +} diff --git a/src/nxt_thread_pool.h b/src/nxt_thread_pool.h new file mode 100644 index 00000000..eb359260 --- /dev/null +++ b/src/nxt_thread_pool.h @@ -0,0 +1,41 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_THREAD_POOL_H_INCLUDED_ +#define _NXT_UNIX_THREAD_POOL_H_INCLUDED_ + + +typedef void (*nxt_thread_pool_init_t)(void); + + +struct nxt_thread_pool_s { + nxt_atomic_t ready; + nxt_atomic_t waiting; + nxt_atomic_t threads; + nxt_uint_t max_threads; + + nxt_sem_t sem; + nxt_nsec_t timeout; + + nxt_locked_work_queue_t work_queue; + + nxt_thread_handle_t main; + + nxt_event_engine_t *engine; + nxt_thread_pool_init_t init; + nxt_work_handler_t exit; +}; + + +NXT_EXPORT nxt_thread_pool_t *nxt_thread_pool_create(nxt_uint_t max_threads, + nxt_nsec_t timeout, nxt_thread_pool_init_t init, + nxt_event_engine_t *engine, nxt_work_handler_t exit); +NXT_EXPORT void nxt_thread_pool_destroy(nxt_thread_pool_t *tp); +NXT_EXPORT nxt_int_t nxt_thread_pool_post(nxt_thread_pool_t *tp, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log); + + +#endif /* _NXT_UNIX_THREAD_POOL_H_INCLUDED_ */ diff --git a/src/nxt_thread_time.c b/src/nxt_thread_time.c new file mode 100644 index 00000000..f359a95e --- /dev/null +++ b/src/nxt_thread_time.c @@ -0,0 +1,468 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * Each thread keeps several time representations in its thread local + * storage: + * the monotonic time in nanoseconds since unspecified point in the past, + * the real time in seconds and nanoseconds since the Epoch, + * the local time and GMT time structs, + * and various user-defined text representations of local and GMT times. + * + * The monotonic time is used mainly by engine timers and is updated after + * a kernel operation which can block for unpredictable duration like event + * polling. Besides getting the monotonic time is generally faster than + * getting the real time, so the monotonic time is also used for milestones + * to update cached real time seconds and, if debug log enabled, milliseconds. + * As a result, the cached real time is updated at most one time per second + * or millisecond respectively. If there is a signal event support or in + * multi-threaded mode, then the cached real time and local time structs + * are updated only on demand. In single-threaded mode without the signal + * event support the cached real and local time are updated synchronously + * with the monotonic time update. GMT time structs and text representations + * are always updated only on demand. + */ + + +#if (NXT_THREADS) +static void nxt_time_thread(void *data); +static void nxt_thread_time_shared(nxt_monotonic_time_t *now); + +static nxt_bool_t nxt_use_shared_time = 0; +static volatile nxt_monotonic_time_t nxt_shared_time; +#endif + +static void nxt_thread_realtime_update(nxt_thread_t *thr, + nxt_monotonic_time_t *now); +static u_char *nxt_thread_time_string_no_cache(nxt_thread_t *thr, + nxt_time_string_t *ts, u_char *buf); +static nxt_atomic_uint_t nxt_thread_time_string_slot(nxt_time_string_t *ts); +static nxt_time_string_cache_t *nxt_thread_time_string_cache(nxt_thread_t *thr, + nxt_atomic_uint_t slot); + + +static nxt_atomic_int_t nxt_gmtoff; + + +void +nxt_thread_time_update(nxt_thread_t *thr) +{ +#if (NXT_THREADS) + + if (nxt_use_shared_time) { + nxt_thread_time_shared(&thr->time.now); + + } else { + nxt_monotonic_time(&thr->time.now); + } + +#else + + nxt_monotonic_time(&thr->time.now); + + if (thr->time.signal >= 0) { + nxt_time_t s; + + /* + * Synchronous real time update: + * single-threaded mode without signal event support. + */ + + s = nxt_thread_time(thr); + + if (thr->time.signal == 0 && thr->time.last_localtime != s) { + /* Synchronous local time update in non-signal context. */ + + nxt_localtime(s, &thr->time.localtime); + thr->time.last_localtime = s; + + nxt_gmtoff = nxt_timezone(&thr->time.localtime); + } + } + +#endif +} + + +void +nxt_thread_time_free(nxt_thread_t *thr) +{ + nxt_uint_t i; + nxt_time_string_cache_t *tsc; + + tsc = thr->time.strings; + + if (tsc) { + thr->time.no_cache = 1; + + for (i = 0; i < thr->time.nstrings; i++) { + nxt_free(tsc[i].string.data); + } + + nxt_free(tsc); + thr->time.strings = NULL; + } +} + + +#if (NXT_THREADS) + +void +nxt_time_thread_start(nxt_msec_t interval) +{ + nxt_thread_link_t *link; + nxt_thread_handle_t handle; + + link = nxt_zalloc(sizeof(nxt_thread_link_t)); + + if (nxt_fast_path(link != NULL)) { + link->start = nxt_time_thread; + link->data = (void *) (uintptr_t) interval; + + (void) nxt_thread_create(&handle, link); + } +} + + +static void +nxt_time_thread(void *data) +{ + nxt_nsec_t interval, rest; + nxt_thread_t *thr; + nxt_monotonic_time_t now; + + interval = (uintptr_t) data; + interval *= 1000000; + + thr = nxt_thread(); + /* + * The time thread is never preempted by asynchronous signals, since + * the signals are processed synchronously by dedicated thread. + */ + thr->time.signal = -1; + + nxt_log_debug(thr->log, "time thread"); + + nxt_memzero(&now, sizeof(nxt_monotonic_time_t)); + + nxt_monotonic_time(&now); + nxt_thread_realtime_update(thr, &now); + + nxt_shared_time = now; + nxt_use_shared_time = 1; + + for ( ;; ) { + rest = 1000000000 - now.realtime.nsec; + + nxt_nanosleep(nxt_min(interval, rest)); + + nxt_monotonic_time(&now); + nxt_thread_realtime_update(thr, &now); + + nxt_shared_time = now; + +#if 0 + thr->time.now = now; + nxt_log_debug(thr->log, "time thread"); +#endif + +#if 0 + if (nxt_exiting) { + nxt_use_shared_time = 0; + return; + } +#endif + } +} + + +static void +nxt_thread_time_shared(nxt_monotonic_time_t *now) +{ + nxt_uint_t n; + nxt_time_t t; + nxt_nsec_t m, u; + + /* Lock-free thread time update. */ + + for ( ;; ) { + *now = nxt_shared_time; + + t = nxt_shared_time.realtime.sec; + n = nxt_shared_time.realtime.nsec; + m = nxt_shared_time.monotonic; + u = nxt_shared_time.update; + + if (now->realtime.sec == t && now->realtime.nsec == n + && now->monotonic == m && now->update == u) + { + return; + } + } +} + +#endif + + +nxt_time_t +nxt_thread_time(nxt_thread_t *thr) +{ + nxt_thread_realtime_update(thr, &thr->time.now); + + return thr->time.now.realtime.sec; +} + + +nxt_realtime_t * +nxt_thread_realtime(nxt_thread_t *thr) +{ + nxt_thread_realtime_update(thr, &thr->time.now); + + return &thr->time.now.realtime; +} + + +static void +nxt_thread_realtime_update(nxt_thread_t *thr, nxt_monotonic_time_t *now) +{ + nxt_nsec_t delta; + +#if (NXT_DEBUG) + + if (nxt_slow_path(thr->log->level == NXT_LOG_DEBUG || nxt_debug)) { + + if (now->monotonic >= now->update) { + nxt_realtime(&now->realtime); + + delta = 1000000 - now->realtime.nsec % 1000000; + now->update = now->monotonic + delta; + } + + return; + } + +#endif + + if (now->monotonic >= now->update) { + nxt_realtime(&now->realtime); + + delta = 1000000000 - now->realtime.nsec; + now->update = now->monotonic + delta; + } +} + + +u_char * +nxt_thread_time_string(nxt_thread_t *thr, nxt_time_string_t *ts, u_char *buf) +{ + u_char *p; + struct tm *tm; + nxt_time_t s; + nxt_bool_t update; + nxt_atomic_uint_t slot; + nxt_time_string_cache_t *tsc; + + if (nxt_slow_path(thr == NULL || thr->time.no_cache)) { + return nxt_thread_time_string_no_cache(thr, ts, buf); + } + + slot = nxt_thread_time_string_slot(ts); + + tsc = nxt_thread_time_string_cache(thr, slot); + if (tsc == NULL) { + return buf; + } + + if (thr->time.signal < 0) { + /* + * Lazy real time update: + * signal event support or multi-threaded mode. + */ + nxt_thread_realtime_update(thr, &thr->time.now); + } + + s = thr->time.now.realtime.sec; + + update = (s != tsc->last); + +#if (NXT_DEBUG) + + if (ts->msec == NXT_THREAD_TIME_MSEC + && (nxt_slow_path(thr->log->level == NXT_LOG_DEBUG || nxt_debug))) + { + nxt_msec_t ms; + + ms = thr->time.now.realtime.nsec / 1000000; + update |= (ms != tsc->last_msec); + tsc->last_msec = ms; + } + +#endif + + if (nxt_slow_path(update)) { + + if (ts->timezone == NXT_THREAD_TIME_LOCAL) { + + tm = &thr->time.localtime; + + if (nxt_slow_path(s != thr->time.last_localtime)) { + + if (thr->time.signal < 0) { + /* + * Lazy local time update: + * signal event support or multi-threaded mode. + */ + nxt_localtime(s, &thr->time.localtime); + thr->time.last_localtime = s; + + } else { + /* + * "thr->time.signal >= 0" means that a thread may be + * interrupted by a signal handler. Since localtime() + * cannot be safely called in a signal context, the + * thread's thr->time.localtime must be updated regularly + * by nxt_thread_time_update() in non-signal context. + * Stale timestamp means that nxt_thread_time_string() + * is being called in a signal context, so here is + * Async-Signal-Safe localtime() emulation using the + * latest cached GMT offset. + * + * The timestamp is not set here intentionally to update + * thr->time.localtime later in non-signal context. The + * real previously cached thr->localtime is used because + * Linux and Solaris strftime() depend on tm.tm_isdst + * and tm.tm_gmtoff fields. + */ + nxt_gmtime(s + nxt_timezone(tm), tm); + } + } + + } else { + tm = &thr->time.gmtime; + + if (nxt_slow_path(s != thr->time.last_gmtime)) { + nxt_gmtime(s, tm); + thr->time.last_gmtime = s; + } + + } + + p = tsc->string.data; + + if (nxt_slow_path(p == NULL)) { + + thr->time.no_cache = 1; + p = nxt_zalloc(ts->size); + thr->time.no_cache = 0; + + if (p == NULL) { + return buf; + } + + tsc->string.data = p; + } + + p = ts->handler(p, &thr->time.now.realtime, tm, ts->size, ts->format); + + tsc->string.len = p - tsc->string.data; + + if (nxt_slow_path(tsc->string.len == 0)) { + return buf; + } + + tsc->last = s; + } + + return nxt_cpymem(buf, tsc->string.data, tsc->string.len); +} + + +static u_char * +nxt_thread_time_string_no_cache(nxt_thread_t *thr, nxt_time_string_t *ts, + u_char *buf) +{ + struct tm tm; + nxt_realtime_t now; + + nxt_realtime(&now); + + if (ts->timezone == NXT_THREAD_TIME_LOCAL) { + + if (thr == NULL || thr->time.signal <= 0) { + /* Non-signal context */ + nxt_localtime(now.sec, &tm); + + } else { + nxt_gmtime(now.sec + nxt_gmtoff, &tm); + } + + } else { + nxt_gmtime(now.sec, &tm); + } + + return ts->handler(buf, &now, &tm, ts->size, ts->format); +} + + +static nxt_atomic_uint_t +nxt_thread_time_string_slot(nxt_time_string_t *ts) +{ + static nxt_atomic_t slot; + + while (nxt_slow_path((nxt_atomic_int_t) ts->slot < 0)) { + /* + * Atomic allocation of a slot number. + * -1 means an uninitialized slot, + * -2 is the initializing lock to assure the single value for the slot. + */ + if (nxt_atomic_cmp_set(&ts->slot, -1, -2)) { + ts->slot = nxt_atomic_fetch_add(&slot, 1); + + /* No "break" here since it adds only dispensable "jmp". */ + } + } + + return (nxt_atomic_uint_t) ts->slot; +} + + +static nxt_time_string_cache_t * +nxt_thread_time_string_cache(nxt_thread_t *thr, nxt_atomic_uint_t slot) +{ + size_t size; + nxt_atomic_uint_t i, nstrings; + nxt_time_string_cache_t *tsc; + + if (nxt_fast_path(slot < thr->time.nstrings)) { + tsc = &thr->time.strings[slot]; + nxt_prefetch(tsc->string.data); + return tsc; + } + + nstrings = slot + 1; + size = nstrings * sizeof(nxt_time_string_cache_t); + + thr->time.no_cache = 1; + tsc = nxt_realloc(thr->time.strings, size); + thr->time.no_cache = 0; + + if (tsc == NULL) { + return NULL; + } + + for (i = thr->time.nstrings; i < nstrings; i++) { + tsc[i].last = -1; + tsc[i].string.data = NULL; + } + + thr->time.strings = tsc; + thr->time.nstrings = nstrings; + + return &tsc[slot]; +} diff --git a/src/nxt_thread_time.h b/src/nxt_thread_time.h new file mode 100644 index 00000000..126709de --- /dev/null +++ b/src/nxt_thread_time.h @@ -0,0 +1,99 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_THREAD_TIME_H_INCLUDED_ +#define _NXT_THREAD_TIME_H_INCLUDED_ + + +#define NXT_THREAD_TIME_LOCAL 0 +#define NXT_THREAD_TIME_GMT 1 + +#define NXT_THREAD_TIME_SEC 0 +#define NXT_THREAD_TIME_MSEC 1 + + +typedef struct { + nxt_atomic_t slot; + u_char *(*handler)(u_char *buf, nxt_realtime_t *now, + struct tm *tm, size_t size, + const char *format); + const char *format; + size_t size; + + uint8_t timezone; /* 1 bit */ + uint8_t msec; /* 1 bit */ +} nxt_time_string_t; + + +typedef struct { + nxt_time_t last; +#if (NXT_DEBUG) + nxt_msec_t last_msec; +#endif + nxt_str_t string; +} nxt_time_string_cache_t; + + +typedef struct { + nxt_monotonic_time_t now; + + nxt_time_t last_gmtime; + nxt_time_t last_localtime; + struct tm gmtime; + struct tm localtime; + + uint32_t no_cache; /* 1 bit */ + + /* + * The flag indicating a signal state of a thread. + * It is used to handle local time of the thread: + * -1 means that the thread never runs in a signal context; + * 0 means that the thread may run in a signal context but not now; + * >0 means that the thread runs in a signal context right now. + */ + nxt_atomic_int_t signal; + + nxt_atomic_uint_t nstrings; + nxt_time_string_cache_t *strings; +} nxt_thread_time_t; + + +#if (NXT_THREADS) +void nxt_time_thread_start(nxt_msec_t interval); +#endif + + +NXT_EXPORT void nxt_thread_time_update(nxt_thread_t *thr); +void nxt_thread_time_free(nxt_thread_t *thr); +NXT_EXPORT nxt_time_t nxt_thread_time(nxt_thread_t *thr); +NXT_EXPORT nxt_realtime_t *nxt_thread_realtime(nxt_thread_t *thr); +NXT_EXPORT u_char *nxt_thread_time_string(nxt_thread_t *thr, + nxt_time_string_t *ts, u_char *buf); + + +#define \ +nxt_thread_monotonic_time(thr) \ + (thr)->time.now.monotonic + + +#if (NXT_DEBUG) + +#define \ +nxt_thread_time_debug_update(thr) \ + nxt_thread_time_update(thr) + +#else + +#define \ +nxt_thread_time_debug_update(thr) + +#endif + + +NXT_EXPORT void nxt_gmtime(nxt_time_t s, struct tm *tm); + + +#endif /* _NXT_THREAD_TIME_H_INCLUDED_ */ diff --git a/src/nxt_time.c b/src/nxt_time.c new file mode 100644 index 00000000..dfead51c --- /dev/null +++ b/src/nxt_time.c @@ -0,0 +1,365 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* OS-specific real, monotonic, and local times and timezone update. */ + + +/* Real time. */ + +#if (NXT_HAVE_CLOCK_REALTIME_COARSE) + +/* + * Linux clock_gettime() resides on the vDSO page. Linux 2.6.32 + * clock_gettime(CLOCK_REALTIME_COARSE) uses only cached values and does + * not read TSC or HPET so it has the kernel jiffy precision (1ms by default) + * and it is several times faster than clock_gettime(CLOCK_REALTIME). + */ + +void +nxt_realtime(nxt_realtime_t *now) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_REALTIME_COARSE, &ts); + + now->sec = (nxt_time_t) ts.tv_sec; + now->nsec = ts.tv_nsec; +} + + +#elif (NXT_HAVE_CLOCK_REALTIME_FAST) + +/* + * FreeBSD 7.0 specific clock_gettime(CLOCK_REALTIME_FAST) may be + * 5-30 times faster than clock_gettime(CLOCK_REALTIME) depending + * on kern.timecounter.hardware. The clock has a precision of 1/HZ + * seconds (HZ is 1000 on modern platforms, thus 1ms precision). + * FreeBSD 9.2 clock_gettime() resides on the vDSO page and reads + * TSC. clock_gettime(CLOCK_REALTIME_FAST) is the same as + * clock_gettime(CLOCK_REALTIME). + */ + +void +nxt_realtime(nxt_realtime_t *now) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_REALTIME_FAST, &ts); + + now->sec = (nxt_time_t) ts.tv_sec; + now->nsec = ts.tv_nsec; +} + + +#elif (NXT_HAVE_CLOCK_REALTIME && !(NXT_HPUX)) + +/* + * clock_gettime(CLOCK_REALTIME) is supported by Linux, FreeBSD 3.0, + * Solaris 8, NetBSD 1.3, and AIX. HP-UX supports it too, however, + * it is implemented through a call to gettimeofday(). Linux + * clock_gettime(CLOCK_REALTIME) resides on the vDSO page and reads + * TSC or HPET. FreeBSD 9.2 clock_gettime(CLOCK_REALTIME) resides + * on the vDSO page and reads TSC. + */ + +void +nxt_realtime(nxt_realtime_t *now) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_REALTIME, &ts); + + now->sec = (nxt_time_t) ts.tv_sec; + now->nsec = ts.tv_nsec; +} + + +#else + +/* MacOSX, HP-UX. */ + +void +nxt_realtime(nxt_realtime_t *now) +{ + struct timeval tv; + + (void) gettimeofday(&tv, NULL); + + now->sec = (nxt_time_t) tv.tv_sec; + now->nsec = tv.tv_usec * 1000; +} + +#endif + + +/* Monotonic time. */ + +#if (NXT_HAVE_CLOCK_MONOTONIC_COARSE) + +/* + * Linux clock_gettime() resides on the vDSO page. Linux 2.6.32 + * clock_gettime(CLOCK_MONOTONIC_COARSE) uses only cached values and does + * not read TSC or HPET so it has the kernel jiffy precision (1ms by default) + * and it is several times faster than clock_gettime(CLOCK_MONOTONIC). + */ + +void +nxt_monotonic_time(nxt_monotonic_time_t *now) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_MONOTONIC_COARSE, &ts); + + now->monotonic = (nxt_nsec_t) ts.tv_sec * 1000000000 + ts.tv_nsec; +} + + +#elif (NXT_HAVE_CLOCK_MONOTONIC_FAST) + +/* + * FreeBSD 7.0 specific clock_gettime(CLOCK_MONOTONIC_FAST) may be + * 5-30 times faster than clock_gettime(CLOCK_MONOTONIC) depending + * on kern.timecounter.hardware. The clock has a precision of 1/HZ + * seconds (HZ is 1000 on modern platforms, thus 1ms precision). + * FreeBSD 9.2 clock_gettime() resides on the vDSO page and reads + * TSC. clock_gettime(CLOCK_MONOTONIC_FAST) is the same as + * clock_gettime(CLOCK_MONOTONIC). + */ + +void +nxt_monotonic_time(nxt_monotonic_time_t *now) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_MONOTONIC_FAST, &ts); + + now->monotonic = (nxt_nsec_t) ts.tv_sec * 1000000000 + ts.tv_nsec; +} + + +#elif (NXT_HAVE_HG_GETHRTIME) + +/* + * HP-UX 11.31 provides fast hg_gethrtime() which uses a chunk of memory + * shared between userspace application and the kernel, and was introduced + * by Project Mercury ("HG"). + */ + +void +nxt_monotonic_time(nxt_monotonic_time_t *now) +{ + now->monotonic = (nxt_nsec_t) hg_gethrtime(); +} + + +#elif (NXT_SOLARIS || NXT_HPUX) + +/* + * Solaris gethrtime(), clock_gettime(CLOCK_REALTIME), and gettimeofday() + * use a fast systrap whereas clock_gettime(CLOCK_MONOTONIC) and other + * clock_gettime()s use normal systrap. However, the difference is + * negligible on x86_64. + * + * HP-UX lacks clock_gettime(CLOCK_MONOTONIC) but has lightweight + * system call gethrtime(). + */ + +void +nxt_monotonic_time(nxt_monotonic_time_t *now) +{ + now->monotonic = (nxt_nsec_t) gethrtime(); +} + + +#elif (NXT_HAVE_CLOCK_MONOTONIC) + +/* + * clock_gettime(CLOCK_MONOTONIC) is supported by Linux, FreeBSD 5.0, + * Solaris 8, NetBSD 1.6, and AIX. Linux clock_gettime(CLOCK_MONOTONIC) + * resides on the vDSO page and reads TSC or HPET. FreeBSD 9.2 + * clock_gettime(CLOCK_MONOTONIC) resides on the vDSO page and reads TSC. + */ + +void +nxt_monotonic_time(nxt_monotonic_time_t *now) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_MONOTONIC, &ts); + + now->monotonic = (nxt_nsec_t) ts.tv_sec * 1000000000 + ts.tv_nsec; +} + + +#elif (NXT_MACOSX) + +/* + * MacOSX does not support clock_gettime(), but mach_absolute_time() returns + * monotonic ticks. To get nanoseconds the ticks should be multiplied then + * divided by numerator/denominator returned by mach_timebase_info(), however + * on modern MacOSX they are 1/1. On PowerPC MacOSX these values were + * 1000000000/33333335 or 1000000000/25000000, on iOS 4+ they were 125/3, + * and on iOS 3 they were 1000000000/24000000. + */ + +void +nxt_monotonic_time(nxt_monotonic_time_t *now) +{ + now->monotonic = mach_absolute_time(); +} + + +#else + +void +nxt_monotonic_time(nxt_monotonic_time_t *now) +{ + nxt_nsec_t current; + nxt_nsec_int_t delta; + struct timeval tv; + + (void) gettimeofday(&tv, NULL); + + now->realtime.sec = (nxt_time_t) tv.tv_sec; + now->realtime.nsec = tv.tv_usec * 1000; + + /* + * Monotonic time emulation using gettimeofday() + * for platforms which lack monotonic time. + */ + + current = (nxt_nsec_t) tv.tv_sec * 1000000000 + tv.tv_usec * 1000; + delta = current - now->previous; + now->previous = current; + + if (delta > 0) { + now->monotonic += delta; + + } else { + /* The time went backward. */ + now->monotonic++; + } + + /* + * Eliminate subsequent gettimeofday() call + * in nxt_thread_realtime_update(). + */ + now->update = now->monotonic + 1; +} + +#endif + + +/* Local time. */ + +#if (NXT_HAVE_LOCALTIME_R) + +void +nxt_localtime(nxt_time_t s, struct tm *tm) +{ + time_t _s; + + _s = (time_t) s; + (void) localtime_r(&_s, tm); +} + + +#else + +void +nxt_localtime(nxt_time_t s, struct tm *tm) +{ + time_t _s; + struct tm *_tm; + + _s = (time_t) s; + _tm = localtime(&_s); + *tm = *_tm; +} + +#endif + + +/* Timezone update. */ + +#if (NXT_LINUX) + +/* + * Linux glibc does not test /etc/localtime change + * in localtime_r(), but tests in localtime(). + */ + +void +nxt_timezone_update(void) +{ + time_t s; + + s = time(NULL); + (void) localtime(&s); +} + + +#elif (NXT_FREEBSD) + +/* + * FreeBSD libc does not test /etc/localtime change, but it can be + * worked around by calling tzset() with TZ and then without TZ + * to update timezone. This trick should work since FreeBSD 2.1.0. + */ + +void +nxt_timezone_update(void) +{ + if (getenv("TZ") != NULL) { + return; + } + + /* The libc uses /etc/localtime if TZ is not set. */ + + (void) putenv((char *) "TZ=UTC"); + tzset(); + + (void) unsetenv("TZ"); + tzset(); +} + + +#elif (NXT_SOLARIS) + +/* + * Solaris 10, patch 142909-17 introduced tzreload(8): + * + * The tzreload command notifies active (running) processes to reread + * timezone information. The timezone information is cached in each + * process, absent a tzreload command, is never reread until a process + * is restarted. In response to a tzreload command, active processes + * reread the current timezone information at the next call to ctime(3C) + * and mktime(3C). By default, the tzreload notification is sent to + * the processes within the current zone. + */ + +void +nxt_timezone_update(void) +{ + time_t s; + + s = time(NULL); + (void) ctime(&s); +} + + +#else + +void +nxt_timezone_update(void) +{ + return; +} + +#endif diff --git a/src/nxt_time.h b/src/nxt_time.h new file mode 100644 index 00000000..d6785eb2 --- /dev/null +++ b/src/nxt_time.h @@ -0,0 +1,111 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UNIX_TIME_H_INCLUDED_ +#define _NXT_UNIX_TIME_H_INCLUDED_ + + +typedef uint64_t nxt_nsec_t; +typedef int64_t nxt_nsec_int_t; +#define NXT_INFINITE_NSEC ((nxt_nsec_t) -1) + + +typedef struct { + nxt_time_t sec; + nxt_uint_t nsec; +} nxt_realtime_t; + + +/* + * nxt_monotonic_time_t includes nxt_realtime_t to eliminate + * surplus gettimeofday() call on platform without monotonic time. + */ + +typedef struct { + nxt_realtime_t realtime; + nxt_nsec_t monotonic; + nxt_nsec_t update; + +#if !(NXT_HAVE_CLOCK_MONOTONIC || NXT_SOLARIS || NXT_HPUX || NXT_MACOSX) + nxt_nsec_t previous; +#endif +} nxt_monotonic_time_t; + + +NXT_EXPORT void nxt_realtime(nxt_realtime_t *now); +NXT_EXPORT void nxt_monotonic_time(nxt_monotonic_time_t *now); +NXT_EXPORT void nxt_localtime(nxt_time_t s, struct tm *tm); +NXT_EXPORT void nxt_timezone_update(void); + +/* + * Both localtime() and localtime_r() are not Async-Signal-Safe, therefore, + * they can not be used in signal handlers. Since Daylight Saving Time (DST) + * state changes no more than twice a year, a simple workaround is to use + * a previously cached GMT offset value and nxt_gmtime(): + * + * nxt_gmtime(GMT seconds + GMT offset, tm); + * + * GMT offset with account of current DST state can be obtained only + * using localtime()'s struct tm because: + * + * 1) gettimeofday() does not return GMT offset at almost all platforms. + * MacOSX returns a value cached after the first localtime() call. + * AIX returns GMT offset without account of DST state and indicates + * only that timezone has DST, but does not indicate current DST state. + * + * 2) There are the "timezone" and "daylight" variables on Linux, Solaris, + * HP-UX, IRIX, and other systems. The "daylight" variable indicates + * only that timezone has DST, but does not indicate current DST state. + * + * 3) Solaris and IRIX have the "altzone" variable which contains GMT offset + * for timezone with DST applied, but without account of DST state. + * + * 4) There is the "struct tm.tm_gmtoff" field on BSD systems and modern Linux. + * This field contains GMT offset with account of DST state. + * + * 5) The "struct tm.tm_isdst" field returned by localtime() indicates + * current DST state on all platforms. This field may have three values: + * positive means DST in effect, zero means DST is not in effect, and + * negative means DST state is unknown. + */ + +#if (NXT_HAVE_TM_GMTOFF) + +#define \ +nxt_timezone(tm) \ + ((tm)->tm_gmtoff) + +#elif (NXT_HAVE_ALTZONE) + +#define \ +nxt_timezone(tm) \ + (-(((tm)->tm_isdst > 0) ? altzone : timezone)) + +#else + +#define \ +nxt_timezone(tm) \ + (-(((tm)->tm_isdst > 0) ? timezone + 3600 : timezone)) + +#endif + + +typedef uint32_t nxt_msec_t; +typedef int32_t nxt_msec_int_t; +#define NXT_INFINITE_MSEC ((nxt_msec_t) -1) + + +/* + * Since nxt_msec_t values are stored just in 32 bits, they overflow + * every 49 days. This signed subtraction takes into account that overflow. + * "nxt_msec_diff(m1, m2) < 0" means that m1 is lesser than m2. + */ +#define \ +nxt_msec_diff(m1, m2) \ + ((int32_t) ((m1) - (m2))) + + +#endif /* _NXT_UNIX_TIME_H_INCLUDED_ */ diff --git a/src/nxt_time_parse.c b/src/nxt_time_parse.c new file mode 100644 index 00000000..f79c78ab --- /dev/null +++ b/src/nxt_time_parse.c @@ -0,0 +1,489 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * nxt_time_parse() parses a time string given in RFC822, RFC850, or ISOC + * formats and returns nxt_time_t value >= 0 on success or -1 on failure. + */ + +nxt_time_t +nxt_time_parse(const u_char *p, size_t len) +{ + size_t n; + u_char c; + uint64_t s; + nxt_int_t yr, month, day, hour, min, sec; + nxt_uint_t year, days; + const u_char *end; + + static nxt_int_t mday[12] = { + 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 + }; + + enum { + RFC822 = 0, /* "Mon, 28 Sep 1970 12:00:00" */ + RFC850, /* "Monday, 28-Sep-70 12:00:00" */ + ISOC, /* "Mon Sep 28 12:00:00 1970" */ + } fmt; + + fmt = RFC822; + end = p + len; + + while (p < end) { + c = *p++; + + if (c == ',') { + break; + } + + if (c == ' ') { + fmt = ISOC; + break; + } + } + + while (p < end) { + if (*p != ' ') { + break; + } + + p++; + } + + if (nxt_slow_path(p + 18 > end)) { + /* Lesser than RFC850 "28-Sep-70 12:00:00" length. */ + return -1; + } + + day = 0; + + if (fmt != ISOC) { + day = nxt_int_parse(p, 2); + if (nxt_slow_path(day <= 0)) { + return -1; + } + p += 2; + + if (*p == ' ') { + if (nxt_slow_path(p + 18 > end)) { + /* Lesser than RFC822 " Sep 1970 12:00:00" length. */ + return -1; + } + + /* RFC822 */ + + } else if (*p == '-') { + fmt = RFC850; + + } else { + return -1; + } + + p++; + } + + switch (*p) { + + case 'J': + month = p[1] == 'a' ? 0 : p[2] == 'n' ? 5 : 6; + break; + + case 'F': + month = 1; + break; + + case 'M': + month = p[2] == 'r' ? 2 : 4; + break; + + case 'A': + month = p[1] == 'p' ? 3 : 7; + break; + + case 'S': + month = 8; + break; + + case 'O': + month = 9; + break; + + case 'N': + month = 10; + break; + + case 'D': + month = 11; + break; + + default: + return -1; + } + + p += 3; + yr = 0; + + switch (fmt) { + + case RFC822: + if (nxt_slow_path(*p++ != ' ')) { + return -1; + } + + yr = nxt_int_parse(p, 4); + if (nxt_slow_path(yr <= 0)) { + return -1; + } + p += 4; + + break; + + case RFC850: + if (nxt_slow_path(*p++ != '-')) { + return -1; + } + + yr = nxt_int_parse(p, 2); + if (nxt_slow_path(yr <= 0)) { + return -1; + } + p += 2; + + yr += (yr < 70) ? 2000 : 1900; + + break; + + default: /* ISOC */ + if (nxt_slow_path(*p++ != ' ')) { + return -1; + } + + if (p[0] != ' ') { + n = 2; + + if (p[1] == ' ') { + n = 1; + } + + } else { + p++; + n = 1; + } + + day = nxt_int_parse(p, n); + if (nxt_slow_path(day <= 0)) { + return -1; + } + p += n; + + if (nxt_slow_path(p + 14 > end)) { + /* Lesser than ISOC " 12:00:00 1970" length. */ + return -1; + } + + break; + } + + if (nxt_slow_path(*p++ != ' ')) { + return -1; + } + + hour = nxt_int_parse(p, 2); + if (nxt_slow_path(hour < 0)) { + return -1; + } + p += 2; + + if (nxt_slow_path(*p++ != ':')) { + return -1; + } + + min = nxt_int_parse(p, 2); + if (nxt_slow_path(min < 0)) { + return -1; + } + p += 2; + + if (nxt_slow_path(*p++ != ':')) { + return -1; + } + + sec = nxt_int_parse(p, 2); + if (nxt_slow_path(sec < 0)) { + return -1; + } + + if (fmt == ISOC) { + p += 2; + + if (nxt_slow_path(*p++ != ' ')) { + return -1; + } + + yr = nxt_int_parse(p, 4); + if (nxt_slow_path(yr < 0)) { + return -1; + } + } + + if (nxt_slow_path(hour > 23 || min > 59 || sec > 59)) { + return -1; + } + + year = yr; + + if (day == 29 && month == 1) { + + if (nxt_slow_path((year & 3) != 0)) { + /* Not a leap year. */ + return -1; + } + + if (nxt_slow_path((year % 100 == 0) && (year % 400) != 0)) { + /* Not a leap year. */ + return -1; + } + + } else if (nxt_slow_path(day > mday[(nxt_uint_t) month])) { + return -1; + } + + /* + * Shift new year to March 1 and start months + * from 1 (not 0), as required for Gauss' formula. + */ + + if (--month <= 0) { + month += 12; + year -= 1; + } + + /* Gauss' formula for Gregorian days since March 1, 1 BCE. */ + + /* Days in years including leap years since March 1, 1 BCE. */ + days = 365 * year + year / 4 - year / 100 + year / 400 + + /* Days before the month. */ + + 367 * (nxt_uint_t) month / 12 - 30 + + /* Days before the day. */ + + (nxt_uint_t) day - 1; + + /* + * 719527 days were between March 1, 1 BCE and March 1, 1970, + * 31 and 28 days were in January and February 1970. + */ + days = days - 719527 + 31 + 28; + + s = (uint64_t) days * 86400 + + (nxt_uint_t) hour * 3600 + + (nxt_uint_t) min * 60 + + (nxt_uint_t) sec; + +#if (NXT_TIME_T_SIZE <= 4) + + /* Y2038 */ + + if (nxt_slow_path(s > 0x7fffffff)) { + return -1; + } + +#endif + + return (nxt_time_t) s; +} + + +/* + * nxt_term_parse() parses term string given in format "200", "10m", + * or "1d 1h" and returns nxt_int_t value >= 0 on success, -1 on failure, + * and -2 on overflow. The maximum valid value is 2^31 - 1 or about + * 68 years in seconds or about 24 days in milliseconds. + */ + +nxt_int_t +nxt_term_parse(const u_char *p, size_t len, nxt_bool_t seconds) +{ + u_char c, ch; + nxt_uint_t val, term, scale, max; + const u_char *end; + + enum { + st_first_digit = 0, + st_digit, + st_letter, + st_space, + } state; + + enum { + st_start = 0, + st_year, + st_month, + st_week, + st_day, + st_hour, + st_min, + st_sec, + st_msec, + st_last, + } step; + + val = 0; + term = 0; + state = st_first_digit; + step = seconds ? st_start : st_month; + + end = p + len; + + while (p < end) { + + ch = *p++; + + if (state == st_space) { + + if (ch == ' ') { + continue; + } + + state = st_first_digit; + } + + if (state != st_letter) { + + /* Values below '0' become >= 208. */ + c = ch - '0'; + + if (c <= 9) { + val = val * 10 + c; + state = st_digit; + continue; + } + + if (state == st_first_digit) { + return -1; + } + + state = st_letter; + } + + switch (ch) { + + case 'y': + if (step > st_start) { + return -1; + } + step = st_year; + max = NXT_INT32_T_MAX / (365 * 24 * 60 * 60); + scale = 365 * 24 * 60 * 60; + break; + + case 'M': + if (step >= st_month) { + return -1; + } + step = st_month; + max = NXT_INT32_T_MAX / (30 * 24 * 60 * 60); + scale = 30 * 24 * 60 * 60; + break; + + case 'w': + if (step >= st_week) { + return -1; + } + step = st_week; + max = NXT_INT32_T_MAX / (7 * 24 * 60 * 60); + scale = 7 * 24 * 60 * 60; + break; + + case 'd': + if (step >= st_day) { + return -1; + } + step = st_day; + max = NXT_INT32_T_MAX / (24 * 60 * 60); + scale = 24 * 60 * 60; + break; + + case 'h': + if (step >= st_hour) { + return -1; + } + step = st_hour; + max = NXT_INT32_T_MAX / (60 * 60); + scale = 60 * 60; + break; + + case 'm': + if (p < end && *p == 's') { + if (seconds || step >= st_msec) { + return -1; + } + p++; + step = st_msec; + max = NXT_INT32_T_MAX; + scale = 1; + break; + } + + if (step >= st_min) { + return -1; + } + step = st_min; + max = NXT_INT32_T_MAX / 60; + scale = 60; + break; + + case 's': + if (step >= st_sec) { + return -1; + } + step = st_sec; + max = NXT_INT32_T_MAX; + scale = 1; + break; + + case ' ': + if (step >= st_sec) { + return -1; + } + step = st_last; + max = NXT_INT32_T_MAX; + scale = 1; + break; + + default: + return -1; + } + + if (!seconds && step != st_msec) { + scale *= 1000; + max /= 1000; + } + + if (val > max) { + return -2; + } + + term += val * scale; + + if (term > NXT_INT32_T_MAX) { + return -2; + } + + val = 0; + + state = st_space; + } + + if (!seconds) { + val *= 1000; + } + + return term + val; +} diff --git a/src/nxt_types.h b/src/nxt_types.h new file mode 100644 index 00000000..4c501556 --- /dev/null +++ b/src/nxt_types.h @@ -0,0 +1,151 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_TYPES_H_INCLUDED_ +#define _NXT_TYPES_H_INCLUDED_ + + +/* + * nxt_int_t corresponds to the most efficient integer type, + * an architecture word. It is usually the long type, + * but on Win64 the long is int32_t, so pointer size suits better. + * nxt_int_t must be no less than int32_t. + */ + +#if (__amd64__) +/* + * AMD64 64-bit multiplication and division operations + * are slower and 64-bit instructions are longer. + */ +#define NXT_INT_T_SIZE 4 +typedef int nxt_int_t; +typedef u_int nxt_uint_t; + +#else +#define NXT_INT_T_SIZE NXT_PTR_SIZE +typedef intptr_t nxt_int_t; +typedef uintptr_t nxt_uint_t; +#endif + + +typedef nxt_uint_t nxt_bool_t; + + +/* + * nxt_off_t corresponds to OS's off_t, a file offset type. + * Although Linux, Solaris, and HP-UX define both off_t and off64_t, + * setting _FILE_OFFSET_BITS to 64 defines off_t as off64_t. + */ +typedef off_t nxt_off_t; + + +/* + * nxt_time_t corresponds to OS's time_t, time in seconds. nxt_time_t is + * a signed integer. OS's time_t may be an integer or real-floating type, + * though it is usually a signed 32-bit or 64-bit integer depending on + * platform bit count. There are however exceptions, e.g., time_t is: + * 32-bit on 64-bit NetBSD prior to 6.0 version; + * 64-bit on 32-bit NetBSD 6.0; + * 32-bit on 64-bit OpenBSD; + * 64-bit in Linux x32 ABI; + * 64-bit in 32-bit Visual Studio C++ 2005. + */ +#if (NXT_QNX) +/* + * QNX defines time_t as uint32_t. + * Y2038 fix: "typedef int64_t nxt_time_t". + */ +typedef int32_t nxt_time_t; + +#else +/* Y2038, if time_t is 32-bit integer. */ +typedef time_t nxt_time_t; +#endif + + +#if (NXT_PTR_SIZE == 8) +#define NXT_64BIT 1 +#define NXT_32BIT 0 + +#else +#define NXT_64BIT 0 +#define NXT_32BIT 1 +#endif + + +#define NXT_INT64_T_LEN (sizeof("-9223372036854775808") - 1) +#define NXT_INT32_T_LEN (sizeof("-2147483648") - 1) + +#define NXT_INT64_T_HEXLEN (sizeof("fffffffffffffffff") - 1) +#define NXT_INT32_T_HEXLEN (sizeof("ffffffff") - 1) + +#define NXT_INT64_T_MAX 0x7fffffffffffffffLL +#define NXT_INT32_T_MAX 0x7fffffff + + +#if (NXT_INT_T_SIZE == 8) +#define NXT_INT_T_LEN NXT_INT64_T_LEN +#define NXT_INT_T_HEXLEN NXT_INT64_T_HEXLEN +#define NXT_INT_T_MAX NXT_INT64_T_MAX + +#else +#define NXT_INT_T_LEN NXT_INT32_T_LEN +#define NXT_INT_T_HEXLEN NXT_INT32_T_HEXLEN +#define NXT_INT_T_MAX NXT_INT32_T_MAX +#endif + + +#if (NXT_64BIT) +#define NXT_ATOMIC_T_LEN NXT_INT64_T_LEN +#define NXT_ATOMIC_T_HEXLEN NXT_INT64_T_HEXLEN +#define NXT_ATOMIC_T_MAX NXT_INT64_T_MAX + +#else +#define NXT_ATOMIC_T_LEN NXT_INT32_T_LEN +#define NXT_ATOMIC_T_HEXLEN NXT_INT32_T_HEXLEN +#define NXT_ATOMIC_T_MAX NXT_INT32_T_MAX +#endif + + +#if (NXT_OFF_T_SIZE == 8) +typedef uint64_t nxt_uoff_t; +#define NXT_OFF_T_LEN NXT_INT64_T_LEN +#define NXT_OFF_T_HEXLEN NXT_INT64_T_HEXLEN +#define NXT_OFF_T_MAX NXT_INT64_T_MAX + +#else +typedef uint32_t nxt_uoff_t; +#define NXT_OFF_T_LEN NXT_INT32_T_LEN +#define NXT_OFF_T_HEXLEN NXT_INT32_T_HEXLEN +#define NXT_OFF_T_MAX NXT_INT32_T_MAX +#endif + + +#if (NXT_SIZE_T_SIZE == 8) +#define NXT_SIZE_T_LEN NXT_INT64_T_LEN +#define NXT_SIZE_T_HEXLEN NXT_INT64_T_HEXLEN +#define NXT_SIZE_T_MAX NXT_INT64_T_MAX + +#else +#define NXT_SIZE_T_LEN NXT_INT32_T_LEN +#define NXT_SIZE_T_HEXLEN NXT_INT32_T_HEXLEN +#define NXT_SIZE_T_MAX NXT_INT32_T_MAX +#endif + + +#if (NXT_TIME_T_SIZE == 8) +#define NXT_TIME_T_LEN NXT_INT64_T_LEN +#define NXT_TIME_T_HEXLEN NXT_INT64_T_HEXLEN +#define NXT_TIME_T_MAX NXT_INT64_T_MAX + +#else +#define NXT_TIME_T_LEN NXT_INT32_T_LEN +#define NXT_TIME_T_HEXLEN NXT_INT32_T_HEXLEN +#define NXT_TIME_T_MAX NXT_INT32_T_MAX +#endif + + +#endif /* _NXT_TYPES_H_INCLUDED_ */ diff --git a/src/nxt_unicode_lowcase.h b/src/nxt_unicode_lowcase.h new file mode 100644 index 00000000..c868ad19 --- /dev/null +++ b/src/nxt_unicode_lowcase.h @@ -0,0 +1,1043 @@ + +/* + * 26 128-bytes blocks, 521 pointers. + * 14920 bytes on 32-bit platforms, 17000 bytes on 64-bit platforms. + */ + +#define NXT_UNICODE_MAX_LOWCASE 0x10427 + +#define NXT_UNICODE_BLOCK_SIZE 128 + + +static const uint32_t nxt_unicode_block_000[128] nxt_aligned(64) = { + 0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007, + 0x00008, 0x00009, 0x0000a, 0x0000b, 0x0000c, 0x0000d, 0x0000e, 0x0000f, + 0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017, + 0x00018, 0x00019, 0x0001a, 0x0001b, 0x0001c, 0x0001d, 0x0001e, 0x0001f, + 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, + 0x00028, 0x00029, 0x0002a, 0x0002b, 0x0002c, 0x0002d, 0x0002e, 0x0002f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0003a, 0x0003b, 0x0003c, 0x0003d, 0x0003e, 0x0003f, + 0x00040, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, + 0x00068, 0x00069, 0x0006a, 0x0006b, 0x0006c, 0x0006d, 0x0006e, 0x0006f, + 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, + 0x00078, 0x00079, 0x0007a, 0x0005b, 0x0005c, 0x0005d, 0x0005e, 0x0005f, + 0x00060, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, + 0x00068, 0x00069, 0x0006a, 0x0006b, 0x0006c, 0x0006d, 0x0006e, 0x0006f, + 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, + 0x00078, 0x00079, 0x0007a, 0x0007b, 0x0007c, 0x0007d, 0x0007e, 0x0007f, +}; + + +static const uint32_t nxt_unicode_block_001[128] nxt_aligned(64) = { + 0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087, + 0x00088, 0x00089, 0x0008a, 0x0008b, 0x0008c, 0x0008d, 0x0008e, 0x0008f, + 0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097, + 0x00098, 0x00099, 0x0009a, 0x0009b, 0x0009c, 0x0009d, 0x0009e, 0x0009f, + 0x000a0, 0x000a1, 0x000a2, 0x000a3, 0x000a4, 0x000a5, 0x000a6, 0x000a7, + 0x000a8, 0x000a9, 0x000aa, 0x000ab, 0x000ac, 0x000ad, 0x000ae, 0x000af, + 0x000b0, 0x000b1, 0x000b2, 0x000b3, 0x000b4, 0x003bc, 0x000b6, 0x000b7, + 0x000b8, 0x000b9, 0x000ba, 0x000bb, 0x000bc, 0x000bd, 0x000be, 0x000bf, + 0x000e0, 0x000e1, 0x000e2, 0x000e3, 0x000e4, 0x000e5, 0x000e6, 0x000e7, + 0x000e8, 0x000e9, 0x000ea, 0x000eb, 0x000ec, 0x000ed, 0x000ee, 0x000ef, + 0x000f0, 0x000f1, 0x000f2, 0x000f3, 0x000f4, 0x000f5, 0x000f6, 0x000d7, + 0x000f8, 0x000f9, 0x000fa, 0x000fb, 0x000fc, 0x000fd, 0x000fe, 0x000df, + 0x000e0, 0x000e1, 0x000e2, 0x000e3, 0x000e4, 0x000e5, 0x000e6, 0x000e7, + 0x000e8, 0x000e9, 0x000ea, 0x000eb, 0x000ec, 0x000ed, 0x000ee, 0x000ef, + 0x000f0, 0x000f1, 0x000f2, 0x000f3, 0x000f4, 0x000f5, 0x000f6, 0x000f7, + 0x000f8, 0x000f9, 0x000fa, 0x000fb, 0x000fc, 0x000fd, 0x000fe, 0x000ff, +}; + + +static const uint32_t nxt_unicode_block_002[128] nxt_aligned(64) = { + 0x00101, 0x00101, 0x00103, 0x00103, 0x00105, 0x00105, 0x00107, 0x00107, + 0x00109, 0x00109, 0x0010b, 0x0010b, 0x0010d, 0x0010d, 0x0010f, 0x0010f, + 0x00111, 0x00111, 0x00113, 0x00113, 0x00115, 0x00115, 0x00117, 0x00117, + 0x00119, 0x00119, 0x0011b, 0x0011b, 0x0011d, 0x0011d, 0x0011f, 0x0011f, + 0x00121, 0x00121, 0x00123, 0x00123, 0x00125, 0x00125, 0x00127, 0x00127, + 0x00129, 0x00129, 0x0012b, 0x0012b, 0x0012d, 0x0012d, 0x0012f, 0x0012f, + 0x00130, 0x00131, 0x00133, 0x00133, 0x00135, 0x00135, 0x00137, 0x00137, + 0x00138, 0x0013a, 0x0013a, 0x0013c, 0x0013c, 0x0013e, 0x0013e, 0x00140, + 0x00140, 0x00142, 0x00142, 0x00144, 0x00144, 0x00146, 0x00146, 0x00148, + 0x00148, 0x00149, 0x0014b, 0x0014b, 0x0014d, 0x0014d, 0x0014f, 0x0014f, + 0x00151, 0x00151, 0x00153, 0x00153, 0x00155, 0x00155, 0x00157, 0x00157, + 0x00159, 0x00159, 0x0015b, 0x0015b, 0x0015d, 0x0015d, 0x0015f, 0x0015f, + 0x00161, 0x00161, 0x00163, 0x00163, 0x00165, 0x00165, 0x00167, 0x00167, + 0x00169, 0x00169, 0x0016b, 0x0016b, 0x0016d, 0x0016d, 0x0016f, 0x0016f, + 0x00171, 0x00171, 0x00173, 0x00173, 0x00175, 0x00175, 0x00177, 0x00177, + 0x000ff, 0x0017a, 0x0017a, 0x0017c, 0x0017c, 0x0017e, 0x0017e, 0x00073, +}; + + +static const uint32_t nxt_unicode_block_003[128] nxt_aligned(64) = { + 0x00180, 0x00253, 0x00183, 0x00183, 0x00185, 0x00185, 0x00254, 0x00188, + 0x00188, 0x00256, 0x00257, 0x0018c, 0x0018c, 0x0018d, 0x001dd, 0x00259, + 0x0025b, 0x00192, 0x00192, 0x00260, 0x00263, 0x00195, 0x00269, 0x00268, + 0x00199, 0x00199, 0x0019a, 0x0019b, 0x0026f, 0x00272, 0x0019e, 0x00275, + 0x001a1, 0x001a1, 0x001a3, 0x001a3, 0x001a5, 0x001a5, 0x00280, 0x001a8, + 0x001a8, 0x00283, 0x001aa, 0x001ab, 0x001ad, 0x001ad, 0x00288, 0x001b0, + 0x001b0, 0x0028a, 0x0028b, 0x001b4, 0x001b4, 0x001b6, 0x001b6, 0x00292, + 0x001b9, 0x001b9, 0x001ba, 0x001bb, 0x001bd, 0x001bd, 0x001be, 0x001bf, + 0x001c0, 0x001c1, 0x001c2, 0x001c3, 0x001c6, 0x001c6, 0x001c6, 0x001c9, + 0x001c9, 0x001c9, 0x001cc, 0x001cc, 0x001cc, 0x001ce, 0x001ce, 0x001d0, + 0x001d0, 0x001d2, 0x001d2, 0x001d4, 0x001d4, 0x001d6, 0x001d6, 0x001d8, + 0x001d8, 0x001da, 0x001da, 0x001dc, 0x001dc, 0x001dd, 0x001df, 0x001df, + 0x001e1, 0x001e1, 0x001e3, 0x001e3, 0x001e5, 0x001e5, 0x001e7, 0x001e7, + 0x001e9, 0x001e9, 0x001eb, 0x001eb, 0x001ed, 0x001ed, 0x001ef, 0x001ef, + 0x001f0, 0x001f3, 0x001f3, 0x001f3, 0x001f5, 0x001f5, 0x00195, 0x001bf, + 0x001f9, 0x001f9, 0x001fb, 0x001fb, 0x001fd, 0x001fd, 0x001ff, 0x001ff, +}; + + +static const uint32_t nxt_unicode_block_004[128] nxt_aligned(64) = { + 0x00201, 0x00201, 0x00203, 0x00203, 0x00205, 0x00205, 0x00207, 0x00207, + 0x00209, 0x00209, 0x0020b, 0x0020b, 0x0020d, 0x0020d, 0x0020f, 0x0020f, + 0x00211, 0x00211, 0x00213, 0x00213, 0x00215, 0x00215, 0x00217, 0x00217, + 0x00219, 0x00219, 0x0021b, 0x0021b, 0x0021d, 0x0021d, 0x0021f, 0x0021f, + 0x0019e, 0x00221, 0x00223, 0x00223, 0x00225, 0x00225, 0x00227, 0x00227, + 0x00229, 0x00229, 0x0022b, 0x0022b, 0x0022d, 0x0022d, 0x0022f, 0x0022f, + 0x00231, 0x00231, 0x00233, 0x00233, 0x00234, 0x00235, 0x00236, 0x00237, + 0x00238, 0x00239, 0x02c65, 0x0023c, 0x0023c, 0x0019a, 0x02c66, 0x0023f, + 0x00240, 0x00242, 0x00242, 0x00180, 0x00289, 0x0028c, 0x00247, 0x00247, + 0x00249, 0x00249, 0x0024b, 0x0024b, 0x0024d, 0x0024d, 0x0024f, 0x0024f, + 0x00250, 0x00251, 0x00252, 0x00253, 0x00254, 0x00255, 0x00256, 0x00257, + 0x00258, 0x00259, 0x0025a, 0x0025b, 0x0025c, 0x0025d, 0x0025e, 0x0025f, + 0x00260, 0x00261, 0x00262, 0x00263, 0x00264, 0x00265, 0x00266, 0x00267, + 0x00268, 0x00269, 0x0026a, 0x0026b, 0x0026c, 0x0026d, 0x0026e, 0x0026f, + 0x00270, 0x00271, 0x00272, 0x00273, 0x00274, 0x00275, 0x00276, 0x00277, + 0x00278, 0x00279, 0x0027a, 0x0027b, 0x0027c, 0x0027d, 0x0027e, 0x0027f, +}; + + +static const uint32_t nxt_unicode_block_006[128] nxt_aligned(64) = { + 0x00300, 0x00301, 0x00302, 0x00303, 0x00304, 0x00305, 0x00306, 0x00307, + 0x00308, 0x00309, 0x0030a, 0x0030b, 0x0030c, 0x0030d, 0x0030e, 0x0030f, + 0x00310, 0x00311, 0x00312, 0x00313, 0x00314, 0x00315, 0x00316, 0x00317, + 0x00318, 0x00319, 0x0031a, 0x0031b, 0x0031c, 0x0031d, 0x0031e, 0x0031f, + 0x00320, 0x00321, 0x00322, 0x00323, 0x00324, 0x00325, 0x00326, 0x00327, + 0x00328, 0x00329, 0x0032a, 0x0032b, 0x0032c, 0x0032d, 0x0032e, 0x0032f, + 0x00330, 0x00331, 0x00332, 0x00333, 0x00334, 0x00335, 0x00336, 0x00337, + 0x00338, 0x00339, 0x0033a, 0x0033b, 0x0033c, 0x0033d, 0x0033e, 0x0033f, + 0x00340, 0x00341, 0x00342, 0x00343, 0x00344, 0x003b9, 0x00346, 0x00347, + 0x00348, 0x00349, 0x0034a, 0x0034b, 0x0034c, 0x0034d, 0x0034e, 0x0034f, + 0x00350, 0x00351, 0x00352, 0x00353, 0x00354, 0x00355, 0x00356, 0x00357, + 0x00358, 0x00359, 0x0035a, 0x0035b, 0x0035c, 0x0035d, 0x0035e, 0x0035f, + 0x00360, 0x00361, 0x00362, 0x00363, 0x00364, 0x00365, 0x00366, 0x00367, + 0x00368, 0x00369, 0x0036a, 0x0036b, 0x0036c, 0x0036d, 0x0036e, 0x0036f, + 0x00371, 0x00371, 0x00373, 0x00373, 0x00374, 0x00375, 0x00377, 0x00377, + 0x00378, 0x00379, 0x0037a, 0x0037b, 0x0037c, 0x0037d, 0x0037e, 0x0037f, +}; + + +static const uint32_t nxt_unicode_block_007[128] nxt_aligned(64) = { + 0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x003ac, 0x00387, + 0x003ad, 0x003ae, 0x003af, 0x0038b, 0x003cc, 0x0038d, 0x003cd, 0x003ce, + 0x00390, 0x003b1, 0x003b2, 0x003b3, 0x003b4, 0x003b5, 0x003b6, 0x003b7, + 0x003b8, 0x003b9, 0x003ba, 0x003bb, 0x003bc, 0x003bd, 0x003be, 0x003bf, + 0x003c0, 0x003c1, 0x003a2, 0x003c3, 0x003c4, 0x003c5, 0x003c6, 0x003c7, + 0x003c8, 0x003c9, 0x003ca, 0x003cb, 0x003ac, 0x003ad, 0x003ae, 0x003af, + 0x003b0, 0x003b1, 0x003b2, 0x003b3, 0x003b4, 0x003b5, 0x003b6, 0x003b7, + 0x003b8, 0x003b9, 0x003ba, 0x003bb, 0x003bc, 0x003bd, 0x003be, 0x003bf, + 0x003c0, 0x003c1, 0x003c3, 0x003c3, 0x003c4, 0x003c5, 0x003c6, 0x003c7, + 0x003c8, 0x003c9, 0x003ca, 0x003cb, 0x003cc, 0x003cd, 0x003ce, 0x003d7, + 0x003b2, 0x003b8, 0x003d2, 0x003d3, 0x003d4, 0x003c6, 0x003c0, 0x003d7, + 0x003d9, 0x003d9, 0x003db, 0x003db, 0x003dd, 0x003dd, 0x003df, 0x003df, + 0x003e1, 0x003e1, 0x003e3, 0x003e3, 0x003e5, 0x003e5, 0x003e7, 0x003e7, + 0x003e9, 0x003e9, 0x003eb, 0x003eb, 0x003ed, 0x003ed, 0x003ef, 0x003ef, + 0x003ba, 0x003c1, 0x003f2, 0x003f3, 0x003b8, 0x003b5, 0x003f6, 0x003f8, + 0x003f8, 0x003f2, 0x003fb, 0x003fb, 0x003fc, 0x0037b, 0x0037c, 0x0037d, +}; + + +static const uint32_t nxt_unicode_block_008[128] nxt_aligned(64) = { + 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, + 0x00458, 0x00459, 0x0045a, 0x0045b, 0x0045c, 0x0045d, 0x0045e, 0x0045f, + 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, + 0x00438, 0x00439, 0x0043a, 0x0043b, 0x0043c, 0x0043d, 0x0043e, 0x0043f, + 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, + 0x00448, 0x00449, 0x0044a, 0x0044b, 0x0044c, 0x0044d, 0x0044e, 0x0044f, + 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, + 0x00438, 0x00439, 0x0043a, 0x0043b, 0x0043c, 0x0043d, 0x0043e, 0x0043f, + 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, + 0x00448, 0x00449, 0x0044a, 0x0044b, 0x0044c, 0x0044d, 0x0044e, 0x0044f, + 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, + 0x00458, 0x00459, 0x0045a, 0x0045b, 0x0045c, 0x0045d, 0x0045e, 0x0045f, + 0x00461, 0x00461, 0x00463, 0x00463, 0x00465, 0x00465, 0x00467, 0x00467, + 0x00469, 0x00469, 0x0046b, 0x0046b, 0x0046d, 0x0046d, 0x0046f, 0x0046f, + 0x00471, 0x00471, 0x00473, 0x00473, 0x00475, 0x00475, 0x00477, 0x00477, + 0x00479, 0x00479, 0x0047b, 0x0047b, 0x0047d, 0x0047d, 0x0047f, 0x0047f, +}; + + +static const uint32_t nxt_unicode_block_009[128] nxt_aligned(64) = { + 0x00481, 0x00481, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487, + 0x00488, 0x00489, 0x0048b, 0x0048b, 0x0048d, 0x0048d, 0x0048f, 0x0048f, + 0x00491, 0x00491, 0x00493, 0x00493, 0x00495, 0x00495, 0x00497, 0x00497, + 0x00499, 0x00499, 0x0049b, 0x0049b, 0x0049d, 0x0049d, 0x0049f, 0x0049f, + 0x004a1, 0x004a1, 0x004a3, 0x004a3, 0x004a5, 0x004a5, 0x004a7, 0x004a7, + 0x004a9, 0x004a9, 0x004ab, 0x004ab, 0x004ad, 0x004ad, 0x004af, 0x004af, + 0x004b1, 0x004b1, 0x004b3, 0x004b3, 0x004b5, 0x004b5, 0x004b7, 0x004b7, + 0x004b9, 0x004b9, 0x004bb, 0x004bb, 0x004bd, 0x004bd, 0x004bf, 0x004bf, + 0x004cf, 0x004c2, 0x004c2, 0x004c4, 0x004c4, 0x004c6, 0x004c6, 0x004c8, + 0x004c8, 0x004ca, 0x004ca, 0x004cc, 0x004cc, 0x004ce, 0x004ce, 0x004cf, + 0x004d1, 0x004d1, 0x004d3, 0x004d3, 0x004d5, 0x004d5, 0x004d7, 0x004d7, + 0x004d9, 0x004d9, 0x004db, 0x004db, 0x004dd, 0x004dd, 0x004df, 0x004df, + 0x004e1, 0x004e1, 0x004e3, 0x004e3, 0x004e5, 0x004e5, 0x004e7, 0x004e7, + 0x004e9, 0x004e9, 0x004eb, 0x004eb, 0x004ed, 0x004ed, 0x004ef, 0x004ef, + 0x004f1, 0x004f1, 0x004f3, 0x004f3, 0x004f5, 0x004f5, 0x004f7, 0x004f7, + 0x004f9, 0x004f9, 0x004fb, 0x004fb, 0x004fd, 0x004fd, 0x004ff, 0x004ff, +}; + + +static const uint32_t nxt_unicode_block_00a[128] nxt_aligned(64) = { + 0x00501, 0x00501, 0x00503, 0x00503, 0x00505, 0x00505, 0x00507, 0x00507, + 0x00509, 0x00509, 0x0050b, 0x0050b, 0x0050d, 0x0050d, 0x0050f, 0x0050f, + 0x00511, 0x00511, 0x00513, 0x00513, 0x00515, 0x00515, 0x00517, 0x00517, + 0x00519, 0x00519, 0x0051b, 0x0051b, 0x0051d, 0x0051d, 0x0051f, 0x0051f, + 0x00521, 0x00521, 0x00523, 0x00523, 0x00525, 0x00525, 0x00527, 0x00527, + 0x00528, 0x00529, 0x0052a, 0x0052b, 0x0052c, 0x0052d, 0x0052e, 0x0052f, + 0x00530, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, + 0x00568, 0x00569, 0x0056a, 0x0056b, 0x0056c, 0x0056d, 0x0056e, 0x0056f, + 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, + 0x00578, 0x00579, 0x0057a, 0x0057b, 0x0057c, 0x0057d, 0x0057e, 0x0057f, + 0x00580, 0x00581, 0x00582, 0x00583, 0x00584, 0x00585, 0x00586, 0x00557, + 0x00558, 0x00559, 0x0055a, 0x0055b, 0x0055c, 0x0055d, 0x0055e, 0x0055f, + 0x00560, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, + 0x00568, 0x00569, 0x0056a, 0x0056b, 0x0056c, 0x0056d, 0x0056e, 0x0056f, + 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, + 0x00578, 0x00579, 0x0057a, 0x0057b, 0x0057c, 0x0057d, 0x0057e, 0x0057f, +}; + + +static const uint32_t nxt_unicode_block_021[128] nxt_aligned(64) = { + 0x01080, 0x01081, 0x01082, 0x01083, 0x01084, 0x01085, 0x01086, 0x01087, + 0x01088, 0x01089, 0x0108a, 0x0108b, 0x0108c, 0x0108d, 0x0108e, 0x0108f, + 0x01090, 0x01091, 0x01092, 0x01093, 0x01094, 0x01095, 0x01096, 0x01097, + 0x01098, 0x01099, 0x0109a, 0x0109b, 0x0109c, 0x0109d, 0x0109e, 0x0109f, + 0x02d00, 0x02d01, 0x02d02, 0x02d03, 0x02d04, 0x02d05, 0x02d06, 0x02d07, + 0x02d08, 0x02d09, 0x02d0a, 0x02d0b, 0x02d0c, 0x02d0d, 0x02d0e, 0x02d0f, + 0x02d10, 0x02d11, 0x02d12, 0x02d13, 0x02d14, 0x02d15, 0x02d16, 0x02d17, + 0x02d18, 0x02d19, 0x02d1a, 0x02d1b, 0x02d1c, 0x02d1d, 0x02d1e, 0x02d1f, + 0x02d20, 0x02d21, 0x02d22, 0x02d23, 0x02d24, 0x02d25, 0x010c6, 0x02d27, + 0x010c8, 0x010c9, 0x010ca, 0x010cb, 0x010cc, 0x02d2d, 0x010ce, 0x010cf, + 0x010d0, 0x010d1, 0x010d2, 0x010d3, 0x010d4, 0x010d5, 0x010d6, 0x010d7, + 0x010d8, 0x010d9, 0x010da, 0x010db, 0x010dc, 0x010dd, 0x010de, 0x010df, + 0x010e0, 0x010e1, 0x010e2, 0x010e3, 0x010e4, 0x010e5, 0x010e6, 0x010e7, + 0x010e8, 0x010e9, 0x010ea, 0x010eb, 0x010ec, 0x010ed, 0x010ee, 0x010ef, + 0x010f0, 0x010f1, 0x010f2, 0x010f3, 0x010f4, 0x010f5, 0x010f6, 0x010f7, + 0x010f8, 0x010f9, 0x010fa, 0x010fb, 0x010fc, 0x010fd, 0x010fe, 0x010ff, +}; + + +static const uint32_t nxt_unicode_block_03c[128] nxt_aligned(64) = { + 0x01e01, 0x01e01, 0x01e03, 0x01e03, 0x01e05, 0x01e05, 0x01e07, 0x01e07, + 0x01e09, 0x01e09, 0x01e0b, 0x01e0b, 0x01e0d, 0x01e0d, 0x01e0f, 0x01e0f, + 0x01e11, 0x01e11, 0x01e13, 0x01e13, 0x01e15, 0x01e15, 0x01e17, 0x01e17, + 0x01e19, 0x01e19, 0x01e1b, 0x01e1b, 0x01e1d, 0x01e1d, 0x01e1f, 0x01e1f, + 0x01e21, 0x01e21, 0x01e23, 0x01e23, 0x01e25, 0x01e25, 0x01e27, 0x01e27, + 0x01e29, 0x01e29, 0x01e2b, 0x01e2b, 0x01e2d, 0x01e2d, 0x01e2f, 0x01e2f, + 0x01e31, 0x01e31, 0x01e33, 0x01e33, 0x01e35, 0x01e35, 0x01e37, 0x01e37, + 0x01e39, 0x01e39, 0x01e3b, 0x01e3b, 0x01e3d, 0x01e3d, 0x01e3f, 0x01e3f, + 0x01e41, 0x01e41, 0x01e43, 0x01e43, 0x01e45, 0x01e45, 0x01e47, 0x01e47, + 0x01e49, 0x01e49, 0x01e4b, 0x01e4b, 0x01e4d, 0x01e4d, 0x01e4f, 0x01e4f, + 0x01e51, 0x01e51, 0x01e53, 0x01e53, 0x01e55, 0x01e55, 0x01e57, 0x01e57, + 0x01e59, 0x01e59, 0x01e5b, 0x01e5b, 0x01e5d, 0x01e5d, 0x01e5f, 0x01e5f, + 0x01e61, 0x01e61, 0x01e63, 0x01e63, 0x01e65, 0x01e65, 0x01e67, 0x01e67, + 0x01e69, 0x01e69, 0x01e6b, 0x01e6b, 0x01e6d, 0x01e6d, 0x01e6f, 0x01e6f, + 0x01e71, 0x01e71, 0x01e73, 0x01e73, 0x01e75, 0x01e75, 0x01e77, 0x01e77, + 0x01e79, 0x01e79, 0x01e7b, 0x01e7b, 0x01e7d, 0x01e7d, 0x01e7f, 0x01e7f, +}; + + +static const uint32_t nxt_unicode_block_03d[128] nxt_aligned(64) = { + 0x01e81, 0x01e81, 0x01e83, 0x01e83, 0x01e85, 0x01e85, 0x01e87, 0x01e87, + 0x01e89, 0x01e89, 0x01e8b, 0x01e8b, 0x01e8d, 0x01e8d, 0x01e8f, 0x01e8f, + 0x01e91, 0x01e91, 0x01e93, 0x01e93, 0x01e95, 0x01e95, 0x01e96, 0x01e97, + 0x01e98, 0x01e99, 0x01e9a, 0x01e61, 0x01e9c, 0x01e9d, 0x000df, 0x01e9f, + 0x01ea1, 0x01ea1, 0x01ea3, 0x01ea3, 0x01ea5, 0x01ea5, 0x01ea7, 0x01ea7, + 0x01ea9, 0x01ea9, 0x01eab, 0x01eab, 0x01ead, 0x01ead, 0x01eaf, 0x01eaf, + 0x01eb1, 0x01eb1, 0x01eb3, 0x01eb3, 0x01eb5, 0x01eb5, 0x01eb7, 0x01eb7, + 0x01eb9, 0x01eb9, 0x01ebb, 0x01ebb, 0x01ebd, 0x01ebd, 0x01ebf, 0x01ebf, + 0x01ec1, 0x01ec1, 0x01ec3, 0x01ec3, 0x01ec5, 0x01ec5, 0x01ec7, 0x01ec7, + 0x01ec9, 0x01ec9, 0x01ecb, 0x01ecb, 0x01ecd, 0x01ecd, 0x01ecf, 0x01ecf, + 0x01ed1, 0x01ed1, 0x01ed3, 0x01ed3, 0x01ed5, 0x01ed5, 0x01ed7, 0x01ed7, + 0x01ed9, 0x01ed9, 0x01edb, 0x01edb, 0x01edd, 0x01edd, 0x01edf, 0x01edf, + 0x01ee1, 0x01ee1, 0x01ee3, 0x01ee3, 0x01ee5, 0x01ee5, 0x01ee7, 0x01ee7, + 0x01ee9, 0x01ee9, 0x01eeb, 0x01eeb, 0x01eed, 0x01eed, 0x01eef, 0x01eef, + 0x01ef1, 0x01ef1, 0x01ef3, 0x01ef3, 0x01ef5, 0x01ef5, 0x01ef7, 0x01ef7, + 0x01ef9, 0x01ef9, 0x01efb, 0x01efb, 0x01efd, 0x01efd, 0x01eff, 0x01eff, +}; + + +static const uint32_t nxt_unicode_block_03e[128] nxt_aligned(64) = { + 0x01f00, 0x01f01, 0x01f02, 0x01f03, 0x01f04, 0x01f05, 0x01f06, 0x01f07, + 0x01f00, 0x01f01, 0x01f02, 0x01f03, 0x01f04, 0x01f05, 0x01f06, 0x01f07, + 0x01f10, 0x01f11, 0x01f12, 0x01f13, 0x01f14, 0x01f15, 0x01f16, 0x01f17, + 0x01f10, 0x01f11, 0x01f12, 0x01f13, 0x01f14, 0x01f15, 0x01f1e, 0x01f1f, + 0x01f20, 0x01f21, 0x01f22, 0x01f23, 0x01f24, 0x01f25, 0x01f26, 0x01f27, + 0x01f20, 0x01f21, 0x01f22, 0x01f23, 0x01f24, 0x01f25, 0x01f26, 0x01f27, + 0x01f30, 0x01f31, 0x01f32, 0x01f33, 0x01f34, 0x01f35, 0x01f36, 0x01f37, + 0x01f30, 0x01f31, 0x01f32, 0x01f33, 0x01f34, 0x01f35, 0x01f36, 0x01f37, + 0x01f40, 0x01f41, 0x01f42, 0x01f43, 0x01f44, 0x01f45, 0x01f46, 0x01f47, + 0x01f40, 0x01f41, 0x01f42, 0x01f43, 0x01f44, 0x01f45, 0x01f4e, 0x01f4f, + 0x01f50, 0x01f51, 0x01f52, 0x01f53, 0x01f54, 0x01f55, 0x01f56, 0x01f57, + 0x01f58, 0x01f51, 0x01f5a, 0x01f53, 0x01f5c, 0x01f55, 0x01f5e, 0x01f57, + 0x01f60, 0x01f61, 0x01f62, 0x01f63, 0x01f64, 0x01f65, 0x01f66, 0x01f67, + 0x01f60, 0x01f61, 0x01f62, 0x01f63, 0x01f64, 0x01f65, 0x01f66, 0x01f67, + 0x01f70, 0x01f71, 0x01f72, 0x01f73, 0x01f74, 0x01f75, 0x01f76, 0x01f77, + 0x01f78, 0x01f79, 0x01f7a, 0x01f7b, 0x01f7c, 0x01f7d, 0x01f7e, 0x01f7f, +}; + + +static const uint32_t nxt_unicode_block_03f[128] nxt_aligned(64) = { + 0x01f80, 0x01f81, 0x01f82, 0x01f83, 0x01f84, 0x01f85, 0x01f86, 0x01f87, + 0x01f80, 0x01f81, 0x01f82, 0x01f83, 0x01f84, 0x01f85, 0x01f86, 0x01f87, + 0x01f90, 0x01f91, 0x01f92, 0x01f93, 0x01f94, 0x01f95, 0x01f96, 0x01f97, + 0x01f90, 0x01f91, 0x01f92, 0x01f93, 0x01f94, 0x01f95, 0x01f96, 0x01f97, + 0x01fa0, 0x01fa1, 0x01fa2, 0x01fa3, 0x01fa4, 0x01fa5, 0x01fa6, 0x01fa7, + 0x01fa0, 0x01fa1, 0x01fa2, 0x01fa3, 0x01fa4, 0x01fa5, 0x01fa6, 0x01fa7, + 0x01fb0, 0x01fb1, 0x01fb2, 0x01fb3, 0x01fb4, 0x01fb5, 0x01fb6, 0x01fb7, + 0x01fb0, 0x01fb1, 0x01f70, 0x01f71, 0x01fb3, 0x01fbd, 0x003b9, 0x01fbf, + 0x01fc0, 0x01fc1, 0x01fc2, 0x01fc3, 0x01fc4, 0x01fc5, 0x01fc6, 0x01fc7, + 0x01f72, 0x01f73, 0x01f74, 0x01f75, 0x01fc3, 0x01fcd, 0x01fce, 0x01fcf, + 0x01fd0, 0x01fd1, 0x01fd2, 0x01fd3, 0x01fd4, 0x01fd5, 0x01fd6, 0x01fd7, + 0x01fd0, 0x01fd1, 0x01f76, 0x01f77, 0x01fdc, 0x01fdd, 0x01fde, 0x01fdf, + 0x01fe0, 0x01fe1, 0x01fe2, 0x01fe3, 0x01fe4, 0x01fe5, 0x01fe6, 0x01fe7, + 0x01fe0, 0x01fe1, 0x01f7a, 0x01f7b, 0x01fe5, 0x01fed, 0x01fee, 0x01fef, + 0x01ff0, 0x01ff1, 0x01ff2, 0x01ff3, 0x01ff4, 0x01ff5, 0x01ff6, 0x01ff7, + 0x01f78, 0x01f79, 0x01f7c, 0x01f7d, 0x01ff3, 0x01ffd, 0x01ffe, 0x01fff, +}; + + +static const uint32_t nxt_unicode_block_042[128] nxt_aligned(64) = { + 0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107, + 0x02108, 0x02109, 0x0210a, 0x0210b, 0x0210c, 0x0210d, 0x0210e, 0x0210f, + 0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117, + 0x02118, 0x02119, 0x0211a, 0x0211b, 0x0211c, 0x0211d, 0x0211e, 0x0211f, + 0x02120, 0x02121, 0x02122, 0x02123, 0x02124, 0x02125, 0x003c9, 0x02127, + 0x02128, 0x02129, 0x0006b, 0x000e5, 0x0212c, 0x0212d, 0x0212e, 0x0212f, + 0x02130, 0x02131, 0x0214e, 0x02133, 0x02134, 0x02135, 0x02136, 0x02137, + 0x02138, 0x02139, 0x0213a, 0x0213b, 0x0213c, 0x0213d, 0x0213e, 0x0213f, + 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x02145, 0x02146, 0x02147, + 0x02148, 0x02149, 0x0214a, 0x0214b, 0x0214c, 0x0214d, 0x0214e, 0x0214f, + 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, + 0x02158, 0x02159, 0x0215a, 0x0215b, 0x0215c, 0x0215d, 0x0215e, 0x0215f, + 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, + 0x02178, 0x02179, 0x0217a, 0x0217b, 0x0217c, 0x0217d, 0x0217e, 0x0217f, + 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, + 0x02178, 0x02179, 0x0217a, 0x0217b, 0x0217c, 0x0217d, 0x0217e, 0x0217f, +}; + + +static const uint32_t nxt_unicode_block_043[128] nxt_aligned(64) = { + 0x02180, 0x02181, 0x02182, 0x02184, 0x02184, 0x02185, 0x02186, 0x02187, + 0x02188, 0x02189, 0x0218a, 0x0218b, 0x0218c, 0x0218d, 0x0218e, 0x0218f, + 0x02190, 0x02191, 0x02192, 0x02193, 0x02194, 0x02195, 0x02196, 0x02197, + 0x02198, 0x02199, 0x0219a, 0x0219b, 0x0219c, 0x0219d, 0x0219e, 0x0219f, + 0x021a0, 0x021a1, 0x021a2, 0x021a3, 0x021a4, 0x021a5, 0x021a6, 0x021a7, + 0x021a8, 0x021a9, 0x021aa, 0x021ab, 0x021ac, 0x021ad, 0x021ae, 0x021af, + 0x021b0, 0x021b1, 0x021b2, 0x021b3, 0x021b4, 0x021b5, 0x021b6, 0x021b7, + 0x021b8, 0x021b9, 0x021ba, 0x021bb, 0x021bc, 0x021bd, 0x021be, 0x021bf, + 0x021c0, 0x021c1, 0x021c2, 0x021c3, 0x021c4, 0x021c5, 0x021c6, 0x021c7, + 0x021c8, 0x021c9, 0x021ca, 0x021cb, 0x021cc, 0x021cd, 0x021ce, 0x021cf, + 0x021d0, 0x021d1, 0x021d2, 0x021d3, 0x021d4, 0x021d5, 0x021d6, 0x021d7, + 0x021d8, 0x021d9, 0x021da, 0x021db, 0x021dc, 0x021dd, 0x021de, 0x021df, + 0x021e0, 0x021e1, 0x021e2, 0x021e3, 0x021e4, 0x021e5, 0x021e6, 0x021e7, + 0x021e8, 0x021e9, 0x021ea, 0x021eb, 0x021ec, 0x021ed, 0x021ee, 0x021ef, + 0x021f0, 0x021f1, 0x021f2, 0x021f3, 0x021f4, 0x021f5, 0x021f6, 0x021f7, + 0x021f8, 0x021f9, 0x021fa, 0x021fb, 0x021fc, 0x021fd, 0x021fe, 0x021ff, +}; + + +static const uint32_t nxt_unicode_block_049[128] nxt_aligned(64) = { + 0x02480, 0x02481, 0x02482, 0x02483, 0x02484, 0x02485, 0x02486, 0x02487, + 0x02488, 0x02489, 0x0248a, 0x0248b, 0x0248c, 0x0248d, 0x0248e, 0x0248f, + 0x02490, 0x02491, 0x02492, 0x02493, 0x02494, 0x02495, 0x02496, 0x02497, + 0x02498, 0x02499, 0x0249a, 0x0249b, 0x0249c, 0x0249d, 0x0249e, 0x0249f, + 0x024a0, 0x024a1, 0x024a2, 0x024a3, 0x024a4, 0x024a5, 0x024a6, 0x024a7, + 0x024a8, 0x024a9, 0x024aa, 0x024ab, 0x024ac, 0x024ad, 0x024ae, 0x024af, + 0x024b0, 0x024b1, 0x024b2, 0x024b3, 0x024b4, 0x024b5, 0x024d0, 0x024d1, + 0x024d2, 0x024d3, 0x024d4, 0x024d5, 0x024d6, 0x024d7, 0x024d8, 0x024d9, + 0x024da, 0x024db, 0x024dc, 0x024dd, 0x024de, 0x024df, 0x024e0, 0x024e1, + 0x024e2, 0x024e3, 0x024e4, 0x024e5, 0x024e6, 0x024e7, 0x024e8, 0x024e9, + 0x024d0, 0x024d1, 0x024d2, 0x024d3, 0x024d4, 0x024d5, 0x024d6, 0x024d7, + 0x024d8, 0x024d9, 0x024da, 0x024db, 0x024dc, 0x024dd, 0x024de, 0x024df, + 0x024e0, 0x024e1, 0x024e2, 0x024e3, 0x024e4, 0x024e5, 0x024e6, 0x024e7, + 0x024e8, 0x024e9, 0x024ea, 0x024eb, 0x024ec, 0x024ed, 0x024ee, 0x024ef, + 0x024f0, 0x024f1, 0x024f2, 0x024f3, 0x024f4, 0x024f5, 0x024f6, 0x024f7, + 0x024f8, 0x024f9, 0x024fa, 0x024fb, 0x024fc, 0x024fd, 0x024fe, 0x024ff, +}; + + +static const uint32_t nxt_unicode_block_058[128] nxt_aligned(64) = { + 0x02c30, 0x02c31, 0x02c32, 0x02c33, 0x02c34, 0x02c35, 0x02c36, 0x02c37, + 0x02c38, 0x02c39, 0x02c3a, 0x02c3b, 0x02c3c, 0x02c3d, 0x02c3e, 0x02c3f, + 0x02c40, 0x02c41, 0x02c42, 0x02c43, 0x02c44, 0x02c45, 0x02c46, 0x02c47, + 0x02c48, 0x02c49, 0x02c4a, 0x02c4b, 0x02c4c, 0x02c4d, 0x02c4e, 0x02c4f, + 0x02c50, 0x02c51, 0x02c52, 0x02c53, 0x02c54, 0x02c55, 0x02c56, 0x02c57, + 0x02c58, 0x02c59, 0x02c5a, 0x02c5b, 0x02c5c, 0x02c5d, 0x02c5e, 0x02c2f, + 0x02c30, 0x02c31, 0x02c32, 0x02c33, 0x02c34, 0x02c35, 0x02c36, 0x02c37, + 0x02c38, 0x02c39, 0x02c3a, 0x02c3b, 0x02c3c, 0x02c3d, 0x02c3e, 0x02c3f, + 0x02c40, 0x02c41, 0x02c42, 0x02c43, 0x02c44, 0x02c45, 0x02c46, 0x02c47, + 0x02c48, 0x02c49, 0x02c4a, 0x02c4b, 0x02c4c, 0x02c4d, 0x02c4e, 0x02c4f, + 0x02c50, 0x02c51, 0x02c52, 0x02c53, 0x02c54, 0x02c55, 0x02c56, 0x02c57, + 0x02c58, 0x02c59, 0x02c5a, 0x02c5b, 0x02c5c, 0x02c5d, 0x02c5e, 0x02c5f, + 0x02c61, 0x02c61, 0x0026b, 0x01d7d, 0x0027d, 0x02c65, 0x02c66, 0x02c68, + 0x02c68, 0x02c6a, 0x02c6a, 0x02c6c, 0x02c6c, 0x00251, 0x00271, 0x00250, + 0x00252, 0x02c71, 0x02c73, 0x02c73, 0x02c74, 0x02c76, 0x02c76, 0x02c77, + 0x02c78, 0x02c79, 0x02c7a, 0x02c7b, 0x02c7c, 0x02c7d, 0x0023f, 0x00240, +}; + + +static const uint32_t nxt_unicode_block_059[128] nxt_aligned(64) = { + 0x02c81, 0x02c81, 0x02c83, 0x02c83, 0x02c85, 0x02c85, 0x02c87, 0x02c87, + 0x02c89, 0x02c89, 0x02c8b, 0x02c8b, 0x02c8d, 0x02c8d, 0x02c8f, 0x02c8f, + 0x02c91, 0x02c91, 0x02c93, 0x02c93, 0x02c95, 0x02c95, 0x02c97, 0x02c97, + 0x02c99, 0x02c99, 0x02c9b, 0x02c9b, 0x02c9d, 0x02c9d, 0x02c9f, 0x02c9f, + 0x02ca1, 0x02ca1, 0x02ca3, 0x02ca3, 0x02ca5, 0x02ca5, 0x02ca7, 0x02ca7, + 0x02ca9, 0x02ca9, 0x02cab, 0x02cab, 0x02cad, 0x02cad, 0x02caf, 0x02caf, + 0x02cb1, 0x02cb1, 0x02cb3, 0x02cb3, 0x02cb5, 0x02cb5, 0x02cb7, 0x02cb7, + 0x02cb9, 0x02cb9, 0x02cbb, 0x02cbb, 0x02cbd, 0x02cbd, 0x02cbf, 0x02cbf, + 0x02cc1, 0x02cc1, 0x02cc3, 0x02cc3, 0x02cc5, 0x02cc5, 0x02cc7, 0x02cc7, + 0x02cc9, 0x02cc9, 0x02ccb, 0x02ccb, 0x02ccd, 0x02ccd, 0x02ccf, 0x02ccf, + 0x02cd1, 0x02cd1, 0x02cd3, 0x02cd3, 0x02cd5, 0x02cd5, 0x02cd7, 0x02cd7, + 0x02cd9, 0x02cd9, 0x02cdb, 0x02cdb, 0x02cdd, 0x02cdd, 0x02cdf, 0x02cdf, + 0x02ce1, 0x02ce1, 0x02ce3, 0x02ce3, 0x02ce4, 0x02ce5, 0x02ce6, 0x02ce7, + 0x02ce8, 0x02ce9, 0x02cea, 0x02cec, 0x02cec, 0x02cee, 0x02cee, 0x02cef, + 0x02cf0, 0x02cf1, 0x02cf3, 0x02cf3, 0x02cf4, 0x02cf5, 0x02cf6, 0x02cf7, + 0x02cf8, 0x02cf9, 0x02cfa, 0x02cfb, 0x02cfc, 0x02cfd, 0x02cfe, 0x02cff, +}; + + +static const uint32_t nxt_unicode_block_14c[128] nxt_aligned(64) = { + 0x0a600, 0x0a601, 0x0a602, 0x0a603, 0x0a604, 0x0a605, 0x0a606, 0x0a607, + 0x0a608, 0x0a609, 0x0a60a, 0x0a60b, 0x0a60c, 0x0a60d, 0x0a60e, 0x0a60f, + 0x0a610, 0x0a611, 0x0a612, 0x0a613, 0x0a614, 0x0a615, 0x0a616, 0x0a617, + 0x0a618, 0x0a619, 0x0a61a, 0x0a61b, 0x0a61c, 0x0a61d, 0x0a61e, 0x0a61f, + 0x0a620, 0x0a621, 0x0a622, 0x0a623, 0x0a624, 0x0a625, 0x0a626, 0x0a627, + 0x0a628, 0x0a629, 0x0a62a, 0x0a62b, 0x0a62c, 0x0a62d, 0x0a62e, 0x0a62f, + 0x0a630, 0x0a631, 0x0a632, 0x0a633, 0x0a634, 0x0a635, 0x0a636, 0x0a637, + 0x0a638, 0x0a639, 0x0a63a, 0x0a63b, 0x0a63c, 0x0a63d, 0x0a63e, 0x0a63f, + 0x0a641, 0x0a641, 0x0a643, 0x0a643, 0x0a645, 0x0a645, 0x0a647, 0x0a647, + 0x0a649, 0x0a649, 0x0a64b, 0x0a64b, 0x0a64d, 0x0a64d, 0x0a64f, 0x0a64f, + 0x0a651, 0x0a651, 0x0a653, 0x0a653, 0x0a655, 0x0a655, 0x0a657, 0x0a657, + 0x0a659, 0x0a659, 0x0a65b, 0x0a65b, 0x0a65d, 0x0a65d, 0x0a65f, 0x0a65f, + 0x0a661, 0x0a661, 0x0a663, 0x0a663, 0x0a665, 0x0a665, 0x0a667, 0x0a667, + 0x0a669, 0x0a669, 0x0a66b, 0x0a66b, 0x0a66d, 0x0a66d, 0x0a66e, 0x0a66f, + 0x0a670, 0x0a671, 0x0a672, 0x0a673, 0x0a674, 0x0a675, 0x0a676, 0x0a677, + 0x0a678, 0x0a679, 0x0a67a, 0x0a67b, 0x0a67c, 0x0a67d, 0x0a67e, 0x0a67f, +}; + + +static const uint32_t nxt_unicode_block_14d[128] nxt_aligned(64) = { + 0x0a681, 0x0a681, 0x0a683, 0x0a683, 0x0a685, 0x0a685, 0x0a687, 0x0a687, + 0x0a689, 0x0a689, 0x0a68b, 0x0a68b, 0x0a68d, 0x0a68d, 0x0a68f, 0x0a68f, + 0x0a691, 0x0a691, 0x0a693, 0x0a693, 0x0a695, 0x0a695, 0x0a697, 0x0a697, + 0x0a698, 0x0a699, 0x0a69a, 0x0a69b, 0x0a69c, 0x0a69d, 0x0a69e, 0x0a69f, + 0x0a6a0, 0x0a6a1, 0x0a6a2, 0x0a6a3, 0x0a6a4, 0x0a6a5, 0x0a6a6, 0x0a6a7, + 0x0a6a8, 0x0a6a9, 0x0a6aa, 0x0a6ab, 0x0a6ac, 0x0a6ad, 0x0a6ae, 0x0a6af, + 0x0a6b0, 0x0a6b1, 0x0a6b2, 0x0a6b3, 0x0a6b4, 0x0a6b5, 0x0a6b6, 0x0a6b7, + 0x0a6b8, 0x0a6b9, 0x0a6ba, 0x0a6bb, 0x0a6bc, 0x0a6bd, 0x0a6be, 0x0a6bf, + 0x0a6c0, 0x0a6c1, 0x0a6c2, 0x0a6c3, 0x0a6c4, 0x0a6c5, 0x0a6c6, 0x0a6c7, + 0x0a6c8, 0x0a6c9, 0x0a6ca, 0x0a6cb, 0x0a6cc, 0x0a6cd, 0x0a6ce, 0x0a6cf, + 0x0a6d0, 0x0a6d1, 0x0a6d2, 0x0a6d3, 0x0a6d4, 0x0a6d5, 0x0a6d6, 0x0a6d7, + 0x0a6d8, 0x0a6d9, 0x0a6da, 0x0a6db, 0x0a6dc, 0x0a6dd, 0x0a6de, 0x0a6df, + 0x0a6e0, 0x0a6e1, 0x0a6e2, 0x0a6e3, 0x0a6e4, 0x0a6e5, 0x0a6e6, 0x0a6e7, + 0x0a6e8, 0x0a6e9, 0x0a6ea, 0x0a6eb, 0x0a6ec, 0x0a6ed, 0x0a6ee, 0x0a6ef, + 0x0a6f0, 0x0a6f1, 0x0a6f2, 0x0a6f3, 0x0a6f4, 0x0a6f5, 0x0a6f6, 0x0a6f7, + 0x0a6f8, 0x0a6f9, 0x0a6fa, 0x0a6fb, 0x0a6fc, 0x0a6fd, 0x0a6fe, 0x0a6ff, +}; + + +static const uint32_t nxt_unicode_block_14e[128] nxt_aligned(64) = { + 0x0a700, 0x0a701, 0x0a702, 0x0a703, 0x0a704, 0x0a705, 0x0a706, 0x0a707, + 0x0a708, 0x0a709, 0x0a70a, 0x0a70b, 0x0a70c, 0x0a70d, 0x0a70e, 0x0a70f, + 0x0a710, 0x0a711, 0x0a712, 0x0a713, 0x0a714, 0x0a715, 0x0a716, 0x0a717, + 0x0a718, 0x0a719, 0x0a71a, 0x0a71b, 0x0a71c, 0x0a71d, 0x0a71e, 0x0a71f, + 0x0a720, 0x0a721, 0x0a723, 0x0a723, 0x0a725, 0x0a725, 0x0a727, 0x0a727, + 0x0a729, 0x0a729, 0x0a72b, 0x0a72b, 0x0a72d, 0x0a72d, 0x0a72f, 0x0a72f, + 0x0a730, 0x0a731, 0x0a733, 0x0a733, 0x0a735, 0x0a735, 0x0a737, 0x0a737, + 0x0a739, 0x0a739, 0x0a73b, 0x0a73b, 0x0a73d, 0x0a73d, 0x0a73f, 0x0a73f, + 0x0a741, 0x0a741, 0x0a743, 0x0a743, 0x0a745, 0x0a745, 0x0a747, 0x0a747, + 0x0a749, 0x0a749, 0x0a74b, 0x0a74b, 0x0a74d, 0x0a74d, 0x0a74f, 0x0a74f, + 0x0a751, 0x0a751, 0x0a753, 0x0a753, 0x0a755, 0x0a755, 0x0a757, 0x0a757, + 0x0a759, 0x0a759, 0x0a75b, 0x0a75b, 0x0a75d, 0x0a75d, 0x0a75f, 0x0a75f, + 0x0a761, 0x0a761, 0x0a763, 0x0a763, 0x0a765, 0x0a765, 0x0a767, 0x0a767, + 0x0a769, 0x0a769, 0x0a76b, 0x0a76b, 0x0a76d, 0x0a76d, 0x0a76f, 0x0a76f, + 0x0a770, 0x0a771, 0x0a772, 0x0a773, 0x0a774, 0x0a775, 0x0a776, 0x0a777, + 0x0a778, 0x0a77a, 0x0a77a, 0x0a77c, 0x0a77c, 0x01d79, 0x0a77f, 0x0a77f, +}; + + +static const uint32_t nxt_unicode_block_14f[128] nxt_aligned(64) = { + 0x0a781, 0x0a781, 0x0a783, 0x0a783, 0x0a785, 0x0a785, 0x0a787, 0x0a787, + 0x0a788, 0x0a789, 0x0a78a, 0x0a78c, 0x0a78c, 0x00265, 0x0a78e, 0x0a78f, + 0x0a791, 0x0a791, 0x0a793, 0x0a793, 0x0a794, 0x0a795, 0x0a796, 0x0a797, + 0x0a798, 0x0a799, 0x0a79a, 0x0a79b, 0x0a79c, 0x0a79d, 0x0a79e, 0x0a79f, + 0x0a7a1, 0x0a7a1, 0x0a7a3, 0x0a7a3, 0x0a7a5, 0x0a7a5, 0x0a7a7, 0x0a7a7, + 0x0a7a9, 0x0a7a9, 0x00266, 0x0a7ab, 0x0a7ac, 0x0a7ad, 0x0a7ae, 0x0a7af, + 0x0a7b0, 0x0a7b1, 0x0a7b2, 0x0a7b3, 0x0a7b4, 0x0a7b5, 0x0a7b6, 0x0a7b7, + 0x0a7b8, 0x0a7b9, 0x0a7ba, 0x0a7bb, 0x0a7bc, 0x0a7bd, 0x0a7be, 0x0a7bf, + 0x0a7c0, 0x0a7c1, 0x0a7c2, 0x0a7c3, 0x0a7c4, 0x0a7c5, 0x0a7c6, 0x0a7c7, + 0x0a7c8, 0x0a7c9, 0x0a7ca, 0x0a7cb, 0x0a7cc, 0x0a7cd, 0x0a7ce, 0x0a7cf, + 0x0a7d0, 0x0a7d1, 0x0a7d2, 0x0a7d3, 0x0a7d4, 0x0a7d5, 0x0a7d6, 0x0a7d7, + 0x0a7d8, 0x0a7d9, 0x0a7da, 0x0a7db, 0x0a7dc, 0x0a7dd, 0x0a7de, 0x0a7df, + 0x0a7e0, 0x0a7e1, 0x0a7e2, 0x0a7e3, 0x0a7e4, 0x0a7e5, 0x0a7e6, 0x0a7e7, + 0x0a7e8, 0x0a7e9, 0x0a7ea, 0x0a7eb, 0x0a7ec, 0x0a7ed, 0x0a7ee, 0x0a7ef, + 0x0a7f0, 0x0a7f1, 0x0a7f2, 0x0a7f3, 0x0a7f4, 0x0a7f5, 0x0a7f6, 0x0a7f7, + 0x0a7f8, 0x0a7f9, 0x0a7fa, 0x0a7fb, 0x0a7fc, 0x0a7fd, 0x0a7fe, 0x0a7ff, +}; + + +static const uint32_t nxt_unicode_block_1fe[128] nxt_aligned(64) = { + 0x0ff00, 0x0ff01, 0x0ff02, 0x0ff03, 0x0ff04, 0x0ff05, 0x0ff06, 0x0ff07, + 0x0ff08, 0x0ff09, 0x0ff0a, 0x0ff0b, 0x0ff0c, 0x0ff0d, 0x0ff0e, 0x0ff0f, + 0x0ff10, 0x0ff11, 0x0ff12, 0x0ff13, 0x0ff14, 0x0ff15, 0x0ff16, 0x0ff17, + 0x0ff18, 0x0ff19, 0x0ff1a, 0x0ff1b, 0x0ff1c, 0x0ff1d, 0x0ff1e, 0x0ff1f, + 0x0ff20, 0x0ff41, 0x0ff42, 0x0ff43, 0x0ff44, 0x0ff45, 0x0ff46, 0x0ff47, + 0x0ff48, 0x0ff49, 0x0ff4a, 0x0ff4b, 0x0ff4c, 0x0ff4d, 0x0ff4e, 0x0ff4f, + 0x0ff50, 0x0ff51, 0x0ff52, 0x0ff53, 0x0ff54, 0x0ff55, 0x0ff56, 0x0ff57, + 0x0ff58, 0x0ff59, 0x0ff5a, 0x0ff3b, 0x0ff3c, 0x0ff3d, 0x0ff3e, 0x0ff3f, + 0x0ff40, 0x0ff41, 0x0ff42, 0x0ff43, 0x0ff44, 0x0ff45, 0x0ff46, 0x0ff47, + 0x0ff48, 0x0ff49, 0x0ff4a, 0x0ff4b, 0x0ff4c, 0x0ff4d, 0x0ff4e, 0x0ff4f, + 0x0ff50, 0x0ff51, 0x0ff52, 0x0ff53, 0x0ff54, 0x0ff55, 0x0ff56, 0x0ff57, + 0x0ff58, 0x0ff59, 0x0ff5a, 0x0ff5b, 0x0ff5c, 0x0ff5d, 0x0ff5e, 0x0ff5f, + 0x0ff60, 0x0ff61, 0x0ff62, 0x0ff63, 0x0ff64, 0x0ff65, 0x0ff66, 0x0ff67, + 0x0ff68, 0x0ff69, 0x0ff6a, 0x0ff6b, 0x0ff6c, 0x0ff6d, 0x0ff6e, 0x0ff6f, + 0x0ff70, 0x0ff71, 0x0ff72, 0x0ff73, 0x0ff74, 0x0ff75, 0x0ff76, 0x0ff77, + 0x0ff78, 0x0ff79, 0x0ff7a, 0x0ff7b, 0x0ff7c, 0x0ff7d, 0x0ff7e, 0x0ff7f, +}; + + +static const uint32_t nxt_unicode_block_208[40] nxt_aligned(64) = { + 0x10428, 0x10429, 0x1042a, 0x1042b, 0x1042c, 0x1042d, 0x1042e, 0x1042f, + 0x10430, 0x10431, 0x10432, 0x10433, 0x10434, 0x10435, 0x10436, 0x10437, + 0x10438, 0x10439, 0x1043a, 0x1043b, 0x1043c, 0x1043d, 0x1043e, 0x1043f, + 0x10440, 0x10441, 0x10442, 0x10443, 0x10444, 0x10445, 0x10446, 0x10447, + 0x10448, 0x10449, 0x1044a, 0x1044b, 0x1044c, 0x1044d, 0x1044e, 0x1044f, +}; + + +static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = { + nxt_unicode_block_000, + nxt_unicode_block_001, + nxt_unicode_block_002, + nxt_unicode_block_003, + nxt_unicode_block_004, + NULL, + nxt_unicode_block_006, + nxt_unicode_block_007, + nxt_unicode_block_008, + nxt_unicode_block_009, + nxt_unicode_block_00a, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_021, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_03c, + nxt_unicode_block_03d, + nxt_unicode_block_03e, + nxt_unicode_block_03f, + NULL, + NULL, + nxt_unicode_block_042, + nxt_unicode_block_043, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_049, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_058, + nxt_unicode_block_059, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_14c, + nxt_unicode_block_14d, + nxt_unicode_block_14e, + nxt_unicode_block_14f, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_1fe, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_208, +}; diff --git a/src/nxt_unicode_lowcase.pl b/src/nxt_unicode_lowcase.pl new file mode 100644 index 00000000..974ae23a --- /dev/null +++ b/src/nxt_unicode_lowcase.pl @@ -0,0 +1,88 @@ +#!/usr/bin/perl + +use warnings; +use strict; + +# BLOCK_SIZE should be 128, 256, 512, etc. The value 128 provides +# the minimum memory footprint for both 32-bit and 64-bit platforms. +use constant BLOCK_SIZE => 128; + +my %lowcase; +my %blocks; +my $max_block = 0; +my $max_lowcase = 0; + +while (<>) { + if (/^(\w+); (C|S); (\w+);/) { + my ($symbol, $folding) = (hex $1, hex $3); + $lowcase{$symbol} = $folding; + $blocks{int($symbol / BLOCK_SIZE)} = 1; + + if ($max_lowcase < $symbol) { + $max_lowcase = $symbol; + } + } +} + + +my $last_block_size = $max_lowcase % BLOCK_SIZE + 1; + + +for my $block (sort { $a <=> $b } keys %blocks) { + if ($max_block < $block) { + $max_block = $block; + } +} + + +my $blocks = scalar keys %blocks; + +printf("\n/*\n" . + " * %d %s-bytes blocks, %d pointers.\n" . + " * %d bytes on 32-bit platforms, %d bytes on 64-bit platforms.\n" . + " */\n\n", + $blocks, BLOCK_SIZE, $max_block + 1, + ($blocks - 1) * BLOCK_SIZE * 4 + $last_block_size + $max_block * 4, + ($blocks - 1) * BLOCK_SIZE * 4 + $last_block_size+ $max_block * 8); + +printf("#define NXT_UNICODE_MAX_LOWCASE 0x%05x\n\n", $max_lowcase); +printf("#define NXT_UNICODE_BLOCK_SIZE %d\n\n\n", BLOCK_SIZE); + + +for my $block (sort { $a <=> $b } keys %blocks) { + my $block_size = ($block != $max_block) ? BLOCK_SIZE : $last_block_size; + + print "static const uint32_t "; + printf("nxt_unicode_block_%03x[%d] nxt_aligned(64) = {", + $block, $block_size); + + for my $c (0 .. $block_size - 1) { + printf "\n " if $c % 8 == 0; + + my $n = $block * BLOCK_SIZE + $c; + + if (exists $lowcase{$n}) { + printf(" 0x%05x,", $lowcase{$n}); + + } else { + #print " .......,"; + printf(" 0x%05x,", $n); + } + } + + print "\n};\n\n\n"; +} + + +print "static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = {\n"; + +for my $block (0 .. $max_block) { + if (exists($blocks{$block})) { + printf(" nxt_unicode_block_%03x,\n", $block); + + } else { + print " NULL,\n"; + } +} + +print "};\n"; diff --git a/src/nxt_unicode_macosx_lowcase.h b/src/nxt_unicode_macosx_lowcase.h new file mode 100644 index 00000000..3c2b6b45 --- /dev/null +++ b/src/nxt_unicode_macosx_lowcase.h @@ -0,0 +1,816 @@ + +/* + * 15 128-bytes blocks, 511 pointers. + * 9267 bytes on 32-bit platforms, 11307 bytes on 64-bit platforms. + */ + +#define NXT_UNICODE_MAX_LOWCASE 0x0ff3a + +#define NXT_UNICODE_BLOCK_SIZE 128 + + +static const uint32_t nxt_unicode_block_000[128] nxt_aligned(64) = { + 0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007, + 0x00008, 0x00009, 0x0000a, 0x0000b, 0x0000c, 0x0000d, 0x0000e, 0x0000f, + 0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017, + 0x00018, 0x00019, 0x0001a, 0x0001b, 0x0001c, 0x0001d, 0x0001e, 0x0001f, + 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, + 0x00028, 0x00029, 0x0002a, 0x0002b, 0x0002c, 0x0002d, 0x0002e, 0x0002f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0003a, 0x0003b, 0x0003c, 0x0003d, 0x0003e, 0x0003f, + 0x00040, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, + 0x00068, 0x00069, 0x0006a, 0x0006b, 0x0006c, 0x0006d, 0x0006e, 0x0006f, + 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, + 0x00078, 0x00079, 0x0007a, 0x0005b, 0x0005c, 0x0005d, 0x0005e, 0x0005f, + 0x00060, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, + 0x00068, 0x00069, 0x0006a, 0x0006b, 0x0006c, 0x0006d, 0x0006e, 0x0006f, + 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, + 0x00078, 0x00079, 0x0007a, 0x0007b, 0x0007c, 0x0007d, 0x0007e, 0x0007f, +}; + + +static const uint32_t nxt_unicode_block_001[128] nxt_aligned(64) = { + 0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087, + 0x00088, 0x00089, 0x0008a, 0x0008b, 0x0008c, 0x0008d, 0x0008e, 0x0008f, + 0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097, + 0x00098, 0x00099, 0x0009a, 0x0009b, 0x0009c, 0x0009d, 0x0009e, 0x0009f, + 0x000a0, 0x000a1, 0x000a2, 0x000a3, 0x000a4, 0x000a5, 0x000a6, 0x000a7, + 0x000a8, 0x000a9, 0x000aa, 0x000ab, 0x000ac, 0x000ad, 0x000ae, 0x000af, + 0x000b0, 0x000b1, 0x000b2, 0x000b3, 0x000b4, 0x000b5, 0x000b6, 0x000b7, + 0x000b8, 0x000b9, 0x000ba, 0x000bb, 0x000bc, 0x000bd, 0x000be, 0x000bf, + 0x000e0, 0x000e1, 0x000e2, 0x000e3, 0x000e4, 0x000e5, 0x000e6, 0x000e7, + 0x000e8, 0x000e9, 0x000ea, 0x000eb, 0x000ec, 0x000ed, 0x000ee, 0x000ef, + 0x000f0, 0x000f1, 0x000f2, 0x000f3, 0x000f4, 0x000f5, 0x000f6, 0x000d7, + 0x000f8, 0x000f9, 0x000fa, 0x000fb, 0x000fc, 0x000fd, 0x000fe, 0x000df, + 0x000e0, 0x000e1, 0x000e2, 0x000e3, 0x000e4, 0x000e5, 0x000e6, 0x000e7, + 0x000e8, 0x000e9, 0x000ea, 0x000eb, 0x000ec, 0x000ed, 0x000ee, 0x000ef, + 0x000f0, 0x000f1, 0x000f2, 0x000f3, 0x000f4, 0x000f5, 0x000f6, 0x000f7, + 0x000f8, 0x000f9, 0x000fa, 0x000fb, 0x000fc, 0x000fd, 0x000fe, 0x000ff, +}; + + +static const uint32_t nxt_unicode_block_002[128] nxt_aligned(64) = { + 0x00101, 0x00101, 0x00103, 0x00103, 0x00105, 0x00105, 0x00107, 0x00107, + 0x00109, 0x00109, 0x0010b, 0x0010b, 0x0010d, 0x0010d, 0x0010f, 0x0010f, + 0x00111, 0x00111, 0x00113, 0x00113, 0x00115, 0x00115, 0x00117, 0x00117, + 0x00119, 0x00119, 0x0011b, 0x0011b, 0x0011d, 0x0011d, 0x0011f, 0x0011f, + 0x00121, 0x00121, 0x00123, 0x00123, 0x00125, 0x00125, 0x00127, 0x00127, + 0x00129, 0x00129, 0x0012b, 0x0012b, 0x0012d, 0x0012d, 0x0012f, 0x0012f, + 0x00130, 0x00131, 0x00133, 0x00133, 0x00135, 0x00135, 0x00137, 0x00137, + 0x00138, 0x0013a, 0x0013a, 0x0013c, 0x0013c, 0x0013e, 0x0013e, 0x00140, + 0x00140, 0x00142, 0x00142, 0x00144, 0x00144, 0x00146, 0x00146, 0x00148, + 0x00148, 0x00149, 0x0014b, 0x0014b, 0x0014d, 0x0014d, 0x0014f, 0x0014f, + 0x00151, 0x00151, 0x00153, 0x00153, 0x00155, 0x00155, 0x00157, 0x00157, + 0x00159, 0x00159, 0x0015b, 0x0015b, 0x0015d, 0x0015d, 0x0015f, 0x0015f, + 0x00161, 0x00161, 0x00163, 0x00163, 0x00165, 0x00165, 0x00167, 0x00167, + 0x00169, 0x00169, 0x0016b, 0x0016b, 0x0016d, 0x0016d, 0x0016f, 0x0016f, + 0x00171, 0x00171, 0x00173, 0x00173, 0x00175, 0x00175, 0x00177, 0x00177, + 0x000ff, 0x0017a, 0x0017a, 0x0017c, 0x0017c, 0x0017e, 0x0017e, 0x0017f, +}; + + +static const uint32_t nxt_unicode_block_003[128] nxt_aligned(64) = { + 0x00180, 0x00253, 0x00183, 0x00183, 0x00185, 0x00185, 0x00254, 0x00188, + 0x00188, 0x00256, 0x00257, 0x0018c, 0x0018c, 0x0018d, 0x001dd, 0x00259, + 0x0025b, 0x00192, 0x00192, 0x00260, 0x00263, 0x00195, 0x00269, 0x00268, + 0x00199, 0x00199, 0x0019a, 0x0019b, 0x0026f, 0x00272, 0x0019e, 0x00275, + 0x001a1, 0x001a1, 0x001a3, 0x001a3, 0x001a5, 0x001a5, 0x001a6, 0x001a8, + 0x001a8, 0x00283, 0x001aa, 0x001ab, 0x001ad, 0x001ad, 0x00288, 0x001b0, + 0x001b0, 0x0028a, 0x0028b, 0x001b4, 0x001b4, 0x001b6, 0x001b6, 0x00292, + 0x001b9, 0x001b9, 0x001ba, 0x001bb, 0x001bd, 0x001bd, 0x001be, 0x001bf, + 0x001c0, 0x001c1, 0x001c2, 0x001c3, 0x001c6, 0x001c6, 0x001c6, 0x001c9, + 0x001c9, 0x001c9, 0x001cc, 0x001cc, 0x001cc, 0x001ce, 0x001ce, 0x001d0, + 0x001d0, 0x001d2, 0x001d2, 0x001d4, 0x001d4, 0x001d6, 0x001d6, 0x001d8, + 0x001d8, 0x001da, 0x001da, 0x001dc, 0x001dc, 0x001dd, 0x001df, 0x001df, + 0x001e1, 0x001e1, 0x001e3, 0x001e3, 0x001e5, 0x001e5, 0x001e7, 0x001e7, + 0x001e9, 0x001e9, 0x001eb, 0x001eb, 0x001ed, 0x001ed, 0x001ef, 0x001ef, + 0x001f0, 0x001f3, 0x001f3, 0x001f3, 0x001f5, 0x001f5, 0x001f6, 0x001f7, + 0x001f9, 0x001f9, 0x001fb, 0x001fb, 0x001fd, 0x001fd, 0x001ff, 0x001ff, +}; + + +static const uint32_t nxt_unicode_block_004[128] nxt_aligned(64) = { + 0x00201, 0x00201, 0x00203, 0x00203, 0x00205, 0x00205, 0x00207, 0x00207, + 0x00209, 0x00209, 0x0020b, 0x0020b, 0x0020d, 0x0020d, 0x0020f, 0x0020f, + 0x00211, 0x00211, 0x00213, 0x00213, 0x00215, 0x00215, 0x00217, 0x00217, + 0x00219, 0x00219, 0x0021b, 0x0021b, 0x0021c, 0x0021d, 0x0021f, 0x0021f, + 0x00220, 0x00221, 0x00222, 0x00223, 0x00224, 0x00225, 0x00227, 0x00227, + 0x00229, 0x00229, 0x0022b, 0x0022b, 0x0022d, 0x0022d, 0x0022f, 0x0022f, + 0x00231, 0x00231, 0x00233, 0x00233, 0x00234, 0x00235, 0x00236, 0x00237, + 0x00238, 0x00239, 0x0023a, 0x0023b, 0x0023c, 0x0023d, 0x0023e, 0x0023f, + 0x00240, 0x00241, 0x00242, 0x00243, 0x00244, 0x00245, 0x00246, 0x00247, + 0x00248, 0x00249, 0x0024a, 0x0024b, 0x0024c, 0x0024d, 0x0024e, 0x0024f, + 0x00250, 0x00251, 0x00252, 0x00253, 0x00254, 0x00255, 0x00256, 0x00257, + 0x00258, 0x00259, 0x0025a, 0x0025b, 0x0025c, 0x0025d, 0x0025e, 0x0025f, + 0x00260, 0x00261, 0x00262, 0x00263, 0x00264, 0x00265, 0x00266, 0x00267, + 0x00268, 0x00269, 0x0026a, 0x0026b, 0x0026c, 0x0026d, 0x0026e, 0x0026f, + 0x00270, 0x00271, 0x00272, 0x00273, 0x00274, 0x00275, 0x00276, 0x00277, + 0x00278, 0x00279, 0x0027a, 0x0027b, 0x0027c, 0x0027d, 0x0027e, 0x0027f, +}; + + +static const uint32_t nxt_unicode_block_007[128] nxt_aligned(64) = { + 0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x003ac, 0x00387, + 0x003ad, 0x003ae, 0x003af, 0x0038b, 0x003cc, 0x0038d, 0x003cd, 0x003ce, + 0x00390, 0x003b1, 0x003b2, 0x003b3, 0x003b4, 0x003b5, 0x003b6, 0x003b7, + 0x003b8, 0x003b9, 0x003ba, 0x003bb, 0x003bc, 0x003bd, 0x003be, 0x003bf, + 0x003c0, 0x003c1, 0x003a2, 0x003c3, 0x003c4, 0x003c5, 0x003c6, 0x003c7, + 0x003c8, 0x003c9, 0x003ca, 0x003cb, 0x003ac, 0x003ad, 0x003ae, 0x003af, + 0x003b0, 0x003b1, 0x003b2, 0x003b3, 0x003b4, 0x003b5, 0x003b6, 0x003b7, + 0x003b8, 0x003b9, 0x003ba, 0x003bb, 0x003bc, 0x003bd, 0x003be, 0x003bf, + 0x003c0, 0x003c1, 0x003c2, 0x003c3, 0x003c4, 0x003c5, 0x003c6, 0x003c7, + 0x003c8, 0x003c9, 0x003ca, 0x003cb, 0x003cc, 0x003cd, 0x003ce, 0x003cf, + 0x003d0, 0x003d1, 0x003d2, 0x003d3, 0x003d4, 0x003d5, 0x003d6, 0x003d7, + 0x003d8, 0x003d9, 0x003da, 0x003db, 0x003dc, 0x003dd, 0x003de, 0x003df, + 0x003e0, 0x003e1, 0x003e3, 0x003e3, 0x003e5, 0x003e5, 0x003e7, 0x003e7, + 0x003e9, 0x003e9, 0x003eb, 0x003eb, 0x003ed, 0x003ed, 0x003ef, 0x003ef, + 0x003f0, 0x003f1, 0x003f2, 0x003f3, 0x003f4, 0x003f5, 0x003f6, 0x003f7, + 0x003f8, 0x003f9, 0x003fa, 0x003fb, 0x003fc, 0x003fd, 0x003fe, 0x003ff, +}; + + +static const uint32_t nxt_unicode_block_008[128] nxt_aligned(64) = { + 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, + 0x00458, 0x00459, 0x0045a, 0x0045b, 0x0045c, 0x0045d, 0x0045e, 0x0045f, + 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, + 0x00438, 0x00439, 0x0043a, 0x0043b, 0x0043c, 0x0043d, 0x0043e, 0x0043f, + 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, + 0x00448, 0x00449, 0x0044a, 0x0044b, 0x0044c, 0x0044d, 0x0044e, 0x0044f, + 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, + 0x00438, 0x00439, 0x0043a, 0x0043b, 0x0043c, 0x0043d, 0x0043e, 0x0043f, + 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, + 0x00448, 0x00449, 0x0044a, 0x0044b, 0x0044c, 0x0044d, 0x0044e, 0x0044f, + 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, + 0x00458, 0x00459, 0x0045a, 0x0045b, 0x0045c, 0x0045d, 0x0045e, 0x0045f, + 0x00461, 0x00461, 0x00463, 0x00463, 0x00465, 0x00465, 0x00467, 0x00467, + 0x00469, 0x00469, 0x0046b, 0x0046b, 0x0046d, 0x0046d, 0x0046f, 0x0046f, + 0x00471, 0x00471, 0x00473, 0x00473, 0x00475, 0x00475, 0x00477, 0x00477, + 0x00479, 0x00479, 0x0047b, 0x0047b, 0x0047d, 0x0047d, 0x0047f, 0x0047f, +}; + + +static const uint32_t nxt_unicode_block_009[128] nxt_aligned(64) = { + 0x00481, 0x00481, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487, + 0x00488, 0x00489, 0x0048a, 0x0048b, 0x0048c, 0x0048d, 0x0048e, 0x0048f, + 0x00491, 0x00491, 0x00493, 0x00493, 0x00495, 0x00495, 0x00497, 0x00497, + 0x00499, 0x00499, 0x0049b, 0x0049b, 0x0049d, 0x0049d, 0x0049f, 0x0049f, + 0x004a1, 0x004a1, 0x004a3, 0x004a3, 0x004a5, 0x004a5, 0x004a7, 0x004a7, + 0x004a9, 0x004a9, 0x004ab, 0x004ab, 0x004ad, 0x004ad, 0x004af, 0x004af, + 0x004b1, 0x004b1, 0x004b3, 0x004b3, 0x004b5, 0x004b5, 0x004b7, 0x004b7, + 0x004b9, 0x004b9, 0x004bb, 0x004bb, 0x004bd, 0x004bd, 0x004bf, 0x004bf, + 0x004c0, 0x004c2, 0x004c2, 0x004c4, 0x004c4, 0x004c5, 0x004c6, 0x004c8, + 0x004c8, 0x004c9, 0x004ca, 0x004cc, 0x004cc, 0x004cd, 0x004ce, 0x004cf, + 0x004d1, 0x004d1, 0x004d3, 0x004d3, 0x004d4, 0x004d5, 0x004d7, 0x004d7, + 0x004d8, 0x004d9, 0x004da, 0x004db, 0x004dd, 0x004dd, 0x004df, 0x004df, + 0x004e0, 0x004e1, 0x004e3, 0x004e3, 0x004e5, 0x004e5, 0x004e7, 0x004e7, + 0x004e8, 0x004e9, 0x004ea, 0x004eb, 0x004ed, 0x004ed, 0x004ef, 0x004ef, + 0x004f1, 0x004f1, 0x004f3, 0x004f3, 0x004f5, 0x004f5, 0x004f6, 0x004f7, + 0x004f9, 0x004f9, 0x004fa, 0x004fb, 0x004fc, 0x004fd, 0x004fe, 0x004ff, +}; + + +static const uint32_t nxt_unicode_block_00a[128] nxt_aligned(64) = { + 0x00500, 0x00501, 0x00502, 0x00503, 0x00504, 0x00505, 0x00506, 0x00507, + 0x00508, 0x00509, 0x0050a, 0x0050b, 0x0050c, 0x0050d, 0x0050e, 0x0050f, + 0x00510, 0x00511, 0x00512, 0x00513, 0x00514, 0x00515, 0x00516, 0x00517, + 0x00518, 0x00519, 0x0051a, 0x0051b, 0x0051c, 0x0051d, 0x0051e, 0x0051f, + 0x00520, 0x00521, 0x00522, 0x00523, 0x00524, 0x00525, 0x00526, 0x00527, + 0x00528, 0x00529, 0x0052a, 0x0052b, 0x0052c, 0x0052d, 0x0052e, 0x0052f, + 0x00530, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, + 0x00568, 0x00569, 0x0056a, 0x0056b, 0x0056c, 0x0056d, 0x0056e, 0x0056f, + 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, + 0x00578, 0x00579, 0x0057a, 0x0057b, 0x0057c, 0x0057d, 0x0057e, 0x0057f, + 0x00580, 0x00581, 0x00582, 0x00583, 0x00584, 0x00585, 0x00586, 0x00557, + 0x00558, 0x00559, 0x0055a, 0x0055b, 0x0055c, 0x0055d, 0x0055e, 0x0055f, + 0x00560, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, + 0x00568, 0x00569, 0x0056a, 0x0056b, 0x0056c, 0x0056d, 0x0056e, 0x0056f, + 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, + 0x00578, 0x00579, 0x0057a, 0x0057b, 0x0057c, 0x0057d, 0x0057e, 0x0057f, +}; + + +static const uint32_t nxt_unicode_block_03c[128] nxt_aligned(64) = { + 0x01e01, 0x01e01, 0x01e03, 0x01e03, 0x01e05, 0x01e05, 0x01e07, 0x01e07, + 0x01e09, 0x01e09, 0x01e0b, 0x01e0b, 0x01e0d, 0x01e0d, 0x01e0f, 0x01e0f, + 0x01e11, 0x01e11, 0x01e13, 0x01e13, 0x01e15, 0x01e15, 0x01e17, 0x01e17, + 0x01e19, 0x01e19, 0x01e1b, 0x01e1b, 0x01e1d, 0x01e1d, 0x01e1f, 0x01e1f, + 0x01e21, 0x01e21, 0x01e23, 0x01e23, 0x01e25, 0x01e25, 0x01e27, 0x01e27, + 0x01e29, 0x01e29, 0x01e2b, 0x01e2b, 0x01e2d, 0x01e2d, 0x01e2f, 0x01e2f, + 0x01e31, 0x01e31, 0x01e33, 0x01e33, 0x01e35, 0x01e35, 0x01e37, 0x01e37, + 0x01e39, 0x01e39, 0x01e3b, 0x01e3b, 0x01e3d, 0x01e3d, 0x01e3f, 0x01e3f, + 0x01e41, 0x01e41, 0x01e43, 0x01e43, 0x01e45, 0x01e45, 0x01e47, 0x01e47, + 0x01e49, 0x01e49, 0x01e4b, 0x01e4b, 0x01e4d, 0x01e4d, 0x01e4f, 0x01e4f, + 0x01e51, 0x01e51, 0x01e53, 0x01e53, 0x01e55, 0x01e55, 0x01e57, 0x01e57, + 0x01e59, 0x01e59, 0x01e5b, 0x01e5b, 0x01e5d, 0x01e5d, 0x01e5f, 0x01e5f, + 0x01e61, 0x01e61, 0x01e63, 0x01e63, 0x01e65, 0x01e65, 0x01e67, 0x01e67, + 0x01e69, 0x01e69, 0x01e6b, 0x01e6b, 0x01e6d, 0x01e6d, 0x01e6f, 0x01e6f, + 0x01e71, 0x01e71, 0x01e73, 0x01e73, 0x01e75, 0x01e75, 0x01e77, 0x01e77, + 0x01e79, 0x01e79, 0x01e7b, 0x01e7b, 0x01e7d, 0x01e7d, 0x01e7f, 0x01e7f, +}; + + +static const uint32_t nxt_unicode_block_03d[128] nxt_aligned(64) = { + 0x01e81, 0x01e81, 0x01e83, 0x01e83, 0x01e85, 0x01e85, 0x01e87, 0x01e87, + 0x01e89, 0x01e89, 0x01e8b, 0x01e8b, 0x01e8d, 0x01e8d, 0x01e8f, 0x01e8f, + 0x01e91, 0x01e91, 0x01e93, 0x01e93, 0x01e95, 0x01e95, 0x01e96, 0x01e97, + 0x01e98, 0x01e99, 0x01e9a, 0x01e9b, 0x01e9c, 0x01e9d, 0x01e9e, 0x01e9f, + 0x01ea1, 0x01ea1, 0x01ea3, 0x01ea3, 0x01ea5, 0x01ea5, 0x01ea7, 0x01ea7, + 0x01ea9, 0x01ea9, 0x01eab, 0x01eab, 0x01ead, 0x01ead, 0x01eaf, 0x01eaf, + 0x01eb1, 0x01eb1, 0x01eb3, 0x01eb3, 0x01eb5, 0x01eb5, 0x01eb7, 0x01eb7, + 0x01eb9, 0x01eb9, 0x01ebb, 0x01ebb, 0x01ebd, 0x01ebd, 0x01ebf, 0x01ebf, + 0x01ec1, 0x01ec1, 0x01ec3, 0x01ec3, 0x01ec5, 0x01ec5, 0x01ec7, 0x01ec7, + 0x01ec9, 0x01ec9, 0x01ecb, 0x01ecb, 0x01ecd, 0x01ecd, 0x01ecf, 0x01ecf, + 0x01ed1, 0x01ed1, 0x01ed3, 0x01ed3, 0x01ed5, 0x01ed5, 0x01ed7, 0x01ed7, + 0x01ed9, 0x01ed9, 0x01edb, 0x01edb, 0x01edd, 0x01edd, 0x01edf, 0x01edf, + 0x01ee1, 0x01ee1, 0x01ee3, 0x01ee3, 0x01ee5, 0x01ee5, 0x01ee7, 0x01ee7, + 0x01ee9, 0x01ee9, 0x01eeb, 0x01eeb, 0x01eed, 0x01eed, 0x01eef, 0x01eef, + 0x01ef1, 0x01ef1, 0x01ef3, 0x01ef3, 0x01ef5, 0x01ef5, 0x01ef7, 0x01ef7, + 0x01ef9, 0x01ef9, 0x01efa, 0x01efb, 0x01efc, 0x01efd, 0x01efe, 0x01eff, +}; + + +static const uint32_t nxt_unicode_block_03e[128] nxt_aligned(64) = { + 0x01f00, 0x01f01, 0x01f02, 0x01f03, 0x01f04, 0x01f05, 0x01f06, 0x01f07, + 0x01f00, 0x01f01, 0x01f02, 0x01f03, 0x01f04, 0x01f05, 0x01f06, 0x01f07, + 0x01f10, 0x01f11, 0x01f12, 0x01f13, 0x01f14, 0x01f15, 0x01f16, 0x01f17, + 0x01f10, 0x01f11, 0x01f12, 0x01f13, 0x01f14, 0x01f15, 0x01f1e, 0x01f1f, + 0x01f20, 0x01f21, 0x01f22, 0x01f23, 0x01f24, 0x01f25, 0x01f26, 0x01f27, + 0x01f20, 0x01f21, 0x01f22, 0x01f23, 0x01f24, 0x01f25, 0x01f26, 0x01f27, + 0x01f30, 0x01f31, 0x01f32, 0x01f33, 0x01f34, 0x01f35, 0x01f36, 0x01f37, + 0x01f30, 0x01f31, 0x01f32, 0x01f33, 0x01f34, 0x01f35, 0x01f36, 0x01f37, + 0x01f40, 0x01f41, 0x01f42, 0x01f43, 0x01f44, 0x01f45, 0x01f46, 0x01f47, + 0x01f40, 0x01f41, 0x01f42, 0x01f43, 0x01f44, 0x01f45, 0x01f4e, 0x01f4f, + 0x01f50, 0x01f51, 0x01f52, 0x01f53, 0x01f54, 0x01f55, 0x01f56, 0x01f57, + 0x01f58, 0x01f51, 0x01f5a, 0x01f53, 0x01f5c, 0x01f55, 0x01f5e, 0x01f57, + 0x01f60, 0x01f61, 0x01f62, 0x01f63, 0x01f64, 0x01f65, 0x01f66, 0x01f67, + 0x01f60, 0x01f61, 0x01f62, 0x01f63, 0x01f64, 0x01f65, 0x01f66, 0x01f67, + 0x01f70, 0x01f71, 0x01f72, 0x01f73, 0x01f74, 0x01f75, 0x01f76, 0x01f77, + 0x01f78, 0x01f79, 0x01f7a, 0x01f7b, 0x01f7c, 0x01f7d, 0x01f7e, 0x01f7f, +}; + + +static const uint32_t nxt_unicode_block_03f[128] nxt_aligned(64) = { + 0x01f80, 0x01f81, 0x01f82, 0x01f83, 0x01f84, 0x01f85, 0x01f86, 0x01f87, + 0x01f80, 0x01f81, 0x01f82, 0x01f83, 0x01f84, 0x01f85, 0x01f86, 0x01f87, + 0x01f90, 0x01f91, 0x01f92, 0x01f93, 0x01f94, 0x01f95, 0x01f96, 0x01f97, + 0x01f90, 0x01f91, 0x01f92, 0x01f93, 0x01f94, 0x01f95, 0x01f96, 0x01f97, + 0x01fa0, 0x01fa1, 0x01fa2, 0x01fa3, 0x01fa4, 0x01fa5, 0x01fa6, 0x01fa7, + 0x01fa0, 0x01fa1, 0x01fa2, 0x01fa3, 0x01fa4, 0x01fa5, 0x01fa6, 0x01fa7, + 0x01fb0, 0x01fb1, 0x01fb2, 0x01fb3, 0x01fb4, 0x01fb5, 0x01fb6, 0x01fb7, + 0x01fb0, 0x01fb1, 0x01f70, 0x01f71, 0x01fb3, 0x01fbd, 0x003b9, 0x01fbf, + 0x01fc0, 0x01fc1, 0x01fc2, 0x01fc3, 0x01fc4, 0x01fc5, 0x01fc6, 0x01fc7, + 0x01f72, 0x01f73, 0x01f74, 0x01f75, 0x01fc3, 0x01fcd, 0x01fce, 0x01fcf, + 0x01fd0, 0x01fd1, 0x01fd2, 0x01fd3, 0x01fd4, 0x01fd5, 0x01fd6, 0x01fd7, + 0x01fd0, 0x01fd1, 0x01f76, 0x01f77, 0x01fdc, 0x01fdd, 0x01fde, 0x01fdf, + 0x01fe0, 0x01fe1, 0x01fe2, 0x01fe3, 0x01fe4, 0x01fe5, 0x01fe6, 0x01fe7, + 0x01fe0, 0x01fe1, 0x01f7a, 0x01f7b, 0x01fe5, 0x01fed, 0x01fee, 0x01fef, + 0x01ff0, 0x01ff1, 0x01ff2, 0x01ff3, 0x01ff4, 0x01ff5, 0x01ff6, 0x01ff7, + 0x01f78, 0x01f79, 0x01f7c, 0x01f7d, 0x01ff3, 0x01ffd, 0x01ffe, 0x01fff, +}; + + +static const uint32_t nxt_unicode_block_042[128] nxt_aligned(64) = { + 0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107, + 0x02108, 0x02109, 0x0210a, 0x0210b, 0x0210c, 0x0210d, 0x0210e, 0x0210f, + 0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117, + 0x02118, 0x02119, 0x0211a, 0x0211b, 0x0211c, 0x0211d, 0x0211e, 0x0211f, + 0x02120, 0x02121, 0x02122, 0x02123, 0x02124, 0x02125, 0x02126, 0x02127, + 0x02128, 0x02129, 0x0212a, 0x0212b, 0x0212c, 0x0212d, 0x0212e, 0x0212f, + 0x02130, 0x02131, 0x02132, 0x02133, 0x02134, 0x02135, 0x02136, 0x02137, + 0x02138, 0x02139, 0x0213a, 0x0213b, 0x0213c, 0x0213d, 0x0213e, 0x0213f, + 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x02145, 0x02146, 0x02147, + 0x02148, 0x02149, 0x0214a, 0x0214b, 0x0214c, 0x0214d, 0x0214e, 0x0214f, + 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, + 0x02158, 0x02159, 0x0215a, 0x0215b, 0x0215c, 0x0215d, 0x0215e, 0x0215f, + 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, + 0x02178, 0x02179, 0x0217a, 0x0217b, 0x0217c, 0x0217d, 0x0217e, 0x0217f, + 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, + 0x02178, 0x02179, 0x0217a, 0x0217b, 0x0217c, 0x0217d, 0x0217e, 0x0217f, +}; + + +static const uint32_t nxt_unicode_block_1fe[59] nxt_aligned(64) = { + 0x0ff00, 0x0ff01, 0x0ff02, 0x0ff03, 0x0ff04, 0x0ff05, 0x0ff06, 0x0ff07, + 0x0ff08, 0x0ff09, 0x0ff0a, 0x0ff0b, 0x0ff0c, 0x0ff0d, 0x0ff0e, 0x0ff0f, + 0x0ff10, 0x0ff11, 0x0ff12, 0x0ff13, 0x0ff14, 0x0ff15, 0x0ff16, 0x0ff17, + 0x0ff18, 0x0ff19, 0x0ff1a, 0x0ff1b, 0x0ff1c, 0x0ff1d, 0x0ff1e, 0x0ff1f, + 0x0ff20, 0x0ff41, 0x0ff42, 0x0ff43, 0x0ff44, 0x0ff45, 0x0ff46, 0x0ff47, + 0x0ff48, 0x0ff49, 0x0ff4a, 0x0ff4b, 0x0ff4c, 0x0ff4d, 0x0ff4e, 0x0ff4f, + 0x0ff50, 0x0ff51, 0x0ff52, 0x0ff53, 0x0ff54, 0x0ff55, 0x0ff56, 0x0ff57, + 0x0ff58, 0x0ff59, 0x0ff5a, +}; + + +static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = { + nxt_unicode_block_000, + nxt_unicode_block_001, + nxt_unicode_block_002, + nxt_unicode_block_003, + nxt_unicode_block_004, + NULL, + NULL, + nxt_unicode_block_007, + nxt_unicode_block_008, + nxt_unicode_block_009, + nxt_unicode_block_00a, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_03c, + nxt_unicode_block_03d, + nxt_unicode_block_03e, + nxt_unicode_block_03f, + NULL, + NULL, + nxt_unicode_block_042, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + nxt_unicode_block_1fe, +}; diff --git a/src/nxt_unix.h b/src/nxt_unix.h new file mode 100644 index 00000000..41b9ae3c --- /dev/null +++ b/src/nxt_unix.h @@ -0,0 +1,285 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + + +#ifndef _NXT_UNIX_H_INCLUDED_ +#define _NXT_UNIX_H_INCLUDED_ + + +#if (NXT_LINUX) + +#ifdef _FORTIFY_SOURCE +/* + * _FORTIFY_SOURCE + * may call sigaltstack() while _longjmp() checking; + * may cause _longjmp() to fail with message: + * "longjmp() causes uninitialized stack frame"; + * does not allow to use "(void) write()"; + * does surplus checks. + */ +#undef _FORTIFY_SOURCE +#endif + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* pread(), pwrite(), gethostname(). */ +#endif + +#define _FILE_OFFSET_BITS 64 + +#include <malloc.h> /* malloc_usable_size(). */ +#include <sys/syscall.h> /* syscall(SYS_gettid). */ + +#if (NXT_GETRANDOM) +#include <linux/random.h> /* getrandom(). */ +#endif + +#if (__GLIBC__ >= 2 && __GLIBC_MINOR__ >= 4) +/* + * POSIX semaphores using NPTL atomic/futex operations + * were introduced during glibc 2.3 development time. + */ +#define NXT_HAVE_SEM_TRYWAIT_FAST 1 +#endif + +#endif /* NXT_LINUX */ + + +#if (NXT_FREEBSD) + +#if (NXT_HAVE_MALLOC_USABLE_SIZE) +#include <malloc_np.h> /* malloc_usable_size(). */ +#endif + +#if (__FreeBSD_version >= 900007) +/* POSIX semaphores using atomic/umtx. */ +#define NXT_HAVE_SEM_TRYWAIT_FAST 1 +#endif + +#endif /* NXT_FREEBSD */ + + +#if (NXT_SOLARIS) + +#define _FILE_OFFSET_BITS 64 /* Must be before <sys/types.h>. */ + +#ifndef _REENTRANT /* May be set by "-mt" options. */ +#define _REENTRANT /* Thread safe errno. */ +#endif + +#define _POSIX_PTHREAD_SEMANTICS /* 2 arguments in sigwait(). */ + +/* + * Solaris provides two sockets API: + * + * 1) 4.3BSD sockets (int instead of socklen_t in accept(), etc.; + * struct msghdr.msg_accrights) in libsocket; + * 2) X/Open sockets (socklen_t, struct msghdr.msg_control) with __xnet_ + * function name prefix in libxnet and libsocket. + */ + +/* Enable X/Open sockets API. */ +#define _XOPEN_SOURCE +#define _XOPEN_SOURCE_EXTENDED 1 +/* Enable Solaris extensions disabled by _XOPEN_SOURCE. */ +#define __EXTENSIONS__ + +#endif /* NXT_SOLARIS */ + + +#if (NXT_MACOSX) + +#define _XOPEN_SOURCE /* ucontext(3). */ +#define _DARWIN_C_SOURCE /* pthread_threadid_np(), mach_port_t. */ + +#include <mach/mach_time.h> /* mach_absolute_time(). */ +#include <malloc/malloc.h> /* malloc_size(). */ + +#endif /* NXT_MACOSX */ + + +#if (NXT_AIX) + +#define _THREAD_SAFE /* Must before any include. */ + +#endif /* NXT_AIX */ + + +#if (NXT_HPUX) + +#define _FILE_OFFSET_BITS 64 + +/* + * HP-UX provides three sockets API: + * + * 1) 4.3BSD sockets (int instead of socklen_t in accept(), etc.; + * struct msghdr.msg_accrights) in libc; + * 2) X/Open sockets (socklen_t, struct msghdr.msg_control) with _xpg_ + * function name prefix in libc; + * 3) and X/Open sockets (socklen_t, struct msghdr.msg_control) in libxnet. + */ + +/* Enable X/Open sockets API. */ +#define _XOPEN_SOURCE +#define _XOPEN_SOURCE_EXTENDED +/* Enable static function wrappers for _xpg_ X/Open sockets API in libc. */ +#define _HPUX_ALT_XOPEN_SOCKET_API + +#include <sys/mpctl.h> + +#if (NXT_HAVE_HG_GETHRTIME) +#include <sys/mercury.h> +#endif + +#endif /* NXT_HPUX */ + + +#if (NXT_HAVE_ALLOCA_H) +#include <alloca.h> +#endif +#include <dlfcn.h> +#include <errno.h> +#include <fcntl.h> +#include <grp.h> +#include <limits.h> +#include <netdb.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <poll.h> +#include <pwd.h> +#include <semaphore.h> +#include <setjmp.h> +#include <sched.h> +#include <signal.h> +#if (NXT_HAVE_POSIX_SPAWN) +#include <spawn.h> +#endif +#include <stdarg.h> +#include <stddef.h> /* offsetof() */ +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <syslog.h> +#if (NXT_HAVE_SYS_FILIO_H) +#include <sys/filio.h> /* FIONBIO */ +#endif +#include <sys/ioctl.h> +#include <sys/mman.h> +#include <sys/param.h> /* MAXPATHLEN */ +#include <sys/resource.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/uio.h> +#if (NXT_HAVE_UNIX_DOMAIN) +#include <sys/un.h> +#endif +#include <sys/wait.h> +#include <time.h> +#include <ucontext.h> +#include <unistd.h> + + +#if (NXT_THREADS) +#include <pthread.h> +#endif + +#if (NXT_HAVE_EPOLL) +#include <sys/epoll.h> + +#ifdef EPOLLRDHUP +/* + * Epoll edge-tiggered mode is pretty much useless without EPOLLRDHUP support. + */ +#define NXT_HAVE_EPOLL_EDGE 1 +#endif + +#endif + +#if (NXT_HAVE_SIGNALFD) +#include <sys/signalfd.h> +#endif + +#if (NXT_HAVE_EVENTFD) +#include <sys/eventfd.h> +#endif + +#if (NXT_HAVE_KQUEUE) +#include <sys/event.h> +#endif + +#if (NXT_HAVE_EVENTPORT) +#include <port.h> +#endif + +#if (NXT_HAVE_DEVPOLL) +#include <sys/devpoll.h> +#endif + +#if (NXT_HAVE_POLLSET) +#include <sys/pollset.h> +#endif + +#if (NXT_HAVE_LINUX_SENDFILE) +#include <sys/sendfile.h> +#endif + +#if (NXT_HAVE_SOLARIS_SENDFILEV) +#include <sys/sendfile.h> +#endif + + +#if (NXT_TEST_BUILD) +#include <unix/nxt_test_build.h> +#endif + + +/* + * On Linux IOV_MAX is 1024. Linux uses kernel stack for 8 iovec's + * to avoid kernel allocation/deallocation. + * + * On FreeBSD IOV_MAX is 1024. FreeBSD used kernel stack for 8 iovec's + * to avoid kernel allocation/deallocation until FreeBSD 5.2. + * FreeBSD 5.2 and later do not use stack at all. + * + * On Solaris IOV_MAX is 16 and Solaris uses only kernel stack. + * + * On MacOSX IOV_MAX is 1024. MacOSX used kernel stack for 8 iovec's + * to avoid kernel allocation/deallocation until MacOSX 10.4 (Tiger). + * MacOSX 10.4 and later do not use stack at all. + * + * On NetBSD, OpenBSD, and DragonFlyBSD IOV_MAX is 1024. All these OSes + * uses kernel stack for 8 iovec's to avoid kernel allocation/deallocation. + * + * On AIX and HP-UX IOV_MAX is 16. + */ +#define NXT_IOBUF_MAX 8 + + +typedef struct iovec nxt_iobuf_t; + +#define \ +nxt_iobuf_data(iob) \ + (iob)->iov_base + +#define \ +nxt_iobuf_size(iob) \ + (iob)->iov_len + +#define \ +nxt_iobuf_set(iob, p, size) \ + do { \ + (iob)->iov_base = (void *) p; \ + (iob)->iov_len = size; \ + } while (0) + +#define \ +nxt_iobuf_add(iob, size) \ + (iob)->iov_len += size + + +#endif /* _NXT_UNIX_H_INCLUDED_ */ diff --git a/src/nxt_upstream.c b/src/nxt_upstream.c new file mode 100644 index 00000000..e1615120 --- /dev/null +++ b/src/nxt_upstream.c @@ -0,0 +1,43 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +typedef struct { + void (*peer_get)(nxt_upstream_peer_t *up); + void (*peer_free)(nxt_upstream_peer_t *up); +} nxt_upstream_name_t; + + +static const nxt_upstream_name_t nxt_upstream_names[] = { + + { "round_robin", &nxt_upstream_round_robin }, +}; + + +void +nxt_upstream_create(nxt_upstream_peer_t *up) +{ + /* TODO: dynamic balancer add & lvlhsh */ + nxt_upstream_names[0].create(up); +} + + +void +nxt_upstream_peer(nxt_upstream_peer_t *up) +{ + nxt_upstream_t *u; + + u = up->upstream; + + if (u != NULL) { + u->peer_get(up); + return; + } + + nxt_upstream_create(up); +} diff --git a/src/nxt_upstream.h b/src/nxt_upstream.h new file mode 100644 index 00000000..2935ccc8 --- /dev/null +++ b/src/nxt_upstream.h @@ -0,0 +1,46 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UPSTREAM_H_INCLUDED_ +#define _NXT_UPSTREAM_H_INCLUDED_ + + +typedef struct nxt_upstream_peer_s nxt_upstream_peer_t; + + +struct nxt_upstream_peer_s { + /* STUB */ + void *upstream; + void *data; + /**/ + + nxt_sockaddr_t *sockaddr; + nxt_nsec_t delay; + + uint32_t tries; + in_port_t port; + + nxt_str_t addr; + nxt_mem_pool_t *mem_pool; + void (*ready_handler)(nxt_upstream_peer_t *up); + + void (*protocol_handler)(nxt_upstream_source_t *us); +}; + + +typedef struct { + void (*ready_handler)(void *data); + nxt_work_handler_t completion_handler; + nxt_work_handler_t error_handler; +} nxt_upstream_state_t; + + +/* STUB */ +NXT_EXPORT void nxt_upstream_round_robin_peer(nxt_upstream_peer_t *up); +/**/ + + +#endif /* _NXT_UPSTREAM_H_INCLUDED_ */ diff --git a/src/nxt_upstream_round_robin.c b/src/nxt_upstream_round_robin.c new file mode 100644 index 00000000..f8035762 --- /dev/null +++ b/src/nxt_upstream_round_robin.c @@ -0,0 +1,200 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +typedef struct { + int32_t weight; + int32_t effective_weight; + int32_t current_weight; + uint32_t down; /* 1 bit */ + nxt_msec_t last_accessed; + nxt_sockaddr_t *sockaddr; +} nxt_upstream_round_robin_peer_t; + + +typedef struct { + nxt_uint_t npeers; + nxt_upstream_round_robin_peer_t *peers; + nxt_thread_spinlock_t lock; +} nxt_upstream_round_robin_t; + + +static void nxt_upstream_round_robin_create(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_upstream_round_robin_peer_error(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_upstream_round_robin_get_peer(nxt_upstream_peer_t *up); + + +void +nxt_upstream_round_robin_peer(nxt_upstream_peer_t *up) +{ + nxt_job_sockaddr_parse_t *jbs; + + if (up->upstream != NULL) { + nxt_upstream_round_robin_get_peer(up); + } + + jbs = nxt_job_create(up->mem_pool, sizeof(nxt_job_sockaddr_parse_t)); + if (nxt_slow_path(jbs == NULL)) { + up->ready_handler(up); + return; + } + + jbs->resolve.job.data = up; + jbs->resolve.port = up->port; + jbs->resolve.log_level = NXT_LOG_ERR; + jbs->resolve.ready_handler = nxt_upstream_round_robin_create; + jbs->resolve.error_handler = nxt_upstream_round_robin_peer_error; + jbs->addr = up->addr; + + nxt_job_sockaddr_parse(jbs); +} + + +static void +nxt_upstream_round_robin_create(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_uint_t i; + nxt_sockaddr_t *sa; + nxt_upstream_peer_t *up; + nxt_job_sockaddr_parse_t *jbs; + nxt_upstream_round_robin_t *urr; + nxt_upstream_round_robin_peer_t *peer; + + jbs = obj; + up = jbs->resolve.job.data; + + urr = nxt_mem_zalloc(up->mem_pool, sizeof(nxt_upstream_round_robin_t)); + if (nxt_slow_path(urr == NULL)) { + goto fail; + } + + urr->npeers = jbs->resolve.count; + + peer = nxt_mem_zalloc(up->mem_pool, + urr->npeers * sizeof(nxt_upstream_round_robin_peer_t)); + if (nxt_slow_path(peer == NULL)) { + goto fail; + } + + urr->peers = peer; + + for (i = 0; i < urr->npeers; i++) { + peer[i].weight = 1; + peer[i].effective_weight = 1; + + sa = jbs->resolve.sockaddrs[i]; + + /* STUB */ + sa->type = SOCK_STREAM; + + /* TODO: test ret */ + (void) nxt_sockaddr_text(up->mem_pool, sa, 1); + + nxt_log_debug(thr->log, "upstream peer: %*s", sa->text_len, sa->text); + + /* TODO: memcpy to shared memory pool. */ + peer[i].sockaddr = sa; + } + + up->upstream = urr; + + /* STUB */ + up->sockaddr = peer[0].sockaddr; + + nxt_job_destroy(jbs); + up->ready_handler(up); + + //nxt_upstream_round_robin_get_peer(up); + return; + +fail: + + nxt_job_destroy(jbs); + + up->ready_handler(up); +} + + +static void +nxt_upstream_round_robin_peer_error(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_upstream_peer_t *up; + nxt_job_sockaddr_parse_t *jbs; + + jbs = obj; + up = jbs->resolve.job.data; + + up->ready_handler(up); +} + + +static void +nxt_upstream_round_robin_get_peer(nxt_upstream_peer_t *up) +{ + int32_t effective_weights; + nxt_uint_t i; + nxt_msec_t now; + nxt_event_engine_t *engine; + nxt_upstream_round_robin_t *urr; + nxt_upstream_round_robin_peer_t *peer, *best; + + urr = up->upstream; + + engine = nxt_thread_event_engine(); + now = engine->timers.now; + + nxt_thread_spin_lock(&urr->lock); + + best = NULL; + effective_weights = 0; + peer = urr->peers; + + for (i = 0; i < urr->npeers; i++) { + + if (peer[i].down) { + continue; + } + +#if 0 + if (peer[i].max_fails != 0 && peer[i].fails >= peer->max_fails) { + good = peer[i].last_accessed + peer[i].fail_timeout; + + if (nxt_msec_diff(now, peer[i].last_accessed) <= 0) { + continue; + } + } +#endif + + peer[i].current_weight += peer[i].effective_weight; + effective_weights += peer[i].effective_weight; + + if (peer[i].effective_weight < peer[i].weight) { + peer[i].effective_weight++; + } + + if (best == NULL || peer[i].current_weight > best->current_weight) { + best = &peer[i]; + } + } + + if (best != NULL) { + best->current_weight -= effective_weights; + best->last_accessed = now; + + up->sockaddr = best->sockaddr; + + } else { + up->sockaddr = NULL; + } + + nxt_thread_spin_unlock(&urr->lock); + + up->ready_handler(up); +} diff --git a/src/nxt_upstream_source.c b/src/nxt_upstream_source.c new file mode 100644 index 00000000..7b595806 --- /dev/null +++ b/src/nxt_upstream_source.c @@ -0,0 +1,71 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +static nxt_int_t nxt_upstream_header_hash_test(nxt_lvlhsh_query_t *lhq, + void *data); + + +const nxt_lvlhsh_proto_t nxt_upstream_header_hash_proto nxt_aligned(64) = { + NXT_LVLHSH_DEFAULT, + 0, + nxt_upstream_header_hash_test, + nxt_mem_lvlhsh_alloc, + nxt_mem_lvlhsh_free, +}; + + +nxt_int_t +nxt_upstream_header_hash_add(nxt_mem_pool_t *mp, nxt_lvlhsh_t *lh, + const nxt_upstream_name_value_t *unv, nxt_uint_t n) +{ + nxt_lvlhsh_query_t lhq; + + while (n != 0) { + lhq.key_hash = nxt_djb_hash(unv->name, unv->len); + lhq.replace = 1; + lhq.key.len = unv->len; + lhq.key.data = (u_char *) unv->name; + lhq.value = (void *) unv; + lhq.proto = &nxt_upstream_header_hash_proto; + lhq.pool = mp; + + if (nxt_lvlhsh_insert(lh, &lhq) != NXT_OK) { + return NXT_ERROR; + } + + unv++; + n--; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_upstream_header_hash_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + nxt_upstream_name_value_t *unv; + + unv = data; + + if (lhq->key.len == unv->len + && nxt_memcasecmp(lhq->key.data, unv->name, unv->len) == 0) + { + return NXT_OK; + } + + return NXT_DECLINED; +} + + +nxt_int_t +nxt_upstream_name_value_ignore(nxt_upstream_source_t *us, nxt_name_value_t *nv) +{ + return NXT_OK; +} diff --git a/src/nxt_upstream_source.h b/src/nxt_upstream_source.h new file mode 100644 index 00000000..706cb5c9 --- /dev/null +++ b/src/nxt_upstream_source.h @@ -0,0 +1,83 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UPSTREAM_SOURCE_H_INCLUDED_ +#define _NXT_UPSTREAM_SOURCE_H_INCLUDED_ + + +typedef struct { + uint32_t hash; + + unsigned value_len:23; + unsigned skip:1; + unsigned name_len:8; + + u_char *value_start; + u_char *name_start; +} nxt_name_value_t; + + +typedef struct { + nxt_list_t *list; + nxt_lvlhsh_t hash; + + uint16_t status; /* 16 bits */ + + nxt_off_t content_length; +} nxt_upstream_header_in_t; + + +typedef nxt_int_t (*nxt_upstream_name_value_handler_t)( + nxt_upstream_source_t *us, nxt_name_value_t *nv); + + +typedef struct { + nxt_upstream_name_value_handler_t handler; + + uint8_t len; + /* + * A name is inlined to test it with one memory access. + * The struct size is aligned to 32 bytes. + */ +#if (NXT_64BIT) + u_char name[23]; +#else + u_char name[27]; +#endif +} nxt_upstream_name_value_t; + + +struct nxt_upstream_source_s { + nxt_upstream_peer_t *peer; + + const nxt_upstream_state_t *state; + + void *protocol_source; + void *data; + nxt_work_queue_t *work_queue; + + nxt_buf_pool_t buffers; + + nxt_lvlhsh_t header_hash; + nxt_stream_source_t *stream; +}; + + +#define NXT_UPSTREAM_NAME_VALUE_MIN_SIZE \ + offsetof(nxt_http_upstream_header_t, name) + +#define nxt_upstream_name_value(s) sizeof(s) - 1, s + + +NXT_EXPORT nxt_int_t nxt_upstream_header_hash_add(nxt_mem_pool_t *mp, + nxt_lvlhsh_t *lh, const nxt_upstream_name_value_t *unv, nxt_uint_t n); +NXT_EXPORT nxt_int_t nxt_upstream_name_value_ignore(nxt_upstream_source_t *us, + nxt_name_value_t *nv); + +NXT_EXPORT extern const nxt_lvlhsh_proto_t nxt_upstream_header_hash_proto; + + +#endif /* _NXT_UPSTREAM_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_utf8.c b/src/nxt_utf8.c new file mode 100644 index 00000000..56cd3dcd --- /dev/null +++ b/src/nxt_utf8.c @@ -0,0 +1,273 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + +/* + * The nxt_unicode_lowcase.h file is the auto-generated file from + * the CaseFolding-6.3.0.txt file provided by Unicode, Inc.: + * + * ./lib/src/nxt_unicode_lowcase.pl CaseFolding-6.3.0.txt + * + * This file should be copied to system specific nxt_unicode_SYSTEM_lowcase.h + * file and utf8_file_name_test should be built with this file. + * Then a correct system specific file should be generated: + * + * ./build/utf8_file_name_test | ./lib/src/nxt_unicode_lowcase.pl + * + * Only common and simple case foldings are supported. Full case foldings + * is not supported. Combined characters are also not supported. + */ + +#if (NXT_MACOSX) +#include <nxt_unicode_macosx_lowcase.h> + +#else +#include <nxt_unicode_lowcase.h> +#endif + + +u_char * +nxt_utf8_encode(u_char *p, uint32_t u) +{ + if (u < 0x80) { + *p++ = (u_char) (u & 0xff); + return p; + } + + if (u < 0x0800) { + *p++ = (u_char) (( u >> 6) | 0xc0); + *p++ = (u_char) (( u & 0x3f) | 0x80); + return p; + } + + if (u < 0x10000) { + *p++ = (u_char) ( (u >> 12) | 0xe0); + *p++ = (u_char) (((u >> 6) & 0x3f) | 0x80); + *p++ = (u_char) (( u & 0x3f) | 0x80); + return p; + } + + if (u < 0x110000) { + *p++ = (u_char) ( (u >> 18) | 0xf0); + *p++ = (u_char) (((u >> 12) & 0x3f) | 0x80); + *p++ = (u_char) (((u >> 6) & 0x3f) | 0x80); + *p++ = (u_char) (( u & 0x3f) | 0x80); + return p; + } + + return NULL; +} + + +/* + * nxt_utf8_decode() decodes UTF-8 sequences and returns a valid + * character 0x00 - 0x10ffff, or 0xffffffff for invalid or overlong + * UTF-8 sequence. + */ + +uint32_t +nxt_utf8_decode(const u_char **start, const u_char *end) +{ + uint32_t u; + + u = (uint32_t) **start; + + if (u < 0x80) { + (*start)++; + return u; + } + + return nxt_utf8_decode2(start, end); +} + + +/* + * nxt_utf8_decode2() decodes two and more bytes UTF-8 sequences only + * and returns a valid character 0x80 - 0x10ffff, or 0xffffffff for + * invalid or overlong UTF-8 sequence. + */ + +uint32_t +nxt_utf8_decode2(const u_char **start, const u_char *end) +{ + u_char c; + size_t n; + uint32_t u, overlong; + const u_char *p; + + p = *start; + u = (uint32_t) *p; + + if (u >= 0xe0) { + + if (u >= 0xf0) { + + if (nxt_slow_path(u > 0xf4)) { + /* + * The maximum valid Unicode character is 0x10ffff + * which is encoded as 0xf4 0x8f 0xbf 0xbf. + */ + return 0xffffffff; + } + + u &= 0x07; + overlong = 0x00ffff; + n = 3; + + } else { + u &= 0x0f; + overlong = 0x07ff; + n = 2; + } + + } else if (u >= 0xc2) { + + /* 0x80 is encoded as 0xc2 0x80. */ + + u &= 0x1f; + overlong = 0x007f; + n = 1; + + } else { + /* u <= 0xc2 */ + return 0xffffffff; + } + + p++; + + if (nxt_fast_path(p + n <= end)) { + + do { + c = *p++; + /* + * The byte must in the 0x80 - 0xbf range. + * Values below 0x80 become >= 0x80. + */ + c = c - 0x80; + + if (nxt_slow_path(c > 0x3f)) { + return 0xffffffff; + } + + u = (u << 6) | c; + n--; + + } while (n != 0); + + if (overlong < u && u < 0x110000) { + *start = p; + return u; + } + } + + return 0xffffffff; +} + + +/* + * nxt_utf8_casecmp() tests only up to the minimum of given lengths, but + * requires lengths of both strings because otherwise nxt_utf8_decode2() + * may fail due to incomplete sequence. + */ + +nxt_int_t +nxt_utf8_casecmp(const u_char *start1, const u_char *start2, size_t len1, + size_t len2) +{ + int32_t n; + uint32_t u1, u2; + const u_char *end1, *end2; + + end1 = start1 + len1; + end2 = start2 + len2; + + while (start1 < end1 && start2 < end2) { + + u1 = nxt_utf8_lowcase(&start1, end1); + + u2 = nxt_utf8_lowcase(&start2, end2); + + if (nxt_slow_path((u1 | u2) == 0xffffffff)) { + return NXT_UTF8_SORT_INVALID; + } + + n = u1 - u2; + + if (n != 0) { + return (nxt_int_t) n; + } + } + + return 0; +} + + +uint32_t +nxt_utf8_lowcase(const u_char **start, const u_char *end) +{ + uint32_t u; + const uint32_t *block; + + u = (uint32_t) **start; + + if (nxt_fast_path(u < 0x80)) { + (*start)++; + + return nxt_unicode_block_000[u]; + } + + u = nxt_utf8_decode2(start, end); + + if (u <= NXT_UNICODE_MAX_LOWCASE) { + block = nxt_unicode_blocks[u / NXT_UNICODE_BLOCK_SIZE]; + + if (block != NULL) { + return block[u % NXT_UNICODE_BLOCK_SIZE]; + } + } + + return u; +} + + +ssize_t +nxt_utf8_length(const u_char *p, size_t len) +{ + ssize_t length; + const u_char *end; + + length = 0; + + end = p + len; + + while (p < end) { + if (nxt_slow_path(nxt_utf8_decode(&p, end) == 0xffffffff)) { + return -1; + } + + length++; + } + + return length; +} + + +nxt_bool_t +nxt_utf8_is_valid(const u_char *p, size_t len) +{ + const u_char *end; + + end = p + len; + + while (p < end) { + if (nxt_slow_path(nxt_utf8_decode(&p, end) == 0xffffffff)) { + return 0; + } + } + + return 1; +} diff --git a/src/nxt_utf8.h b/src/nxt_utf8.h new file mode 100644 index 00000000..92847545 --- /dev/null +++ b/src/nxt_utf8.h @@ -0,0 +1,60 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_UTF8_H_INCLUDED_ +#define _NXT_UTF8_H_INCLUDED_ + + +/* + * Since the maximum valid Unicode character is 0x0010ffff, the maximum + * difference between Unicode characters is lesser 0x0010ffff and + * 0x0eee0eee can be used as value to indicate UTF-8 encoding error. + */ +#define NXT_UTF8_SORT_INVALID 0x0eee0eee + + +NXT_EXPORT u_char *nxt_utf8_encode(u_char *p, uint32_t u); +NXT_EXPORT uint32_t nxt_utf8_decode(const u_char **start, const u_char *end); +NXT_EXPORT uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end); +NXT_EXPORT nxt_int_t nxt_utf8_casecmp(const u_char *start1, + const u_char *start2, size_t len1, size_t len2); +NXT_EXPORT uint32_t nxt_utf8_lowcase(const u_char **start, const u_char *end); +NXT_EXPORT ssize_t nxt_utf8_length(const u_char *p, size_t len); +NXT_EXPORT nxt_bool_t nxt_utf8_is_valid(const u_char *p, size_t len); + + +/* nxt_utf8_next() expects a valid UTF-8 string. */ + +nxt_inline const u_char * +nxt_utf8_next(const u_char *p, const u_char *end) +{ + u_char c; + + c = *p++; + + if ((c & 0x80) != 0) { + + do { + /* + * The first UTF-8 byte is either 0xxxxxxx or 11xxxxxx. + * The next UTF-8 bytes are 10xxxxxx. + */ + c = *p; + + if ((c & 0xc0) != 0x80) { + return p; + } + + p++; + + } while (p < end); + } + + return p; +} + + +#endif /* _NXT_UTF8_H_INCLUDED_ */ diff --git a/src/nxt_vector.c b/src/nxt_vector.c new file mode 100644 index 00000000..faaf2f54 --- /dev/null +++ b/src/nxt_vector.c @@ -0,0 +1,156 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +nxt_vector_t * +nxt_vector_create(nxt_uint_t items, size_t item_size, + const nxt_mem_proto_t *proto, void *pool) +{ + nxt_vector_t *vector; + + vector = proto->alloc(pool, sizeof(nxt_vector_t) + items * item_size); + + if (nxt_fast_path(vector != NULL)) { + vector->start = (char *) vector + sizeof(nxt_vector_t); + vector->items = 0; + vector->item_size = item_size; + vector->avalaible = items; + vector->type = NXT_VECTOR_EMBEDDED; + } + + return vector; +} + + +void * +nxt_vector_init(nxt_vector_t *vector, nxt_uint_t items, size_t item_size, + const nxt_mem_proto_t *proto, void *pool) +{ + vector->start = proto->alloc(pool, items * item_size); + + if (nxt_fast_path(vector->start != NULL)) { + vector->items = 0; + vector->item_size = item_size; + vector->avalaible = items; + vector->type = NXT_VECTOR_INITED; + } + + return vector->start; +} + + +void +nxt_vector_destroy(nxt_vector_t *vector, const nxt_mem_proto_t *proto, + void *pool) +{ + switch (vector->type) { + + case NXT_VECTOR_INITED: + proto->free(pool, vector->start); +#if (NXT_DEBUG) + vector->start = NULL; + vector->items = 0; + vector->avalaible = 0; +#endif + break; + + case NXT_VECTOR_DESCRETE: + proto->free(pool, vector->start); + + /* Fall through. */ + + case NXT_VECTOR_EMBEDDED: + proto->free(pool, vector); + break; + } +} + + +void * +nxt_vector_add(nxt_vector_t *vector, const nxt_mem_proto_t *proto, void *pool) +{ + void *item, *start, *old; + size_t size; + uint32_t n; + + n = vector->avalaible; + + if (n == vector->items) { + + if (n < 16) { + /* Allocate new vector twice as much as current. */ + n *= 2; + + } else { + /* Allocate new vector half as much as current. */ + n += n / 2; + } + + size = n * vector->item_size; + + start = proto->alloc(pool, size); + if (nxt_slow_path(start == NULL)) { + return NULL; + } + + vector->avalaible = n; + old = vector->start; + vector->start = start; + + nxt_memcpy(start, old, size); + + if (vector->type == NXT_VECTOR_EMBEDDED) { + vector->type = NXT_VECTOR_DESCRETE; + + } else { + proto->free(pool, old); + } + } + + item = (char *) vector->start + vector->item_size * vector->items; + + vector->items++; + + return item; +} + + +void * +nxt_vector_zero_add(nxt_vector_t *vector, const nxt_mem_proto_t *proto, + void *pool) +{ + void *item; + + item = nxt_vector_add(vector, proto, pool); + + if (nxt_fast_path(item != NULL)) { + nxt_memzero(item, vector->item_size); + } + + return item; +} + + +void +nxt_vector_remove(nxt_vector_t *vector, void *item) +{ + u_char *next, *last, *end; + uint32_t item_size; + + item_size = vector->item_size; + end = (u_char *) vector->start + item_size * vector->items; + last = end - item_size; + + if (item != last) { + next = (u_char *) item + item_size; + + nxt_memmove(item, next, end - next); + } + + vector->items--; +} diff --git a/src/nxt_vector.h b/src/nxt_vector.h new file mode 100644 index 00000000..ddd3e220 --- /dev/null +++ b/src/nxt_vector.h @@ -0,0 +1,69 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_VECTOR_H_INCLUDED_ +#define _NXT_VECTOR_H_INCLUDED_ + + +typedef enum { + NXT_VECTOR_INITED = 0, + NXT_VECTOR_DESCRETE, + NXT_VECTOR_EMBEDDED, +} nxt_vector_type_t; + + +typedef struct { + void *start; + /* + * A vector can hold no more than 65536 items. + * The item size is no more than 64K. + */ + uint16_t items; + uint16_t avalaible; + uint16_t item_size; + nxt_vector_type_t type:8; +} nxt_vector_t; + + +NXT_EXPORT nxt_vector_t *nxt_vector_create(nxt_uint_t items, size_t item_size, + const nxt_mem_proto_t *proto, void *pool); +NXT_EXPORT void *nxt_vector_init(nxt_vector_t *vector, nxt_uint_t items, + size_t item_size, const nxt_mem_proto_t *proto, void *pool); +NXT_EXPORT void nxt_vector_destroy(nxt_vector_t *vector, + const nxt_mem_proto_t *proto, void *pool); +NXT_EXPORT void *nxt_vector_add(nxt_vector_t *vector, + const nxt_mem_proto_t *proto, void *pool); +NXT_EXPORT void *nxt_vector_zero_add(nxt_vector_t *vector, + const nxt_mem_proto_t *proto, void *pool); +NXT_EXPORT void nxt_vector_remove(nxt_vector_t *vector, void *item); + + +#define \ +nxt_vector_last(vector) \ + ((void *) \ + ((char *) (vector)->start \ + + (vector)->item_size * ((vector)->items - 1))) + + +#define \ +nxt_vector_reset(vector) \ + (vector)->items = 0; + + +#define \ +nxt_vector_is_empty(vector) \ + ((vector)->items == 0) + + +nxt_inline void * +nxt_vector_remove_last(nxt_vector_t *vector) +{ + vector->items--; + return (char *) vector->start + vector->item_size * vector->items; +} + + +#endif /* _NXT_VECTOR_H_INCLUDED_ */ diff --git a/src/nxt_work_queue.c b/src/nxt_work_queue.c new file mode 100644 index 00000000..914d6125 --- /dev/null +++ b/src/nxt_work_queue.c @@ -0,0 +1,610 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> + + +/* + * Available work items are crucial for overall engine operation, so + * the items are preallocated in two chunks: cache and spare chunks. + * By default each chunk preallocates 409 work items on two or four + * CPU pages depending on platform. If all items in a cache chunk are + * exhausted then a spare chunk becomes a cache chunk, and a new spare + * chunk is allocated. This two-step allocation mitigates low memory + * condition impact on work queue operation. However, if both chunks + * are exhausted then a thread will sleep in reliance on another thread + * frees some memory. However, this may lead to deadlock and probably + * a process should be aborted. This behaviour should be considered as + * abort on program stack exhaustion. + * + * The cache and spare chunks initially are also allocated in two steps: + * a spare chunk is allocated first, then it becomes the cache chunk and + * a new spare chunk is allocated again. + */ + +static void nxt_work_queue_allocate(nxt_work_queue_cache_t *cache, + nxt_thread_spinlock_t *lock); +static void nxt_work_queue_sleep(nxt_thread_spinlock_t *lock); +static nxt_work_queue_t *nxt_thread_current_work_queue(nxt_thread_t *thr); +static nxt_work_handler_t nxt_locked_work_queue_pop_work( + nxt_locked_work_queue_t *lwq, void **obj, void **data, nxt_log_t **log); + + +/* It should be adjusted with the "work_queue_bucket_items" directive. */ +static nxt_uint_t nxt_work_queue_bucket_items = 409; + + +void +nxt_thread_work_queue_create(nxt_thread_t *thr, size_t chunk_size) +{ + nxt_memzero(&thr->work_queue, sizeof(nxt_thread_work_queue_t)); + + nxt_work_queue_name(&thr->work_queue.main, "main"); + nxt_work_queue_name(&thr->work_queue.last, "last"); + + if (chunk_size == 0) { + chunk_size = nxt_work_queue_bucket_items; + } + + /* nxt_work_queue_chunk_t already has one work item. */ + thr->work_queue.cache.chunk_size = chunk_size - 1; + + while (thr->work_queue.cache.next == NULL) { + nxt_work_queue_allocate(&thr->work_queue.cache, NULL); + } +} + + +void +nxt_thread_work_queue_destroy(nxt_thread_t *thr) +{ + nxt_work_queue_chunk_t *chunk, *next; + + for (chunk = thr->work_queue.cache.chunk; chunk; chunk = next) { + next = chunk->next; + nxt_free(chunk); + } +} + + +static void +nxt_work_queue_allocate(nxt_work_queue_cache_t *cache, + nxt_thread_spinlock_t *lock) +{ + size_t size; + nxt_uint_t i, n; + nxt_work_t *work; + nxt_work_queue_chunk_t *chunk; + + n = cache->chunk_size; + size = sizeof(nxt_work_queue_chunk_t) + n * sizeof(nxt_work_t); + + chunk = nxt_malloc(size); + + if (nxt_fast_path(chunk != NULL)) { + + chunk->next = cache->chunk; + cache->chunk = chunk; + work = &chunk->work; + + for (i = 0; i < n; i++) { + work[i].next = &work[i + 1]; + } + + work[i].next = NULL; + work++; + + } else if (cache->spare != NULL) { + + work = NULL; + + } else { + nxt_work_queue_sleep(lock); + return; + } + + cache->next = cache->spare; + cache->spare = work; +} + + +static void +nxt_work_queue_sleep(nxt_thread_spinlock_t *lock) +{ + if (lock != NULL) { + nxt_thread_spin_unlock(lock); + } + + nxt_nanosleep(100 * 1000000); /* 100ms */ + + if (lock != NULL) { + nxt_thread_spin_lock(lock); + } +} + + +/* Add a work to a work queue tail. */ + +void +nxt_thread_work_queue_add(nxt_thread_t *thr, nxt_work_queue_t *wq, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log) +{ + nxt_work_t *work; + + nxt_work_queue_attach(thr, wq); + + for ( ;; ) { + work = thr->work_queue.cache.next; + + if (nxt_fast_path(work != NULL)) { + thr->work_queue.cache.next = work->next; + work->next = NULL; + + work->handler = handler; + work->obj = obj; + work->data = data; + work->log = log; + + if (wq->tail != NULL) { + wq->tail->next = work; + + } else { + wq->head = work; + } + + wq->tail = work; + + return; + } + + nxt_work_queue_allocate(&thr->work_queue.cache, NULL); + } +} + + +/* Push a work to a work queue head. */ + +void +nxt_thread_work_queue_push(nxt_thread_t *thr, nxt_work_queue_t *wq, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log) +{ + nxt_work_t *work; + + nxt_work_queue_attach(thr, wq); + + for ( ;; ) { + work = thr->work_queue.cache.next; + + if (nxt_fast_path(work != NULL)) { + thr->work_queue.cache.next = work->next; + work->next = wq->head; + + work->handler = handler; + work->obj = obj; + work->data = data; + work->log = log; + + wq->head = work; + + if (wq->tail == NULL) { + wq->tail = work; + } + + return; + } + + nxt_work_queue_allocate(&thr->work_queue.cache, NULL); + } +} + + +/* Attach a work queue to a thread work queue. */ + +void +nxt_work_queue_attach(nxt_thread_t *thr, nxt_work_queue_t *wq) +{ + if (wq->next == NULL && wq != thr->work_queue.tail) { + + if (thr->work_queue.tail != NULL) { + thr->work_queue.tail->next = wq; + + } else { + thr->work_queue.head = wq; + } + + thr->work_queue.tail = wq; + } +} + + +/* Pop a work from a thread work queue head. */ + +nxt_work_handler_t +nxt_thread_work_queue_pop(nxt_thread_t *thr, void **obj, void **data, + nxt_log_t **log) +{ + nxt_work_t *work; + nxt_work_queue_t *wq; + + wq = nxt_thread_current_work_queue(thr); + + if (wq != NULL) { + + work = wq->head; + + if (work != NULL) { + wq->head = work->next; + + if (work->next == NULL) { + wq->tail = NULL; + } + + *obj = work->obj; + nxt_prefetch(*obj); + *data = work->data; + nxt_prefetch(*data); + + work->next = thr->work_queue.cache.next; + thr->work_queue.cache.next = work; + + *log = work->log; + +#if (NXT_DEBUG) + + if (work->handler == NULL) { + nxt_log_alert(thr->log, "null work handler"); + nxt_abort(); + } + +#endif + + return work->handler; + } + } + + return NULL; +} + + +static nxt_work_queue_t * +nxt_thread_current_work_queue(nxt_thread_t *thr) +{ + nxt_work_queue_t *wq, *next; + + for (wq = thr->work_queue.head; wq != NULL; wq = next) { + + if (wq->head != NULL) { + nxt_log_debug(thr->log, "work queue: %s", wq->name); + return wq; + } + + /* Detach empty work queue. */ + next = wq->next; + wq->next = NULL; + thr->work_queue.head = next; + } + + thr->work_queue.tail = NULL; + + return NULL; +} + + +/* Drop a work with specified data from a thread work queue. */ + +void +nxt_thread_work_queue_drop(nxt_thread_t *thr, void *data) +{ + nxt_work_t *work, *prev, *next, **link; + nxt_work_queue_t *wq; + + for (wq = thr->work_queue.head; wq != NULL; wq = wq->next) { + + prev = NULL; + link = &wq->head; + + for (work = wq->head; work != NULL; work = next) { + + next = work->next; + + if (data != work->obj) { + prev = work; + link = &work->next; + + } else { + if (next == NULL) { + wq->tail = prev; + } + + nxt_log_debug(thr->log, "work queue drop"); + + *link = next; + + work->next = thr->work_queue.cache.next; + thr->work_queue.cache.next = work; + } + } + } +} + + +/* Add a work to the thread last work queue's tail. */ + +void +nxt_thread_last_work_queue_add(nxt_thread_t *thr, nxt_work_handler_t handler, + void *obj, void *data, nxt_log_t *log) +{ + nxt_work_t *work; + + for ( ;; ) { + work = thr->work_queue.cache.next; + + if (nxt_fast_path(work != NULL)) { + thr->work_queue.cache.next = work->next; + work->next = NULL; + + work->handler = handler; + work->obj = obj; + work->data = data; + work->log = log; + + if (thr->work_queue.last.tail != NULL) { + thr->work_queue.last.tail->next = work; + + } else { + thr->work_queue.last.head = work; + } + + thr->work_queue.last.tail = work; + + return; + } + + nxt_work_queue_allocate(&thr->work_queue.cache, NULL); + } +} + + +/* Pop a work from the thread last work queue's head. */ + +nxt_work_handler_t +nxt_thread_last_work_queue_pop(nxt_thread_t *thr, void **obj, void **data, + nxt_log_t **log) +{ + nxt_work_t *work; + + work = thr->work_queue.last.head; + + if (work != NULL) { + nxt_log_debug(thr->log, "work queue: %s", thr->work_queue.last.name); + + thr->work_queue.last.head = work->next; + + if (work->next == NULL) { + thr->work_queue.last.tail = NULL; + } + + *obj = work->obj; + nxt_prefetch(*obj); + *data = work->data; + nxt_prefetch(*data); + + work->next = thr->work_queue.cache.next; + thr->work_queue.cache.next = work; + + *log = work->log; + +#if (NXT_DEBUG) + + if (work->handler == NULL) { + nxt_log_alert(thr->log, "null work handler"); + nxt_abort(); + } + +#endif + + return work->handler; + } + + return NULL; +} + + +void +nxt_work_queue_destroy(nxt_work_queue_t *wq) +{ + nxt_thread_t *thr; + nxt_work_queue_t *q; + + thr = nxt_thread(); + + /* Detach from a thread work queue. */ + + if (thr->work_queue.head == wq) { + thr->work_queue.head = wq->next; + q = NULL; + goto found; + } + + for (q = thr->work_queue.head; q != NULL; q = q->next) { + if (q->next == wq) { + q->next = wq->next; + goto found; + } + } + + return; + +found: + + if (thr->work_queue.tail == wq) { + thr->work_queue.tail = q; + } + + /* Move all queue's works to a thread work queue cache. */ + + if (wq->tail != NULL) { + wq->tail->next = thr->work_queue.cache.next; + } + + if (wq->head != NULL) { + thr->work_queue.cache.next = wq->head; + } +} + + +/* Locked work queue operations. */ + +void +nxt_locked_work_queue_create(nxt_locked_work_queue_t *lwq, size_t chunk_size) +{ + nxt_memzero(lwq, sizeof(nxt_locked_work_queue_t)); + + if (chunk_size == 0) { + chunk_size = nxt_work_queue_bucket_items; + } + + lwq->cache.chunk_size = chunk_size; + + while (lwq->cache.next == NULL) { + nxt_work_queue_allocate(&lwq->cache, NULL); + } +} + + +void +nxt_locked_work_queue_destroy(nxt_locked_work_queue_t *lwq) +{ + nxt_work_queue_chunk_t *chunk, *next; + + for (chunk = lwq->cache.chunk; chunk; chunk = next) { + next = chunk->next; + nxt_free(chunk); + } +} + + +/* Add a work to a locked work queue tail. */ + +void +nxt_locked_work_queue_add(nxt_locked_work_queue_t *lwq, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log) +{ + nxt_work_t *work; + + nxt_thread_spin_lock(&lwq->lock); + + for ( ;; ) { + work = lwq->cache.next; + + if (nxt_fast_path(work != NULL)) { + lwq->cache.next = work->next; + + work->next = NULL; + work->handler = handler; + work->obj = obj; + work->data = data; + work->log = log; + + if (lwq->tail != NULL) { + lwq->tail->next = work; + + } else { + lwq->head = work; + } + + lwq->tail = work; + + break; + } + + nxt_work_queue_allocate(&lwq->cache, &lwq->lock); + } + + nxt_thread_spin_unlock(&lwq->lock); +} + + +/* Pop a work from a locked work queue head. */ + +nxt_work_handler_t +nxt_locked_work_queue_pop(nxt_locked_work_queue_t *lwq, void **obj, + void **data, nxt_log_t **log) +{ + nxt_work_handler_t handler; + + nxt_thread_spin_lock(&lwq->lock); + + handler = nxt_locked_work_queue_pop_work(lwq, obj, data, log); + + nxt_thread_spin_unlock(&lwq->lock); + + return handler; +} + + +static nxt_work_handler_t +nxt_locked_work_queue_pop_work(nxt_locked_work_queue_t *lwq, void **obj, + void **data, nxt_log_t **log) +{ + nxt_work_t *work; + + work = lwq->head; + + if (work == NULL) { + return NULL; + } + + *obj = work->obj; + nxt_prefetch(*obj); + *data = work->data; + nxt_prefetch(*data); + + lwq->head = work->next; + + if (work->next == NULL) { + lwq->tail = NULL; + } + + work->next = lwq->cache.next; + lwq->cache.next = work; + + *log = work->log; + + return work->handler; +} + + +/* Move all works from a locked work queue to a usual work queue. */ + +void +nxt_locked_work_queue_move(nxt_thread_t *thr, nxt_locked_work_queue_t *lwq, + nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_log_t *log; + nxt_work_handler_t handler; + + /* Locked work queue head can be tested without a lock. */ + + if (nxt_fast_path(lwq->head == NULL)) { + return; + } + + nxt_thread_spin_lock(&lwq->lock); + + for ( ;; ) { + handler = nxt_locked_work_queue_pop_work(lwq, &obj, &data, &log); + + if (handler == NULL) { + break; + } + + nxt_thread_work_queue_add(thr, wq, handler, obj, data, log); + } + + nxt_thread_spin_unlock(&lwq->lock); +} diff --git a/src/nxt_work_queue.h b/src/nxt_work_queue.h new file mode 100644 index 00000000..2b168b00 --- /dev/null +++ b/src/nxt_work_queue.h @@ -0,0 +1,137 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_WORK_QUEUE_H_INCLUDED_ +#define _NXT_WORK_QUEUE_H_INCLUDED_ + + +/* + * A work handler with just the obj and data arguments instead + * of pointer to a possibly large a work struct allows to call + * the handler not only via a work queue but also directly. + * The only obj argument is enough for the most cases expect the + * source filters, so the data argument has been introduced and + * is used where appropriate. + */ +typedef void (*nxt_work_handler_t)(nxt_thread_t *thr, void *obj, void *data); + +typedef struct nxt_work_s nxt_work_t; + + +struct nxt_work_s { + nxt_work_t *next; + nxt_work_handler_t handler; + void *obj; + void *data; + nxt_log_t *log; +}; + + +typedef struct nxt_work_queue_chunk_s nxt_work_queue_chunk_t; + +struct nxt_work_queue_chunk_s { + nxt_work_queue_chunk_t *next; + nxt_work_t work; +}; + + +typedef struct { + nxt_work_t *next; + nxt_work_t *spare; + nxt_work_queue_chunk_t *chunk; + size_t chunk_size; +} nxt_work_queue_cache_t; + + +typedef struct nxt_work_queue_s nxt_work_queue_t; + +struct nxt_work_queue_s { + nxt_work_t *head; + nxt_work_t *tail; + nxt_work_queue_t *next; +#if (NXT_DEBUG) + const char *name; +#endif +}; + + +typedef struct { + nxt_work_queue_t *head; + nxt_work_queue_t *tail; + nxt_work_queue_t main; + nxt_work_queue_t last; + nxt_work_queue_cache_t cache; +} nxt_thread_work_queue_t; + + +typedef struct { + nxt_thread_spinlock_t lock; + nxt_work_t *head; + nxt_work_t *tail; + nxt_work_queue_cache_t cache; +} nxt_locked_work_queue_t; + + +NXT_EXPORT void nxt_thread_work_queue_create(nxt_thread_t *thr, + size_t chunk_size); +NXT_EXPORT void nxt_thread_work_queue_destroy(nxt_thread_t *thr); +NXT_EXPORT void nxt_thread_work_queue_add(nxt_thread_t *thr, + nxt_work_queue_t *wq, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log); +NXT_EXPORT void nxt_thread_work_queue_push(nxt_thread_t *thr, + nxt_work_queue_t *wq, nxt_work_handler_t handler, void *obj, void *data, + nxt_log_t *log); +NXT_EXPORT void nxt_work_queue_attach(nxt_thread_t *thr, nxt_work_queue_t *wq); +NXT_EXPORT nxt_work_handler_t nxt_thread_work_queue_pop(nxt_thread_t *thr, + void **obj, void **data, nxt_log_t **log); +NXT_EXPORT void nxt_thread_work_queue_drop(nxt_thread_t *thr, void *data); + + +#define \ +nxt_thread_current_work_queue_add(thr, handler, obj, data, log) \ + do { \ + nxt_thread_t *_thr = thr; \ + \ + nxt_thread_work_queue_add(_thr, _thr->work_queue.head, \ + handler, obj, data, log); \ + } while (0) + + +NXT_EXPORT void nxt_work_queue_destroy(nxt_work_queue_t *wq); + + +#if (NXT_DEBUG) + +#define \ +nxt_work_queue_name(_wq, _name) \ + (_wq)->name = _name + +#else + +#define \ +nxt_work_queue_name(_wq, _name) + +#endif + + +NXT_EXPORT void nxt_thread_last_work_queue_add(nxt_thread_t *thr, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log); +NXT_EXPORT nxt_work_handler_t nxt_thread_last_work_queue_pop(nxt_thread_t *thr, + void **obj, void **data, nxt_log_t **log); + + +NXT_EXPORT void nxt_locked_work_queue_create(nxt_locked_work_queue_t *lwq, + size_t chunk_size); +NXT_EXPORT void nxt_locked_work_queue_destroy(nxt_locked_work_queue_t *lwq); +NXT_EXPORT void nxt_locked_work_queue_add(nxt_locked_work_queue_t *lwq, + nxt_work_handler_t handler, void *obj, void *data, nxt_log_t *log); +NXT_EXPORT nxt_work_handler_t nxt_locked_work_queue_pop( + nxt_locked_work_queue_t *lwq, void **obj, void **data, nxt_log_t **log); +NXT_EXPORT void nxt_locked_work_queue_move(nxt_thread_t *thr, + nxt_locked_work_queue_t *lwq, nxt_work_queue_t *wq); + + +#endif /* _NXT_WORK_QUEUE_H_INCLUDED_ */ diff --git a/src/nxt_worker_process.c b/src/nxt_worker_process.c new file mode 100644 index 00000000..6f64b8f9 --- /dev/null +++ b/src/nxt_worker_process.c @@ -0,0 +1,213 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_cycle.h> +#include <nxt_process_chan.h> +#include <nxt_master_process.h> + + +static void nxt_worker_process_quit(nxt_thread_t *thr); +static void nxt_worker_process_quit_handler(nxt_thread_t *thr, + nxt_chan_recv_msg_t *msg); +static void nxt_worker_process_signal_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_worker_process_sigterm_handler(nxt_thread_t *thr, void *obj, + void *data); +static void nxt_worker_process_sigquit_handler(nxt_thread_t *thr, void *obj, + void *data); + + +static nxt_process_chan_handler_t nxt_worker_process_chan_handlers[] = { + nxt_worker_process_quit_handler, + nxt_process_chan_new_handler, + nxt_process_chan_change_log_file_handler, + nxt_process_chan_data_handler, +}; + + +static const nxt_event_sig_t nxt_worker_process_signals[] = { + nxt_event_signal(SIGHUP, nxt_worker_process_signal_handler), + nxt_event_signal(SIGINT, nxt_worker_process_sigterm_handler), + nxt_event_signal(SIGQUIT, nxt_worker_process_sigterm_handler), + nxt_event_signal(SIGTERM, nxt_worker_process_sigquit_handler), + nxt_event_signal(SIGCHLD, nxt_worker_process_signal_handler), + nxt_event_signal(SIGUSR1, nxt_worker_process_signal_handler), + nxt_event_signal(SIGUSR1, nxt_worker_process_signal_handler), + nxt_event_signal_end, +}; + + +void +nxt_worker_process_start(void *data) +{ + nxt_int_t n; + nxt_cycle_t *cycle; + nxt_thread_t *thr; + nxt_process_chan_t *proc; + const nxt_event_set_ops_t *event_set; + + cycle = data; + + nxt_thread_init_data(nxt_thread_cycle_data); + nxt_thread_cycle_set(cycle); + + thr = nxt_thread(); + + nxt_log_error(NXT_LOG_INFO, thr->log, "worker process"); + + nxt_process_title("nginman: worker process"); + + cycle->type = NXT_PROCESS_WORKER; + + nxt_random_init(&nxt_random_data); + + if (getuid() == 0) { + /* Super-user. */ + + n = nxt_user_cred_set(&cycle->user_cred); + if (n != NXT_OK) { + goto fail; + } + } + + /* Update inherited master process event engine and signals processing. */ + thr->engine->signals->sigev = nxt_worker_process_signals; + + event_set = nxt_service_get(cycle->services, "engine", cycle->engine); + if (event_set == NULL) { + goto fail; + } + + if (nxt_event_engine_change(thr, event_set, cycle->batch) != NXT_OK) { + goto fail; + } + +#if 0 + if (nxt_cycle_listen_sockets_enable(thr, cycle) != NXT_OK) { + goto fail; + } +#endif + + proc = cycle->processes->elts; + + /* A master process chan. */ + nxt_chan_read_close(proc[0].chan); + nxt_chan_write_enable(thr, proc[0].chan); + + /* A worker process chan. */ + nxt_process_chan_create(thr, &proc[cycle->current_process], + nxt_worker_process_chan_handlers); + +#if (NXT_THREADS) + { + nxt_int_t ret; + + ret = nxt_cycle_thread_pool_create(thr, cycle, cycle->auxiliary_threads, + 60000 * 1000000LL); + + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + } + + nxt_app_start(cycle); +#endif + + return; + +fail: + + exit(1); + nxt_unreachable(); +} + + +static void +nxt_worker_process_quit(nxt_thread_t *thr) +{ + nxt_uint_t n; + nxt_cycle_t *cycle; + nxt_queue_t *listen; + nxt_queue_link_t *link, *next; + nxt_listen_socket_t *ls; + nxt_event_conn_listen_t *cls; + + cycle = nxt_thread_cycle(); + + nxt_log_debug(thr->log, "close listen connections"); + + listen = &thr->engine->listen_connections; + + for (link = nxt_queue_first(listen); + link != nxt_queue_tail(listen); + link = next) + { + next = nxt_queue_next(link); + cls = nxt_queue_link_data(link, nxt_event_conn_listen_t, link); + nxt_queue_remove(link); + + nxt_event_fd_close(thr->engine, &cls->socket); + } + + if (cycle->listen_sockets != NULL) { + + ls = cycle->listen_sockets->elts; + n = cycle->listen_sockets->nelts; + + while (n != 0) { + nxt_socket_close(ls->socket); + ls->socket = -1; + + ls++; + n--; + } + + cycle->listen_sockets->nelts = 0; + } + + nxt_cycle_quit(thr, cycle); +} + + +static void +nxt_worker_process_signal_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_log_error(NXT_LOG_INFO, thr->log, + "signal signo:%d (%s) recevied, ignored", + (int) (uintptr_t) obj, data); +} + + +static void +nxt_worker_process_quit_handler(nxt_thread_t *thr, nxt_chan_recv_msg_t *msg) +{ + nxt_worker_process_quit(thr); +} + + +static void +nxt_worker_process_sigterm_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_log_debug(thr->log, "sigterm handler signo:%d (%s)", + (int) (uintptr_t) obj, data); + + /* A fast exit. */ + + nxt_cycle_quit(thr, NULL); +} + + +static void +nxt_worker_process_sigquit_handler(nxt_thread_t *thr, void *obj, void *data) +{ + nxt_log_debug(thr->log, "sigquit handler signo:%d (%s)", + (int) (uintptr_t) obj, data); + + /* A graceful exit. */ + + nxt_worker_process_quit(thr); +} |