Fixed crash that occurs when idle connections are closed forcibly.

3 files changed, 77 insertions, 35 deletions

diff --git a/src/nxt_conn_accept.c b/src/nxt_conn_accept.c
index 4ad2d02f..d4c3942c 100644
--- a/src/nxt_conn_accept.c
+++ b/src/nxt_conn_accept.c
@@ -24,8 +24,10 @@ static void nxt_conn_listen_handler(nxt_task_t *task, void *obj,
     void *data);
 static nxt_conn_t *nxt_conn_accept_next(nxt_task_t *task,
     nxt_listen_event_t *lev);
-static nxt_int_t nxt_conn_accept_close_idle(nxt_task_t *task,
+static void nxt_conn_accept_close_idle(nxt_task_t *task,
     nxt_listen_event_t *lev);
+static void nxt_conn_accept_close_idle_handler(nxt_task_t *task, void *obj,
+    void *data);
 static void nxt_conn_listen_event_error(nxt_task_t *task, void *obj,
     void *data);
 static void nxt_conn_listen_timer_handler(nxt_task_t *task, void *obj,
@@ -230,60 +232,76 @@ nxt_conn_accept_next(nxt_task_t *task, nxt_listen_event_t *lev)
 
     lev->next = NULL;
 
-    do {
-        c = nxt_conn_accept_alloc(task, lev);
+    c = nxt_conn_accept_alloc(task, lev);
 
-        if (nxt_fast_path(c != NULL)) {
-            return c;
-        }
+    if (nxt_slow_path(c == NULL)) {
+        nxt_conn_accept_close_idle(task, lev);
+    }
 
-    } while (nxt_conn_accept_close_idle(task, lev) == NXT_OK);
+    return c;
+}
 
-    nxt_alert(task, "no available connections, "
-              "new connections are not accepted within 1s");
 
-    return NULL;
+static void
+nxt_conn_accept_close_idle(nxt_task_t *task, nxt_listen_event_t *lev)
+{
+    nxt_event_engine_t  *engine;
+
+    engine = task->thread->engine;
+
+    nxt_work_queue_add(&engine->close_work_queue,
+                       nxt_conn_accept_close_idle_handler, task, NULL, NULL);
+
+    nxt_timer_add(engine, &lev->timer, 100);
+
+    nxt_fd_event_disable_read(engine, &lev->socket);
+
+    nxt_alert(task, "new connections are not accepted within 100ms");
 }
 
 
-static nxt_int_t
-nxt_conn_accept_close_idle(nxt_task_t *task, nxt_listen_event_t *lev)
+static void
+nxt_conn_accept_close_idle_handler(nxt_task_t *task, void *obj, void *data)
 {
+    nxt_uint_t          times;
     nxt_conn_t          *c;
     nxt_queue_t         *idle;
-    nxt_queue_link_t    *link;
+    nxt_queue_link_t    *link, *next;
     nxt_event_engine_t  *engine;
 
     static nxt_log_moderation_t  nxt_idle_close_log_moderation = {
         NXT_LOG_INFO, 2, "idle connections closed", NXT_LOG_MODERATION
     };
 
+    times = 10;
     engine = task->thread->engine;
-
     idle = &engine->idle_connections;
 
     for (link = nxt_queue_last(idle);
          link != nxt_queue_head(idle);
-         link = nxt_queue_next(link))
+         link = next)
     {
+        next = nxt_queue_next(link);
+
         c = nxt_queue_link_data(link, nxt_conn_t, link);
 
+        nxt_debug(c->socket.task, "idle connection: %d rdy:%d",
+                  c->socket.fd, c->socket.read_ready);
+
         if (!c->socket.read_ready) {
             nxt_log_moderate(&nxt_idle_close_log_moderation, NXT_LOG_INFO,
                              task->log, "no available connections, "
                              "close idle connection");
-            nxt_queue_remove(link);
-            nxt_conn_close(engine, c);
 
-            return NXT_OK;
-        }
-    }
+            c->read_state->close_handler(c->socket.task, c, c->socket.data);
 
-    nxt_timer_add(engine, &lev->timer, 1000);
+            times--;
 
-    nxt_fd_event_disable_read(engine, &lev->socket);
-
-    return NXT_DECLINED;
+            if (times == 0) {
+                break;
+            }
+        }
+    }
 }
 
 
@@ -313,12 +331,10 @@ nxt_conn_accept_error(nxt_task_t *task, nxt_listen_event_t *lev,
     case ENFILE:
     case ENOBUFS:
     case ENOMEM:
-        if (nxt_conn_accept_close_idle(task, lev) != NXT_OK) {
-            nxt_alert(task, "%s(%d) failed %E, "
-                      "new connections are not accepted within 1s",
-                      accept_syscall, lev->socket.fd, err);
-        }
+        nxt_alert(task, "%s(%d) failed %E",
+                  accept_syscall, lev->socket.fd, err);
 
+        nxt_conn_accept_close_idle(task, lev);
         return;
 
     default:
diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c
index 5e3b2f82..c2e65397 100644
--- a/src/nxt_h1proto.c
+++ b/src/nxt_h1proto.c
@@ -74,6 +74,8 @@ static void nxt_h1p_idle_close(nxt_task_t *task, void *obj, void *data);
 static void nxt_h1p_idle_timeout(nxt_task_t *task, void *obj, void *data);
 static void nxt_h1p_idle_response(nxt_task_t *task, nxt_conn_t *c);
 static void nxt_h1p_idle_response_sent(nxt_task_t *task, void *obj, void *data);
+static void nxt_h1p_idle_response_error(nxt_task_t *task, void *obj,
+    void *data);
 static void nxt_h1p_idle_response_timeout(nxt_task_t *task, void *obj,
     void *data);
 static nxt_msec_t nxt_h1p_idle_response_timer_value(nxt_conn_t *c,
@@ -470,6 +472,8 @@ nxt_h1p_conn_request_init(nxt_task_t *task, void *obj, void *data)
 
     nxt_debug(task, "h1p conn request init");
 
+    nxt_queue_remove(&c->link);
+
     r = nxt_http_request_create(task);
 
     if (nxt_fast_path(r != NULL)) {
@@ -1714,6 +1718,8 @@ nxt_h1p_conn_close(nxt_task_t *task, void *obj, void *data)
 
     nxt_debug(task, "h1p conn close");
 
+    nxt_queue_remove(&c->link);
+
     nxt_h1p_shutdown(task, c);
 }
 
@@ -1727,6 +1733,8 @@ nxt_h1p_conn_error(nxt_task_t *task, void *obj, void *data)
 
     nxt_debug(task, "h1p conn error");
 
+    nxt_queue_remove(&c->link);
+
     nxt_h1p_shutdown(task, c);
 }
 
@@ -1745,8 +1753,9 @@ nxt_h1p_conn_timer_value(nxt_conn_t *c, uintptr_t data)
 static void
 nxt_h1p_keepalive(nxt_task_t *task, nxt_h1proto_t *h1p, nxt_conn_t *c)
 {
-    size_t     size;
-    nxt_buf_t  *in;
+    size_t              size;
+    nxt_buf_t           *in;
+    nxt_event_engine_t  *engine;
 
     nxt_debug(task, "h1p keepalive");
 
@@ -1762,10 +1771,13 @@ nxt_h1p_keepalive(nxt_task_t *task, nxt_h1proto_t *h1p, nxt_conn_t *c)
 
     c->sent = 0;
 
+    engine = task->thread->engine;
+    nxt_queue_insert_head(&engine->idle_connections, &c->link);
+
     if (in == NULL) {
         c->read_state = &nxt_h1p_keepalive_state;
 
-        nxt_conn_read(task->thread->engine, c);
+        nxt_conn_read(engine, c);
 
     } else {
         size = nxt_buf_mem_used_size(&in->mem);
@@ -1831,6 +1843,8 @@ nxt_h1p_idle_timeout(nxt_task_t *task, void *obj, void *data)
     c = nxt_read_timer_conn(timer);
     c->block_read = 1;
 
+    nxt_queue_remove(&c->link);
+
     nxt_h1p_idle_response(task, c);
 }
 
@@ -1898,7 +1912,7 @@ static const nxt_conn_state_t  nxt_h1p_timeout_response_state
     nxt_aligned(64) =
 {
     .ready_handler = nxt_h1p_conn_sent,
-    .error_handler = nxt_h1p_conn_error,
+    .error_handler = nxt_h1p_idle_response_error,
 
     .timer_handler = nxt_h1p_idle_response_timeout,
     .timer_value = nxt_h1p_idle_response_timer_value,
@@ -1919,6 +1933,19 @@ nxt_h1p_idle_response_sent(nxt_task_t *task, void *obj, void *data)
 
 
 static void
+nxt_h1p_idle_response_error(nxt_task_t *task, void *obj, void *data)
+{
+    nxt_conn_t  *c;
+
+    c = obj;
+
+    nxt_debug(task, "h1p response error");
+
+    nxt_h1p_shutdown(task, c);
+}
+
+
+static void
 nxt_h1p_idle_response_timeout(nxt_task_t *task, void *obj, void *data)
 {
     nxt_conn_t   *c;
@@ -2057,8 +2084,6 @@ nxt_h1p_conn_free(nxt_task_t *task, void *obj, void *data)
 
     nxt_debug(task, "h1p conn free");
 
-    nxt_queue_remove(&c->link);
-
     engine = task->thread->engine;
 
     nxt_sockaddr_cache_free(engine, c);
diff --git a/src/nxt_log_moderation.c b/src/nxt_log_moderation.c
index 7c2d7a50..95f9cbfe 100644
--- a/src/nxt_log_moderation.c
+++ b/src/nxt_log_moderation.c
@@ -61,6 +61,7 @@ nxt_log_moderate_allow(nxt_log_moderation_t *mod)
         mod->timer.work_queue = &thr->engine->fast_work_queue;
         mod->timer.handler = nxt_log_moderate_timer_handler;
         mod->timer.log = &nxt_main_log;
+        mod->timer.task = &nxt_main_task;
 
         nxt_timer_add(thr->engine, &mod->timer, 1000);
     }