conf, router: Make the listen(2) backlog configurable

@oopsoop2 on GitHub reported a performance issue related to the default
listen(2) backlog size of 511 on nginx. They found that increasing it
helped, nginx has a config option to configure this.

They would like to be able to do the same on Unit (which also defaults
to 511 on some systems). This seems reasonable.

NOTE: On Linux before commit 97c15fa38 ("socket: Use a default listen
backlog of -1 on Linux") we defaulted to 511. Since that commit we
default to the Kernels default, which before 5.4 is 128 and after is
4096.

This adds a new per-listener 'backlog' config option, e.g

  {
      "listeners": {
          "[::1]:8080": {
              "pass": "routes",
              "backlog": 1024
          },
      }

      ...
  }

This doesn't effect the control socket.

Closes: https://github.com/nginx/unit/issues/1384
Reported-by: <https://github.com/oopsoop2>
Signed-off-by: Andrew Clayton <a.clayton@nginx.com>

2 files changed, 50 insertions, 9 deletions

diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c
index c9c51ac1..8f31bd18 100644
--- a/src/nxt_conf_validation.c
+++ b/src/nxt_conf_validation.c
@@ -178,6 +178,8 @@ static nxt_int_t nxt_conf_vldt_app_name(nxt_conf_validation_t *vldt,
     nxt_conf_value_t *value, void *data);
 static nxt_int_t nxt_conf_vldt_forwarded(nxt_conf_validation_t *vldt,
     nxt_conf_value_t *value, void *data);
+static nxt_int_t nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt,
+    nxt_conf_value_t *value, void *data);
 static nxt_int_t nxt_conf_vldt_app(nxt_conf_validation_t *vldt,
     nxt_str_t *name, nxt_conf_value_t *value);
 static nxt_int_t nxt_conf_vldt_object(nxt_conf_validation_t *vldt,
@@ -430,6 +432,10 @@ static nxt_conf_vldt_object_t  nxt_conf_vldt_listener_members[] = {
         .type       = NXT_CONF_VLDT_OBJECT,
         .validator  = nxt_conf_vldt_object,
         .u.members  = nxt_conf_vldt_client_ip_members
+    }, {
+        .name       = nxt_string("backlog"),
+        .type       = NXT_CONF_VLDT_NUMBER,
+        .validator  = nxt_conf_vldt_listen_backlog,
     },
 
 #if (NXT_TLS)
@@ -2705,6 +2711,32 @@ nxt_conf_vldt_forwarded(nxt_conf_validation_t *vldt, nxt_conf_value_t *value,
 
 
 static nxt_int_t
+nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt,
+    nxt_conf_value_t *value, void *data)
+{
+    int64_t  backlog;
+
+    backlog = nxt_conf_get_number(value);
+
+    /*
+     * POSIX allows this to be 0 and some systems use -1 to
+     * indicate to use the OS's default value.
+     */
+    if (backlog < -1) {
+        return nxt_conf_vldt_error(vldt, "The \"backlog\" number must be "
+                                   "equal to or greater than -1.");
+    }
+
+    if (backlog > NXT_INT32_T_MAX) {
+        return nxt_conf_vldt_error(vldt, "The \"backlog\" number must "
+                                   "not exceed %d.", NXT_INT32_T_MAX);
+    }
+
+    return NXT_OK;
+}
+
+
+static nxt_int_t
 nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name,
     nxt_conf_value_t *value)
 {
diff --git a/src/nxt_router.c b/src/nxt_router.c
index c8ba4744..076cd134 100644
--- a/src/nxt_router.c
+++ b/src/nxt_router.c
@@ -40,6 +40,7 @@ typedef struct {
 typedef struct {
     nxt_str_t         pass;
     nxt_str_t         application;
+    int               backlog;
 } nxt_router_listener_conf_t;
 
 
@@ -166,7 +167,7 @@ static void nxt_router_app_prefork_ready(nxt_task_t *task,
 static void nxt_router_app_prefork_error(nxt_task_t *task,
     nxt_port_recv_msg_t *msg, void *data);
 static nxt_socket_conf_t *nxt_router_socket_conf(nxt_task_t *task,
-    nxt_router_temp_conf_t *tmcf, nxt_str_t *name);
+    nxt_router_temp_conf_t *tmcf, nxt_str_t *name, int backlog);
 static nxt_int_t nxt_router_listen_socket_find(nxt_router_temp_conf_t *tmcf,
     nxt_socket_conf_t *nskcf, nxt_sockaddr_t *sa);
 
@@ -1494,6 +1495,12 @@ static nxt_conf_map_t  nxt_router_listener_conf[] = {
         NXT_CONF_MAP_STR_COPY,
         offsetof(nxt_router_listener_conf_t, application),
     },
+
+    {
+        nxt_string("backlog"),
+        NXT_CONF_MAP_INT32,
+        offsetof(nxt_router_listener_conf_t, backlog),
+    },
 };
 
 
@@ -1968,13 +1975,10 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
                 break;
             }
 
-            skcf = nxt_router_socket_conf(task, tmcf, &name);
-            if (skcf == NULL) {
-                goto fail;
-            }
-
             nxt_memzero(&lscf, sizeof(lscf));
 
+            lscf.backlog = -1;
+
             ret = nxt_conf_map_object(mp, listener, nxt_router_listener_conf,
                                       nxt_nitems(nxt_router_listener_conf),
                                       &lscf);
@@ -1985,6 +1989,11 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
 
             nxt_debug(task, "application: %V", &lscf.application);
 
+            skcf = nxt_router_socket_conf(task, tmcf, &name, lscf.backlog);
+            if (skcf == NULL) {
+                goto fail;
+            }
+
             // STUB, default values if http block is not defined.
             skcf->header_buffer_size = 2048;
             skcf->large_header_buffer_size = 8192;
@@ -2688,7 +2697,7 @@ nxt_router_application_init(nxt_router_conf_t *rtcf, nxt_str_t *name,
 
 static nxt_socket_conf_t *
 nxt_router_socket_conf(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
-    nxt_str_t *name)
+    nxt_str_t *name, int backlog)
 {
     size_t               size;
     nxt_int_t            ret;
@@ -2732,7 +2741,7 @@ nxt_router_socket_conf(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
         nxt_listen_socket_remote_size(ls);
 
         ls->socket = -1;
-        ls->backlog = NXT_LISTEN_BACKLOG;
+        ls->backlog = backlog > -1 ? backlog : NXT_LISTEN_BACKLOG;
         ls->flags = NXT_NONBLOCK;
         ls->read_after_accept = 1;
     }
@@ -2879,7 +2888,7 @@ nxt_router_listen_socket_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg,
 
     nxt_socket_defer_accept(task, s, rpc->socket_conf->listen->sockaddr);
 
-    ret = nxt_listen_socket(task, s, NXT_LISTEN_BACKLOG);
+    ret = nxt_listen_socket(task, s, rpc->socket_conf->listen->backlog);
     if (nxt_slow_path(ret != NXT_OK)) {
         goto fail;
     }