router: Make the number of router threads configurable

Unit generally creates an extra number of router threads (to handle
client connections, not incl the main thread) to match the number of
available CPUs.

There are cases when this can go wrong, e.g on a high CPU count machine
and Unit is being effectively limited to a few CPUs via the cgroups cpu
controller. So Unit may create a large number of router threads when
they are only going to effectively run on a couple of CPUs or so.

There may be other cases where you would like to tweak the number of
router threads, depending on your workload.

As it turns out it looks like it was intended to be made configurable
but was just never hooked up to the config system.

This adds a new '/settings/listen_threads' config option which can be
set like

  {
      "listen": {
          ...
      },

      "settings": {
          "listen_threads": 2,

          ...
      },

      ...
  }

Before this patch (on a four cpu system)

  $  ps -efL | grep router
  andrew   419832 419829 419832  0    5 Aug12 pts/10   00:00:00 unit: router
  andrew   419832 419829 419833  0    5 Aug12 pts/10   00:00:00 unit: router
  andrew   419832 419829 419834  0    5 Aug12 pts/10   00:00:00 unit: router
  andrew   419832 419829 445145  0    5 03:31 pts/10   00:00:00 unit: router
  andrew   419832 419829 445146  0    5 03:31 pts/10   00:00:00 unit: router

After, with a threads setting of 2

  $ ps -efL | grep router
  andrew   419832 419829 419832  0    3 Aug12 pts/10   00:00:00 unit: router
  andrew   419832 419829 419833  0    3 Aug12 pts/10   00:00:00 unit: router
  andrew   419832 419829 419834  0    3 Aug12 pts/10   00:00:00 unit: router

Closes: https://github.com/nginx/unit/issues/1042
Signed-off-by: Andrew Clayton <a.clayton@nginx.com>

2 files changed, 38 insertions, 7 deletions

diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c
index 04091745..c9c51ac1 100644
--- a/src/nxt_conf_validation.c
+++ b/src/nxt_conf_validation.c
@@ -134,6 +134,8 @@ static nxt_int_t nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt,
     nxt_conf_value_t *value, void *data);
 static nxt_int_t nxt_conf_vldt_python_prefix(nxt_conf_validation_t *vldt,
     nxt_conf_value_t *value, void *data);
+static nxt_int_t nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt,
+    nxt_conf_value_t *value, void *data);
 static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt,
     nxt_conf_value_t *value, void *data);
 static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt,
@@ -305,6 +307,10 @@ static nxt_conf_vldt_object_t  nxt_conf_vldt_root_members[] = {
 
 static nxt_conf_vldt_object_t  nxt_conf_vldt_setting_members[] = {
     {
+        .name       = nxt_string("listen_threads"),
+        .type       = NXT_CONF_VLDT_INTEGER,
+        .validator  = nxt_conf_vldt_listen_threads,
+    }, {
         .name       = nxt_string("http"),
         .type       = NXT_CONF_VLDT_OBJECT,
         .validator  = nxt_conf_vldt_object,
@@ -2078,6 +2084,27 @@ nxt_conf_vldt_python_prefix(nxt_conf_validation_t *vldt,
     return NXT_OK;
 }
 
+static nxt_int_t
+nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt,
+    nxt_conf_value_t *value, void *data)
+{
+    int64_t  threads;
+
+    threads = nxt_conf_get_number(value);
+
+    if (threads < 1) {
+        return nxt_conf_vldt_error(vldt, "The \"listen_threads\" number must "
+                                   "be equal to or greater than 1.");
+    }
+
+    if (threads > NXT_INT32_T_MAX) {
+        return nxt_conf_vldt_error(vldt, "The \"listen_threads\" number must "
+                                   "not exceed %d.", NXT_INT32_T_MAX);
+    }
+
+    return NXT_OK;
+}
+
 
 static nxt_int_t
 nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value,
diff --git a/src/nxt_router.c b/src/nxt_router.c
index 43209451..c8ba4744 100644
--- a/src/nxt_router.c
+++ b/src/nxt_router.c
@@ -1412,7 +1412,7 @@ nxt_router_conf_send(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
 
 static nxt_conf_map_t  nxt_router_conf[] = {
     {
-        nxt_string("listeners_threads"),
+        nxt_string("listen_threads"),
         NXT_CONF_MAP_INT32,
         offsetof(nxt_router_conf_t, threads),
     },
@@ -1630,7 +1630,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
     nxt_conf_value_t            *js_module;
 #endif
     nxt_conf_value_t            *root, *conf, *http, *value, *websocket;
-    nxt_conf_value_t            *applications, *application;
+    nxt_conf_value_t            *applications, *application, *settings;
     nxt_conf_value_t            *listeners, *listener;
     nxt_socket_conf_t           *skcf;
     nxt_router_conf_t           *rtcf;
@@ -1640,6 +1640,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
     nxt_router_app_conf_t       apcf;
     nxt_router_listener_conf_t  lscf;
 
+    static const nxt_str_t  settings_path = nxt_string("/settings");
     static const nxt_str_t  http_path = nxt_string("/settings/http");
     static const nxt_str_t  applications_path = nxt_string("/applications");
     static const nxt_str_t  listeners_path = nxt_string("/listeners");
@@ -1673,11 +1674,14 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
     rtcf = tmcf->router_conf;
     mp = rtcf->mem_pool;
 
-    ret = nxt_conf_map_object(mp, root, nxt_router_conf,
-                              nxt_nitems(nxt_router_conf), rtcf);
-    if (ret != NXT_OK) {
-        nxt_alert(task, "root map error");
-        return NXT_ERROR;
+    settings = nxt_conf_get_path(root, &settings_path);
+    if (settings != NULL) {
+        ret = nxt_conf_map_object(mp, settings, nxt_router_conf,
+                                  nxt_nitems(nxt_router_conf), rtcf);
+        if (ret != NXT_OK) {
+            nxt_alert(task, "router_conf map error");
+            return NXT_ERROR;
+        }
     }
 
     if (rtcf->threads == 0) {